11#ifndef EIGEN_TENSOR_TENSOR_CONTRACTION_H
12#define EIGEN_TENSOR_TENSOR_CONTRACTION_H
15#include "./InternalHeaderCheck.h"
21template <
typename Dimensions,
typename LhsXprType,
typename RhsXprType,
typename OutputKernelType>
22struct traits<TensorContractionOp<Dimensions, LhsXprType, RhsXprType, OutputKernelType>> {
24 using Scalar =
typename gebp_traits<std::remove_const_t<typename LhsXprType::Scalar>,
25 std::remove_const_t<typename RhsXprType::Scalar>>::ResScalar;
27 using StorageKind =
typename promote_storage_type<typename traits<LhsXprType>::StorageKind,
28 typename traits<RhsXprType>::StorageKind>::ret;
30 typename promote_index_type<typename traits<LhsXprType>::Index,
typename traits<RhsXprType>::Index>::type;
32 static constexpr int NumDimensions =
33 traits<LhsXprType>::NumDimensions + traits<RhsXprType>::NumDimensions - 2 * array_size<Dimensions>::value;
34 static constexpr int Layout = traits<LhsXprType>::Layout;
36 std::conditional_t<Pointer_type_promotion<typename LhsXprType::Scalar, Scalar>::val,
37 typename traits<LhsXprType>::PointerType,
typename traits<RhsXprType>::PointerType>;
39 static constexpr int Flags = 0;
42template <
typename Dimensions,
typename LhsXprType,
typename RhsXprType,
typename OutputKernelType>
43struct eval<TensorContractionOp<Dimensions, LhsXprType, RhsXprType, OutputKernelType>, Eigen::Dense> {
44 using type =
const TensorContractionOp<Dimensions, LhsXprType, RhsXprType, OutputKernelType>&;
47template <
typename Indices_,
typename LeftArgType_,
typename RightArgType_,
typename OutputKernelType_,
50 TensorEvaluator<const TensorContractionOp<Indices_, LeftArgType_, RightArgType_, OutputKernelType_>, Device_>> {
51 using Indices = Indices_;
52 using LeftArgType = LeftArgType_;
53 using RightArgType = RightArgType_;
54 using OutputKernelType = OutputKernelType_;
55 using Device = Device_;
58 static constexpr int NumDimensions =
59 traits<LeftArgType_>::NumDimensions + traits<RightArgType_>::NumDimensions - 2 * array_size<Indices_>::value;
63template <
typename LhsScalar,
typename RhsScalar,
64 bool RequireInitialization =
65 NumTraits<LhsScalar>::RequireInitialization || NumTraits<RhsScalar>::RequireInitialization>
66struct TensorContractionBlockMemAllocator {
67 using BlockMemHandle =
void*;
69 template <
typename Device>
70 EIGEN_DEVICE_FUNC
static BlockMemHandle allocate(Device& d,
const Index bm,
const Index bk,
const Index bn,
71 LhsScalar** lhs_block, RhsScalar** rhs_block) {
72 eigen_assert(lhs_block);
73 eigen_assert(rhs_block);
74 BlockSizes sz = ComputeLhsRhsBlockSizes(bm, bk, bn);
75 char* block_mem =
static_cast<char*
>(d.allocate(sz.lhs_size + sz.rhs_size));
76 *lhs_block =
static_cast<LhsScalar*
>(
static_cast<void*
>(block_mem));
77 *rhs_block =
static_cast<RhsScalar*
>(
static_cast<void*
>(block_mem + sz.lhs_size));
81 template <
typename Device>
82 EIGEN_DEVICE_FUNC
static BlockMemHandle allocateSlices(Device& d,
const Index bm,
const Index bk,
const Index bn,
83 const Index num_lhs,
const Index num_rhs,
84 const Index num_slices, std::vector<LhsScalar*>* lhs_blocks,
85 std::vector<RhsScalar*>* rhs_blocks) {
86 eigen_assert(num_slices > 0);
87 eigen_assert(num_lhs >= 0 && num_rhs >= 0);
88 eigen_assert(num_lhs == 0 || lhs_blocks);
89 eigen_assert(num_rhs == 0 || rhs_blocks);
90 BlockSizes sz = ComputeLhsRhsBlockSizes(bm, bk, bn);
91 void* block_mem = d.allocate((num_lhs * sz.lhs_size + num_rhs * sz.rhs_size) * num_slices);
92 eigen_assert(block_mem);
93 char* mem =
static_cast<char*
>(block_mem);
95 for (Index x = 0; x < num_slices; x++) {
96 if (num_lhs > 0) lhs_blocks[x].resize(num_lhs);
97 for (Index m = 0; m < num_lhs; m++) {
98 lhs_blocks[x][m] =
static_cast<LhsScalar*
>(
static_cast<void*
>(mem));
101 if (num_rhs > 0) rhs_blocks[x].resize(num_rhs);
102 for (Index n = 0; n < num_rhs; n++) {
103 rhs_blocks[x][n] =
static_cast<RhsScalar*
>(
static_cast<void*
>(mem));
111 template <
typename Device>
112 EIGEN_DEVICE_FUNC
static void deallocate(Device& d, BlockMemHandle handle) {
113 d.deallocate(handle);
121 EIGEN_DEVICE_FUNC
static BlockSizes ComputeLhsRhsBlockSizes(
const Index bm,
const Index bk,
const Index bn) {
122 Index align = numext::maxi(EIGEN_MAX_ALIGN_BYTES, 1);
124 sz.lhs_size = numext::div_ceil<Index>(bm * bk *
sizeof(LhsScalar), align) * align;
125 sz.rhs_size = numext::div_ceil<Index>(bn * bk *
sizeof(RhsScalar), align) * align;
132template <
typename LhsScalar,
typename RhsScalar>
133struct TensorContractionBlockMemAllocator<LhsScalar, RhsScalar, true> {
134 using RawAllocator = TensorContractionBlockMemAllocator<LhsScalar, RhsScalar, false>;
136 struct BlockMemHandle {
138 Index lhs_block_size = 0;
139 Index rhs_block_size = 0;
140 std::vector<LhsScalar*> lhs_blocks;
141 std::vector<RhsScalar*> rhs_blocks;
144 template <
typename Device>
145 static BlockMemHandle allocate(Device& d,
const Index bm,
const Index bk,
const Index bn, LhsScalar** lhs_block,
146 RhsScalar** rhs_block) {
147 BlockMemHandle handle;
148 handle.mem = RawAllocator::allocate(d, bm, bk, bn, lhs_block, rhs_block);
150 handle.lhs_blocks.push_back(*lhs_block);
151 handle.rhs_blocks.push_back(*rhs_block);
152 construct(handle, bm * bk, bn * bk);
155 RawAllocator::deallocate(d, handle.mem);
161 template <
typename Device>
162 static BlockMemHandle allocateSlices(Device& d,
const Index bm,
const Index bk,
const Index bn,
const Index num_lhs,
163 const Index num_rhs,
const Index num_slices, std::vector<LhsScalar*>* lhs_blocks,
164 std::vector<RhsScalar*>* rhs_blocks) {
165 BlockMemHandle handle;
166 handle.mem = RawAllocator::allocateSlices(d, bm, bk, bn, num_lhs, num_rhs, num_slices, lhs_blocks, rhs_blocks);
168 for (Index x = 0; x < num_slices; x++) {
169 if (num_lhs > 0) handle.lhs_blocks.insert(handle.lhs_blocks.end(), lhs_blocks[x].begin(), lhs_blocks[x].end());
170 if (num_rhs > 0) handle.rhs_blocks.insert(handle.rhs_blocks.end(), rhs_blocks[x].begin(), rhs_blocks[x].end());
172 construct(handle, bm * bk, bn * bk);
175 RawAllocator::deallocate(d, handle.mem);
181 template <
typename Device>
182 static void deallocate(Device& d,
const BlockMemHandle& handle) {
183 for (LhsScalar* block : handle.lhs_blocks) destruct_elements_of_array(block, handle.lhs_block_size);
184 for (RhsScalar* block : handle.rhs_blocks) destruct_elements_of_array(block, handle.rhs_block_size);
185 RawAllocator::deallocate(d, handle.mem);
190 static void construct(BlockMemHandle& handle,
const Index lhs_block_size,
const Index rhs_block_size) {
191 std::size_t lhs_done = 0;
192 std::size_t rhs_done = 0;
194 for (; lhs_done < handle.lhs_blocks.size(); ++lhs_done) {
195 default_construct_elements_of_array(handle.lhs_blocks[lhs_done], lhs_block_size);
197 for (; rhs_done < handle.rhs_blocks.size(); ++rhs_done) {
198 default_construct_elements_of_array(handle.rhs_blocks[rhs_done], rhs_block_size);
202 while (rhs_done > 0) destruct_elements_of_array(handle.rhs_blocks[--rhs_done], rhs_block_size);
203 while (lhs_done > 0) destruct_elements_of_array(handle.lhs_blocks[--lhs_done], lhs_block_size);
206 handle.lhs_block_size = lhs_block_size;
207 handle.rhs_block_size = rhs_block_size;
239template <
typename ResScalar,
typename LhsScalar,
typename RhsScalar,
typename StorageIndex,
typename OutputMapper,
240 typename LhsMapper,
typename RhsMapper>
241struct TensorContractionKernel {
244 static constexpr bool HasBeta =
false;
246 EIGEN_DEVICE_FUNC TensorContractionKernel(StorageIndex m_, StorageIndex k_, StorageIndex n_, StorageIndex bm_,
247 StorageIndex bk_, StorageIndex bn_)
248 : m(m_), k(k_), n(n_), bm(bm_), bk(bk_), bn(bn_) {}
251 using LhsBlock = LhsScalar*;
252 using RhsBlock = RhsScalar*;
255 using BlockMemAllocator = TensorContractionBlockMemAllocator<LhsScalar, RhsScalar>;
256 using BlockMemHandle =
typename BlockMemAllocator::BlockMemHandle;
258 using Traits =
typename internal::gebp_traits<LhsScalar, RhsScalar>;
260 using LhsPacker = internal::gemm_pack_lhs<LhsScalar, StorageIndex,
typename LhsMapper::SubMapper, Traits::mr,
261 Traits::LhsProgress,
typename Traits::LhsPacket4Packing,
ColMajor>;
264 internal::gemm_pack_rhs<RhsScalar, StorageIndex, typename RhsMapper::SubMapper, Traits::nr, ColMajor>;
266 using GebpKernel = internal::gebp_kernel<LhsScalar, RhsScalar, StorageIndex, OutputMapper, Traits::mr, Traits::nr,
269 template <
typename Device>
270 EIGEN_DEVICE_FUNC BlockMemHandle allocate(Device& d, LhsBlock* lhs_block, RhsBlock* rhs_block) {
271 return BlockMemAllocator::allocate(d, bm, bk, bn, lhs_block, rhs_block);
274 template <
typename Device>
275 EIGEN_DEVICE_FUNC BlockMemHandle allocateSlices(Device& d,
const StorageIndex num_lhs,
const StorageIndex num_rhs,
276 const StorageIndex num_slices, std::vector<LhsBlock>* lhs_blocks,
277 std::vector<RhsBlock>* rhs_blocks) {
278 return BlockMemAllocator::allocateSlices(d, bm, bk, bn, num_lhs, num_rhs, num_slices, lhs_blocks, rhs_blocks);
281 template <
typename Device>
282 EIGEN_DEVICE_FUNC
static void deallocate(Device& d,
const BlockMemHandle& handle) {
283 BlockMemAllocator::deallocate(d, handle);
286 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE
void packLhs(LhsBlock* lhsBlock,
const typename LhsMapper::SubMapper& data_mapper,
287 const StorageIndex depth,
const StorageIndex rows) {
288 LhsPacker()(*lhsBlock, data_mapper, depth, rows, 0,
292 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE
void packRhs(RhsBlock* rhsBlock,
const typename RhsMapper::SubMapper& data_mapper,
293 const StorageIndex depth,
const StorageIndex cols) {
294 RhsPacker()(*rhsBlock, data_mapper, depth, cols);
297 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE
void invoke(
const OutputMapper& output_mapper,
const LhsBlock& lhsBlock,
298 const RhsBlock& rhsBlock,
const StorageIndex rows,
299 const StorageIndex depth,
const StorageIndex cols,
300 const ResScalar alpha,
const ResScalar beta) {
302 EIGEN_ONLY_USED_FOR_DEBUG(beta);
303 eigen_assert(beta == ResScalar(1));
304 static constexpr int kComputeStrideFromBlockDimensions = -1;
305 GebpKernel()(output_mapper, lhsBlock, rhsBlock, rows, depth, cols, alpha,
306 kComputeStrideFromBlockDimensions,
307 kComputeStrideFromBlockDimensions,
315 const StorageIndex m;
316 const StorageIndex k;
317 const StorageIndex n;
318 const StorageIndex bm;
319 const StorageIndex bk;
320 const StorageIndex bn;
327template <
typename Func>
328EIGEN_STRONG_INLINE
void tensor_contraction_dispatch(Func&& fn,
bool lhs_inner_dim_contiguous,
329 bool rhs_inner_dim_contiguous,
bool rhs_inner_dim_reordered) {
330 if (lhs_inner_dim_contiguous) {
331 if (rhs_inner_dim_contiguous) {
332 if (rhs_inner_dim_reordered)
333 fn(bool_constant<true>{}, bool_constant<true>{}, bool_constant<true>{});
335 fn(bool_constant<true>{}, bool_constant<true>{}, bool_constant<false>{});
337 if (rhs_inner_dim_reordered)
338 fn(bool_constant<true>{}, bool_constant<false>{}, bool_constant<true>{});
340 fn(bool_constant<true>{}, bool_constant<false>{}, bool_constant<false>{});
343 if (rhs_inner_dim_contiguous) {
344 if (rhs_inner_dim_reordered)
345 fn(bool_constant<false>{}, bool_constant<true>{}, bool_constant<true>{});
347 fn(bool_constant<false>{}, bool_constant<true>{}, bool_constant<false>{});
349 if (rhs_inner_dim_reordered)
350 fn(bool_constant<false>{}, bool_constant<false>{}, bool_constant<true>{});
352 fn(bool_constant<false>{}, bool_constant<false>{}, bool_constant<false>{});
362#ifndef TENSOR_CONTRACTION_DISPATCH
363#define TENSOR_CONTRACTION_DISPATCH(METHOD, ALIGNMENT, ARGS) \
364 ::Eigen::internal::tensor_contraction_dispatch( \
365 [&](auto lhs_c, auto rhs_c, auto rhs_r) { METHOD<lhs_c(), rhs_c(), rhs_r(), ALIGNMENT> ARGS; }, \
366 this->m_lhs_inner_dim_contiguous, this->m_rhs_inner_dim_contiguous, this->m_rhs_inner_dim_reordered)
369#ifndef TENSOR_CONTRACTION_ASYNC_DISPATCH
370#define TENSOR_CONTRACTION_ASYNC_DISPATCH(METHOD, DONE, ALIGNMENT, ARGS, FN) \
371 ::Eigen::internal::tensor_contraction_dispatch( \
372 [&](auto lhs_c, auto rhs_c, auto rhs_r) { (new METHOD<DONE, lhs_c(), rhs_c(), rhs_r(), ALIGNMENT> ARGS)->FN; }, \
373 this->m_lhs_inner_dim_contiguous, this->m_rhs_inner_dim_contiguous, this->m_rhs_inner_dim_reordered)
378struct TensorContractionParams {
381 bool swapped_arguments;
391struct NoOpOutputKernel {
407 template <
typename Index,
typename Scalar>
408 EIGEN_ALWAYS_INLINE
void operator()(
const internal::blas_data_mapper<Scalar, Index, ColMajor>&,
409 const TensorContractionParams&, Index, Index, Index, Index)
const {}
415template <
typename Indices,
typename LhsXprType,
typename RhsXprType,
416 typename OutputKernelType =
const NoOpOutputKernel>
417class TensorContractionOp
418 :
public TensorBase<TensorContractionOp<Indices, LhsXprType, RhsXprType, OutputKernelType>, ReadOnlyAccessors> {
420 using Scalar =
typename Eigen::internal::traits<TensorContractionOp>::Scalar;
421 using CoeffReturnType =
typename internal::gebp_traits<
typename LhsXprType::CoeffReturnType,
422 typename RhsXprType::CoeffReturnType>::ResScalar;
423 using Nested =
typename Eigen::internal::ref_selector<TensorContractionOp>::type;
424 using StorageKind =
typename Eigen::internal::traits<TensorContractionOp>::StorageKind;
425 using Index =
typename Eigen::internal::traits<TensorContractionOp>::Index;
427 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorContractionOp(
const LhsXprType& lhs,
const RhsXprType& rhs,
429 const OutputKernelType& output_kernel = OutputKernelType())
430 : m_lhs_xpr(lhs), m_rhs_xpr(rhs), m_indices(dims), m_output_kernel(output_kernel) {}
432 EIGEN_DEVICE_FUNC
const Indices& indices()
const {
return m_indices; }
435 EIGEN_DEVICE_FUNC
const internal::remove_all_t<typename LhsXprType::Nested>&
lhsExpression()
const {
439 EIGEN_DEVICE_FUNC
const internal::remove_all_t<typename RhsXprType::Nested>& rhsExpression()
const {
443 EIGEN_DEVICE_FUNC
const OutputKernelType& outputKernel()
const {
return m_output_kernel; }
446 typename LhsXprType::Nested m_lhs_xpr;
447 typename RhsXprType::Nested m_rhs_xpr;
448 const Indices m_indices;
449 const OutputKernelType m_output_kernel;
453template <
bool TypesMatch,
int StorageOrder,
bool MatrixIsRight>
454struct GemvDirectDispatcher {
455 template <
typename Evaluator,
typename Scalar>
456 static bool run(
const Evaluator* self, Scalar* buffer) {
457 self->template evalGemvDirect<StorageOrder, MatrixIsRight>(buffer);
462template <
int StorageOrder,
bool MatrixIsRight>
463struct GemvDirectDispatcher<false, StorageOrder, MatrixIsRight> {
464 template <
typename Evaluator,
typename Scalar>
465 static bool run(
const Evaluator*, Scalar*) {
471template <
typename Derived>
472struct TensorContractionEvaluatorBase {
473 using Indices =
typename internal::traits<Derived>::Indices;
474 using LeftArgType =
typename internal::traits<Derived>::LeftArgType;
475 using RightArgType =
typename internal::traits<Derived>::RightArgType;
476 using OutputKernelType =
typename internal::traits<Derived>::OutputKernelType;
477 using Device =
typename internal::traits<Derived>::Device;
479 using XprType = TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>;
480 using Scalar = std::remove_const_t<typename XprType::Scalar>;
481 using Index =
typename XprType::Index;
482 using CoeffReturnType =
typename XprType::CoeffReturnType;
483 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
484 using Storage = StorageMemory<Scalar, Device>;
485 using EvaluatorPointerType =
typename Storage::Type;
487 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
488 static constexpr bool IsAligned =
true;
489 static constexpr bool PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1);
490 static constexpr bool BlockAccess =
false;
491 static constexpr bool PreferBlockAccess =
false;
492 static constexpr bool CoordAccess =
false;
493 static constexpr bool RawAccess =
true;
496 using TensorBlock = internal::TensorBlockNotImplemented;
503 using EvalLeftArgType =
504 std::conditional_t<static_cast<int>(Layout) ==
static_cast<int>(
ColMajor), LeftArgType, RightArgType>;
505 using EvalRightArgType =
506 std::conditional_t<static_cast<int>(Layout) ==
static_cast<int>(
ColMajor), RightArgType, LeftArgType>;
508 static constexpr int LDims =
509 internal::array_size<typename TensorEvaluator<EvalLeftArgType, Device>::Dimensions>::value;
510 static constexpr int RDims =
511 internal::array_size<typename TensorEvaluator<EvalRightArgType, Device>::Dimensions>::value;
512 static constexpr int ContractDims = internal::array_size<Indices>::value;
513 static constexpr int NumDims = LDims + RDims - 2 * ContractDims;
515 using contract_t = array<Index, ContractDims>;
516 using left_nocontract_t = array<Index, LDims - ContractDims>;
517 using right_nocontract_t = array<Index, RDims - ContractDims>;
519 using Dimensions = DSizes<Index, NumDims>;
521 EIGEN_STRONG_INLINE TensorContractionEvaluatorBase(
const XprType& op,
const Device& device)
522 : m_leftImpl(choose(Cond<static_cast<int>(Layout) == static_cast<int>(
ColMajor)>(), op.lhsExpression(),
525 m_rightImpl(choose(Cond<static_cast<int>(Layout) == static_cast<int>(
ColMajor)>(), op.rhsExpression(),
529 m_output_kernel(op.outputKernel()),
531 EIGEN_STATIC_ASSERT((
static_cast<int>(TensorEvaluator<LeftArgType, Device>::Layout) ==
532 static_cast<int>(TensorEvaluator<RightArgType, Device>::Layout)),
533 YOU_MADE_A_PROGRAMMING_MISTAKE);
535 DSizes<Index, LDims> eval_left_dims;
536 DSizes<Index, RDims> eval_right_dims;
537 array<IndexPair<Index>, ContractDims> eval_op_indices;
538 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
540 for (
int i = 0; i < LDims; i++) {
541 eval_left_dims[i] = m_leftImpl.dimensions()[i];
543 for (
int i = 0; i < RDims; i++) {
544 eval_right_dims[i] = m_rightImpl.dimensions()[i];
547 for (
int i = 0; i < ContractDims; i++) {
548 eval_op_indices[i].first = op.indices()[i].first;
549 eval_op_indices[i].second = op.indices()[i].second;
553 for (
int i = 0; i < LDims; i++) {
554 eval_left_dims[i] = m_leftImpl.dimensions()[LDims - i - 1];
556 for (
int i = 0; i < RDims; i++) {
557 eval_right_dims[i] = m_rightImpl.dimensions()[RDims - i - 1];
561 for (
int i = 0; i < ContractDims; i++) {
562 eval_op_indices[i].first = LDims - 1 - op.indices()[ContractDims - 1 - i].second;
563 eval_op_indices[i].second = RDims - 1 - op.indices()[ContractDims - 1 - i].first;
569 for (
int i = 0; i < ContractDims; i++) {
570 for (
int j = i + 1; j < ContractDims; j++) {
571 eigen_assert(eval_op_indices[j].first != eval_op_indices[i].first &&
572 eval_op_indices[j].second != eval_op_indices[i].second &&
"contraction axes should be unique");
573 if (eval_op_indices[j].first < eval_op_indices[i].first) {
574 numext::swap(eval_op_indices[j], eval_op_indices[i]);
579 array<Index, LDims> lhs_strides;
581 for (
int i = 0; i < LDims - 1; ++i) {
582 lhs_strides[i + 1] = lhs_strides[i] * eval_left_dims[i];
585 array<Index, RDims> rhs_strides;
587 for (
int i = 0; i < RDims - 1; ++i) {
588 rhs_strides[i + 1] = rhs_strides[i] * eval_right_dims[i];
599 m_lhs_inner_dim_contiguous =
true;
601 Index nocontract_idx = 0;
603 for (
int i = 0; i < LDims; i++) {
605 bool contracting =
false;
606 for (
int j = 0; j < ContractDims; j++) {
607 if (eval_op_indices[j].first == i) {
614 m_dimensions[dim_idx] = eval_left_dims[i];
615 m_left_nocontract_strides[nocontract_idx] = lhs_strides[i];
617 m_lhs_inner_dim_contiguous =
false;
619 m_i_strides[nocontract_idx] = m_i_size;
620 m_i_size *= eval_left_dims[i];
627 for (
int i = 0; i < RDims; i++) {
628 bool contracting =
false;
630 for (
int j = 0; j < ContractDims; j++) {
631 if (eval_op_indices[j].second == i) {
637 m_dimensions[dim_idx] = eval_right_dims[i];
638 m_j_strides[nocontract_idx] = m_j_size;
639 m_j_size *= eval_right_dims[i];
640 m_right_nocontract_strides[nocontract_idx] = rhs_strides[i];
651 m_rhs_inner_dim_contiguous =
true;
652 m_rhs_inner_dim_reordered =
false;
653 for (
int i = 0; i < ContractDims; i++) {
654 Index left = eval_op_indices[i].first;
655 Index right = eval_op_indices[i].second;
657 Index size = eval_left_dims[left];
658 eigen_assert(size == eval_right_dims[right] &&
"Contraction axes must be same size");
660 m_k_strides[i] = m_k_size;
662 m_left_contracting_strides[i] = lhs_strides[left];
663 m_right_contracting_strides[i] = rhs_strides[right];
665 if (i > 0 && right < eval_op_indices[i - 1].second) {
666 m_rhs_inner_dim_reordered =
true;
669 m_rhs_inner_dim_contiguous =
false;
672 eigen_assert(!m_rhs_inner_dim_reordered || (ContractDims > 1 && !m_rhs_inner_dim_contiguous));
679 m_lhs_contracted_dims_leading =
true;
680 m_rhs_contracted_dims_leading =
true;
681 m_rhs_contracted_dims_trailing =
true;
682 const int rhs_trail_start = RDims - ContractDims;
683 for (
int i = 0; i < ContractDims; i++) {
684 if (eval_op_indices[i].first != i) m_lhs_contracted_dims_leading =
false;
685 if (eval_op_indices[i].second != i) m_rhs_contracted_dims_leading =
false;
686 if (eval_op_indices[i].second != rhs_trail_start + i) m_rhs_contracted_dims_trailing =
false;
690 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
RowMajor)) {
691 for (
int i = 0, j = NumDims - 1; i < j; i++, j--) {
692 numext::swap(m_dimensions[i], m_dimensions[j]);
700 m_tensor_contraction_params.swapped_arguments =
static_cast<int>(Layout) ==
RowMajor;
703 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Dimensions& dimensions()
const {
return m_dimensions; }
705 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(EvaluatorPointerType data) {
706 m_leftImpl.evalSubExprsIfNeeded(
nullptr);
708 EIGEN_TRY { m_rightImpl.evalSubExprsIfNeeded(
nullptr); }
710 m_leftImpl.cleanup();
717 m_result =
static_cast<EvaluatorPointerType
>(m_device.allocate(dimensions().TotalSize() *
sizeof(Scalar)));
726 return data ==
nullptr;
729#ifdef EIGEN_USE_THREADS
730 template <
typename EvalSubExprsCallback>
731 EIGEN_STRONG_INLINE
void evalSubExprsIfNeededAsync(EvaluatorPointerType dest, EvalSubExprsCallback done) {
732 m_leftImpl.evalSubExprsIfNeededAsync(
nullptr, [
this, done, dest](
bool) {
733 m_rightImpl.evalSubExprsIfNeededAsync(
nullptr, [
this, done, dest](
bool) {
734 const bool need_assign = (dest ==
nullptr);
735 EvaluatorPointerType buffer = dest;
737 m_result =
static_cast<EvaluatorPointerType
>(m_device.allocate(dimensions().TotalSize() *
sizeof(Scalar)));
742 evalToAsync(buffer, [done, need_assign]() { done(need_assign); });
748 EIGEN_DEVICE_FUNC
void evalTo(Scalar* buffer)
const {
749 static_cast<const Derived*
>(
this)->
template evalProduct<Unaligned>(buffer);
752#ifdef EIGEN_USE_THREADS
753 template <
typename EvalToCallback>
754 void evalToAsync(Scalar* buffer, EvalToCallback done)
const {
755 static_cast<const Derived*
>(
this)->
template evalProductAsync<EvalToCallback, Unaligned>(buffer, std::move(done));
759 template <
typename Func>
760 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
static void dispatchContraction(Func&& fn,
bool lhs_inner_dim_contiguous,
761 bool rhs_inner_dim_contiguous,
762 bool rhs_inner_dim_reordered) {
767 internal::tensor_contraction_dispatch(
768 [&](
auto lhs_c,
auto rhs_c,
auto rhs_r) {
769 constexpr bool kRhsReordered = (ContractDims > 1) && !rhs_c() && rhs_r();
770 fn(lhs_c, rhs_c, internal::bool_constant<kRhsReordered>{});
772 lhs_inner_dim_contiguous, rhs_inner_dim_contiguous,
773 (ContractDims > 1) && !rhs_inner_dim_contiguous && rhs_inner_dim_reordered);
776 template <
bool lhs_inner_dim_contiguous,
bool rhs_inner_dim_contiguous,
bool rhs_inner_dim_reordered,
int Alignment>
777 void evalProductSequential(Scalar* buffer)
const {
778 if (this->m_i_size == 0 || this->m_j_size == 0) {
781 if (this->m_k_size == 0) {
783 this->m_device.fill(buffer, buffer + this->m_i_size * this->m_j_size, Scalar(0));
784 using OutputMapper = internal::blas_data_mapper<Scalar, Index, ColMajor>;
785 this->m_output_kernel(OutputMapper(buffer, this->m_i_size), this->m_tensor_contraction_params,
786 static_cast<Index
>(0),
static_cast<Index
>(0), this->m_i_size, this->m_j_size);
807 constexpr bool types_match = std::is_same<typename EvalLeftArgType::Scalar, Scalar>::value &&
808 std::is_same<typename EvalRightArgType::Scalar, Scalar>::value;
809 if (this->m_j_size == 1) {
810 if (lhs_inner_dim_contiguous) {
811 this->
template evalGemv<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Alignment>(
815 EIGEN_IF_CONSTEXPR (types_match) {
817 if (m_lhs_contracted_dims_leading && !rhs_inner_dim_reordered && m_leftImpl.data() !=
nullptr &&
818 m_rightImpl.data() !=
nullptr) {
819 if (internal::GemvDirectDispatcher<types_match, RowMajor, false>::run(
this, buffer)) {
824 }
else if (this->m_i_size == 1 && m_leftImpl.data() !=
nullptr && m_rightImpl.data() !=
nullptr) {
825 EIGEN_IF_CONSTEXPR (types_match) {
826 if (m_rhs_contracted_dims_leading) {
827 if (internal::GemvDirectDispatcher<types_match, RowMajor, true>::run(
this, buffer)) {
831 if (m_rhs_contracted_dims_trailing) {
832 if (internal::GemvDirectDispatcher<types_match, ColMajor, true>::run(
this, buffer)) {
838 this->
template evalGemm<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Alignment>(
842 template <
bool lhs_inner_dim_contiguous,
bool rhs_inner_dim_contiguous,
bool rhs_inner_dim_reordered,
int Alignment>
843#if !defined(EIGEN_HIPCC)
847 evalGemv(Scalar* buffer)
const {
848 const Index rows = m_i_size;
849 const Index cols = m_k_size;
851 using LhsScalar = std::remove_const_t<typename EvalLeftArgType::Scalar>;
852 using RhsScalar = std::remove_const_t<typename EvalRightArgType::Scalar>;
853 using LeftEvaluator = TensorEvaluator<EvalLeftArgType, Device>;
854 using RightEvaluator = TensorEvaluator<EvalRightArgType, Device>;
855 const int lhs_packet_size = internal::unpacket_traits<typename LeftEvaluator::PacketReturnType>::size;
856 const int rhs_packet_size = internal::unpacket_traits<typename RightEvaluator::PacketReturnType>::size;
857 constexpr int lhs_alignment = LeftEvaluator::IsAligned ?
Aligned :
Unaligned;
858 constexpr int rhs_alignment = RightEvaluator::IsAligned ?
Aligned :
Unaligned;
859 using LhsMapper = internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs, LeftEvaluator,
860 left_nocontract_t, contract_t, lhs_packet_size,
861 lhs_inner_dim_contiguous,
false, lhs_alignment>;
864 internal::TensorContractionInputMapper<RhsScalar, Index, internal::Rhs, RightEvaluator, right_nocontract_t,
865 contract_t, rhs_packet_size, rhs_inner_dim_contiguous,
866 rhs_inner_dim_reordered, rhs_alignment>;
868 LhsMapper lhs(m_leftImpl, m_left_nocontract_strides, m_i_strides, m_left_contracting_strides, m_k_strides);
869 RhsMapper rhs(m_rightImpl, m_right_nocontract_strides, m_j_strides, m_right_contracting_strides, m_k_strides);
871 const Scalar alpha(1);
872 const Index resIncr(1);
875 m_device.fill(buffer, buffer + rows, Scalar(0));
877 internal::general_matrix_vector_product<Index, LhsScalar, LhsMapper,
ColMajor,
false, RhsScalar, RhsMapper,
878 false>::run(rows, cols, lhs, rhs, buffer, resIncr, alpha);
880 using OutputMapper = internal::blas_data_mapper<Scalar, Index, ColMajor>;
881 m_output_kernel(OutputMapper(buffer, rows), m_tensor_contraction_params,
static_cast<Index
>(0),
882 static_cast<Index
>(0), rows,
static_cast<Index
>(1));
895 template <
int StorageOrder,
bool MatrixIsRight>
896#if !defined(EIGEN_HIPCC)
900 evalGemvDirect(Scalar* buffer)
const {
901 using MatScalar = std::remove_const_t<
902 std::conditional_t<MatrixIsRight, typename EvalRightArgType::Scalar, typename EvalLeftArgType::Scalar>>;
903 using VecScalar = std::remove_const_t<
904 std::conditional_t<MatrixIsRight, typename EvalLeftArgType::Scalar, typename EvalRightArgType::Scalar>>;
906 const Index rows = MatrixIsRight ? m_j_size : m_i_size;
907 const Index cols = m_k_size;
910 const Index mat_stride = (StorageOrder ==
RowMajor) ? cols : rows;
912 const MatScalar* mat_data = MatrixIsRight ? m_rightImpl.data() : m_leftImpl.data();
913 const VecScalar* vec_data = MatrixIsRight ? m_leftImpl.data() : m_rightImpl.data();
914 eigen_assert(mat_data !=
nullptr && vec_data !=
nullptr);
916 using LhsMapper = internal::const_blas_data_mapper<MatScalar, Index, StorageOrder>;
917 using RhsMapper = internal::const_blas_data_mapper<VecScalar, Index, ColMajor>;
918 LhsMapper lhs(mat_data, mat_stride);
919 RhsMapper rhs(vec_data, 1);
921 m_device.fill(buffer, buffer + rows, Scalar(0));
923 internal::general_matrix_vector_product<Index, MatScalar, LhsMapper, StorageOrder,
false, VecScalar, RhsMapper,
924 false>::run(rows, cols, lhs, rhs, buffer, 1, Scalar(1));
926 using OutputMapper = internal::blas_data_mapper<Scalar, Index, ColMajor>;
927 m_output_kernel(OutputMapper(buffer, rows), m_tensor_contraction_params,
static_cast<Index
>(0),
928 static_cast<Index
>(0), rows,
static_cast<Index
>(1));
931 template <
bool lhs_inner_dim_contiguous,
bool rhs_inner_dim_contiguous,
bool rhs_inner_dim_reordered,
int Alignment>
932#if !defined(EIGEN_HIPCC)
936 evalGemm(Scalar* buffer)
const {
938 const Index k = this->m_k_size;
939 this->
template evalGemmPartial<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered,
940 Alignment,
true>(buffer, 0, k, 1);
943 template <
bool lhs_inner_dim_contiguous,
bool rhs_inner_dim_contiguous,
bool rhs_inner_dim_reordered,
int Alignment>
944 EIGEN_DEVICE_FUNC
void evalGemmPartialWithoutOutputKernel(Scalar* buffer, Index k_start, Index k_end,
945 int num_threads)
const {
946 evalGemmPartial<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Alignment,
947 false>(buffer, k_start, k_end, num_threads);
950 template <
bool lhs_inner_dim_contiguous,
bool rhs_inner_dim_contiguous,
bool rhs_inner_dim_reordered,
int Alignment,
951 bool use_output_kernel>
952 EIGEN_DEVICE_FUNC
void evalGemmPartial(Scalar* buffer, Index k_start, Index k_end,
int num_threads)
const {
953 eigen_assert(k_end >= k_start && k_start >= 0 && k_end <= this->m_k_size);
955 const Index k_slice = k_end - k_start;
958 const Index m = this->m_i_size;
961 const Index n = this->m_j_size;
963 if (m == 0 || n == 0)
return;
965 this->m_device.fill(buffer, buffer + m * n, Scalar(0));
966 if (use_output_kernel) {
967 using OutputMapper = internal::blas_data_mapper<Scalar, Index, ColMajor>;
968 m_output_kernel(OutputMapper(buffer, m), m_tensor_contraction_params,
static_cast<Index
>(0),
969 static_cast<Index
>(0), m, n);
975 using LhsScalar = std::remove_const_t<typename EvalLeftArgType::Scalar>;
976 using RhsScalar = std::remove_const_t<typename EvalRightArgType::Scalar>;
978 using LeftEvaluator = TensorEvaluator<EvalLeftArgType, Device>;
979 using RightEvaluator = TensorEvaluator<EvalRightArgType, Device>;
981 const int lhs_packet_size = internal::unpacket_traits<typename LeftEvaluator::PacketReturnType>::size;
982 const int rhs_packet_size = internal::unpacket_traits<typename RightEvaluator::PacketReturnType>::size;
985 internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs, LeftEvaluator, left_nocontract_t,
986 contract_t, lhs_packet_size, lhs_inner_dim_contiguous,
false,
Unaligned>;
989 internal::TensorContractionInputMapper<RhsScalar, Index, internal::Rhs, RightEvaluator, right_nocontract_t,
990 contract_t, rhs_packet_size, rhs_inner_dim_contiguous,
993 using OutputMapper = internal::blas_data_mapper<Scalar, Index, ColMajor>;
995 using TensorContractionKernel =
996 internal::TensorContractionKernel<Scalar, LhsScalar, RhsScalar, Index, OutputMapper, LhsMapper, RhsMapper>;
999 LhsMapper lhs(this->m_leftImpl, this->m_left_nocontract_strides, this->m_i_strides,
1000 this->m_left_contracting_strides, this->m_k_strides);
1002 RhsMapper rhs(this->m_rightImpl, this->m_right_nocontract_strides, this->m_j_strides,
1003 this->m_right_contracting_strides, this->m_k_strides);
1005 OutputMapper output(buffer, m);
1008 internal::TensorContractionBlocking<Scalar, LhsScalar, RhsScalar, Index, internal::ShardByCol> blocking(
1009 k_slice, m, n, num_threads);
1010 const Index kc = blocking.kc();
1011 const Index mc = numext::mini(m, blocking.mc());
1012 const Index nc = numext::mini(n, blocking.nc());
1014 using LhsBlock =
typename TensorContractionKernel::LhsBlock;
1015 using RhsBlock =
typename TensorContractionKernel::RhsBlock;
1020 TensorContractionKernel kernel(m, k_slice, n, mc, kc, nc);
1022 using BlockMemHandle =
typename TensorContractionKernel::BlockMemHandle;
1023 const BlockMemHandle packed_mem = kernel.allocate(this->m_device, &blockA, &blockB);
1025 struct PackedMemGuard {
1026 TensorContractionKernel& kernel;
1027 const Device& device;
1028 const BlockMemHandle& mem;
1029 EIGEN_DEVICE_FUNC ~PackedMemGuard() { kernel.deallocate(device, mem); }
1030 } packed_mem_guard{kernel, this->m_device, packed_mem};
1034 EIGEN_IF_CONSTEXPR (!TensorContractionKernel::HasBeta) {
1035 this->m_device.fill(buffer, buffer + m * n, Scalar(0));
1038 for (Index i2 = 0; i2 < m; i2 += mc) {
1039 const Index actual_mc = numext::mini(i2 + mc, m) - i2;
1040 for (Index k2 = k_start; k2 < k_end; k2 += kc) {
1042 const Index actual_kc = numext::mini(k2 + kc, k_end) - k2;
1043 kernel.packLhs(&blockA, lhs.getSubMapper(i2, k2), actual_kc, actual_mc);
1047 const Scalar alpha = Scalar(1);
1048 const Scalar beta = (TensorContractionKernel::HasBeta && k2 == k_start) ? Scalar(0) : Scalar(1);
1051 for (Index j2 = 0; j2 < n; j2 += nc) {
1053 const Index actual_nc = numext::mini(j2 + nc, n) - j2;
1054 kernel.packRhs(&blockB, rhs.getSubMapper(k2, j2), actual_kc, actual_nc);
1058 const OutputMapper output_mapper = output.getSubMapper(i2, j2);
1059 kernel.invoke(output_mapper, blockA, blockB, actual_mc, actual_kc, actual_nc, alpha, beta);
1062 if (use_output_kernel && k2 + kc >= k_end) {
1063 m_output_kernel(output_mapper, m_tensor_contraction_params, i2, j2, actual_mc, actual_nc);
1070 EIGEN_STRONG_INLINE
void cleanup() {
1071 m_leftImpl.cleanup();
1072 m_rightImpl.cleanup();
1076 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index)
const {
return m_result[index]; }
1078 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool)
const {
1079 return TensorOpCost(
sizeof(CoeffReturnType), 0, 0);
1082 template <
int LoadMode>
1083 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index)
const {
1084 return internal::ploadt<PacketReturnType, LoadMode>(m_result + index);
1087 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EvaluatorPointerType data()
const {
return m_result; }
1096 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements()
const {
1097 return internal::TensorBlockResourceRequirements::any();
1104 EIGEN_STRONG_INLINE
void constructResult() {
1105 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
1106 EIGEN_TRY { internal::default_construct_elements_of_array(m_result, dimensions().TotalSize()); }
1108 m_device.deallocate(m_result);
1112 m_result_constructed =
true;
1116 EIGEN_STRONG_INLINE
void releaseResult() {
1117 if (m_result !=
nullptr) {
1118 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
1119 if (m_result_constructed) internal::destruct_elements_of_array(m_result, dimensions().TotalSize());
1120 m_result_constructed =
false;
1122 m_device.deallocate(m_result);
1127 Dimensions m_dimensions;
1129 contract_t m_k_strides{};
1130 contract_t m_left_contracting_strides{};
1131 contract_t m_right_contracting_strides{};
1133 bool m_lhs_inner_dim_contiguous;
1134 bool m_rhs_inner_dim_contiguous;
1135 bool m_rhs_inner_dim_reordered;
1139 bool m_lhs_contracted_dims_leading;
1140 bool m_rhs_contracted_dims_leading;
1141 bool m_rhs_contracted_dims_trailing;
1143 left_nocontract_t m_i_strides{};
1144 right_nocontract_t m_j_strides{};
1145 left_nocontract_t m_left_nocontract_strides{};
1146 right_nocontract_t m_right_nocontract_strides{};
1152 TensorContractionParams m_tensor_contraction_params;
1154 TensorEvaluator<EvalLeftArgType, Device> m_leftImpl;
1155 TensorEvaluator<EvalRightArgType, Device> m_rightImpl;
1156 const Device EIGEN_DEVICE_REF m_device;
1157 OutputKernelType m_output_kernel;
1158 EvaluatorPointerType m_result;
1159 bool m_result_constructed =
false;
1163template <
typename Indices,
typename LeftArgType,
typename RightArgType,
typename OutputKernelType,
typename Device>
1165 :
public TensorContractionEvaluatorBase<
1166 TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Device>> {
1167 using Self = TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Device>;
1168 using Base = TensorContractionEvaluatorBase<Self>;
1170 using XprType = TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>;
1171 using Scalar = std::remove_const_t<typename XprType::Scalar>;
1172 using Index =
typename XprType::Index;
1173 using CoeffReturnType =
typename XprType::CoeffReturnType;
1174 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
1176 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
1182 using EvalLeftArgType = std::conditional_t<Layout == static_cast<int>(
ColMajor), LeftArgType, RightArgType>;
1183 using EvalRightArgType = std::conditional_t<Layout == static_cast<int>(
ColMajor), RightArgType, LeftArgType>;
1185 static constexpr int LDims =
1186 internal::array_size<typename TensorEvaluator<EvalLeftArgType, Device>::Dimensions>::value;
1187 static constexpr int RDims =
1188 internal::array_size<typename TensorEvaluator<EvalRightArgType, Device>::Dimensions>::value;
1189 static constexpr int ContractDims = internal::array_size<Indices>::value;
1191 using contract_t = array<Index, ContractDims>;
1192 using left_nocontract_t = array<Index, LDims - ContractDims>;
1193 using right_nocontract_t = array<Index, RDims - ContractDims>;
1195 static constexpr int NumDims = LDims + RDims - 2 * ContractDims;
1198 using Dimensions = DSizes<Index, NumDims>;
1200 TensorEvaluator(
const XprType& op,
const Device& device) : Base(op, device) {}
1202 template <
int Alignment>
1203 void evalProduct(Scalar* buffer)
const {
1204 Base::dispatchContraction(
1205 [&](
auto lhs_c,
auto rhs_c,
auto rhs_r) {
1206 this->
template evalProductSequential<lhs_c(), rhs_c(), rhs_r(), Alignment>(buffer);
1208 this->m_lhs_inner_dim_contiguous, this->m_rhs_inner_dim_contiguous, this->m_rhs_inner_dim_reordered);
The tensor base class.
Definition TensorForwardDeclarations.h:69
Definition TensorContraction.h:418
const internal::remove_all_t< typename LhsXprType::Nested > & lhsExpression() const
Definition TensorContraction.h:435
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47