Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TensorContraction.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_TENSOR_TENSOR_CONTRACTION_H
12#define EIGEN_TENSOR_TENSOR_CONTRACTION_H
13
14// IWYU pragma: private
15#include "./InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21template <typename Dimensions, typename LhsXprType, typename RhsXprType, typename OutputKernelType>
22struct traits<TensorContractionOp<Dimensions, LhsXprType, RhsXprType, OutputKernelType>> {
23 // Type promotion to handle the case where the types of the lhs and the rhs are different.
24 using Scalar = typename gebp_traits<std::remove_const_t<typename LhsXprType::Scalar>,
25 std::remove_const_t<typename RhsXprType::Scalar>>::ResScalar;
26
27 using StorageKind = typename promote_storage_type<typename traits<LhsXprType>::StorageKind,
28 typename traits<RhsXprType>::StorageKind>::ret;
29 using Index =
30 typename promote_index_type<typename traits<LhsXprType>::Index, typename traits<RhsXprType>::Index>::type;
31 // From NumDims below.
32 static constexpr int NumDimensions =
33 traits<LhsXprType>::NumDimensions + traits<RhsXprType>::NumDimensions - 2 * array_size<Dimensions>::value;
34 static constexpr int Layout = traits<LhsXprType>::Layout;
35 using PointerType =
36 std::conditional_t<Pointer_type_promotion<typename LhsXprType::Scalar, Scalar>::val,
37 typename traits<LhsXprType>::PointerType, typename traits<RhsXprType>::PointerType>;
38
39 static constexpr int Flags = 0;
40};
41
42template <typename Dimensions, typename LhsXprType, typename RhsXprType, typename OutputKernelType>
43struct eval<TensorContractionOp<Dimensions, LhsXprType, RhsXprType, OutputKernelType>, Eigen::Dense> {
44 using type = const TensorContractionOp<Dimensions, LhsXprType, RhsXprType, OutputKernelType>&;
45};
46
47template <typename Indices_, typename LeftArgType_, typename RightArgType_, typename OutputKernelType_,
48 typename Device_>
49struct traits<
50 TensorEvaluator<const TensorContractionOp<Indices_, LeftArgType_, RightArgType_, OutputKernelType_>, Device_>> {
51 using Indices = Indices_;
52 using LeftArgType = LeftArgType_;
53 using RightArgType = RightArgType_;
54 using OutputKernelType = OutputKernelType_;
55 using Device = Device_;
56
57 // From NumDims below.
58 static constexpr int NumDimensions =
59 traits<LeftArgType_>::NumDimensions + traits<RightArgType_>::NumDimensions - 2 * array_size<Indices_>::value;
60};
61
62// Helper class to allocate and deallocate temporary memory for packed buffers.
63template <typename LhsScalar, typename RhsScalar,
64 bool RequireInitialization =
65 NumTraits<LhsScalar>::RequireInitialization || NumTraits<RhsScalar>::RequireInitialization>
66struct TensorContractionBlockMemAllocator {
67 using BlockMemHandle = void*;
68
69 template <typename Device>
70 EIGEN_DEVICE_FUNC static BlockMemHandle allocate(Device& d, const Index bm, const Index bk, const Index bn,
71 LhsScalar** lhs_block, RhsScalar** rhs_block) {
72 eigen_assert(lhs_block);
73 eigen_assert(rhs_block);
74 BlockSizes sz = ComputeLhsRhsBlockSizes(bm, bk, bn);
75 char* block_mem = static_cast<char*>(d.allocate(sz.lhs_size + sz.rhs_size));
76 *lhs_block = static_cast<LhsScalar*>(static_cast<void*>(block_mem));
77 *rhs_block = static_cast<RhsScalar*>(static_cast<void*>(block_mem + sz.lhs_size));
78 return block_mem;
79 }
80
81 template <typename Device>
82 EIGEN_DEVICE_FUNC static BlockMemHandle allocateSlices(Device& d, const Index bm, const Index bk, const Index bn,
83 const Index num_lhs, const Index num_rhs,
84 const Index num_slices, std::vector<LhsScalar*>* lhs_blocks,
85 std::vector<RhsScalar*>* rhs_blocks) {
86 eigen_assert(num_slices > 0);
87 eigen_assert(num_lhs >= 0 && num_rhs >= 0);
88 eigen_assert(num_lhs == 0 || lhs_blocks);
89 eigen_assert(num_rhs == 0 || rhs_blocks);
90 BlockSizes sz = ComputeLhsRhsBlockSizes(bm, bk, bn);
91 void* block_mem = d.allocate((num_lhs * sz.lhs_size + num_rhs * sz.rhs_size) * num_slices);
92 eigen_assert(block_mem);
93 char* mem = static_cast<char*>(block_mem);
94
95 for (Index x = 0; x < num_slices; x++) {
96 if (num_lhs > 0) lhs_blocks[x].resize(num_lhs);
97 for (Index m = 0; m < num_lhs; m++) {
98 lhs_blocks[x][m] = static_cast<LhsScalar*>(static_cast<void*>(mem));
99 mem += sz.lhs_size;
100 }
101 if (num_rhs > 0) rhs_blocks[x].resize(num_rhs);
102 for (Index n = 0; n < num_rhs; n++) {
103 rhs_blocks[x][n] = static_cast<RhsScalar*>(static_cast<void*>(mem));
104 mem += sz.rhs_size;
105 }
106 }
107
108 return block_mem;
109 }
110
111 template <typename Device>
112 EIGEN_DEVICE_FUNC static void deallocate(Device& d, BlockMemHandle handle) {
113 d.deallocate(handle);
114 }
115
116 private:
117 struct BlockSizes {
118 Index lhs_size;
119 Index rhs_size;
120 };
121 EIGEN_DEVICE_FUNC static BlockSizes ComputeLhsRhsBlockSizes(const Index bm, const Index bk, const Index bn) {
122 Index align = numext::maxi(EIGEN_MAX_ALIGN_BYTES, 1);
123 BlockSizes sz;
124 sz.lhs_size = numext::div_ceil<Index>(bm * bk * sizeof(LhsScalar), align) * align;
125 sz.rhs_size = numext::div_ceil<Index>(bn * bk * sizeof(RhsScalar), align) * align;
126 return sz;
127 }
128};
129
130// Packed blocks of scalars that require initialization (e.g. AutoDiffScalar) are constructed after allocation and
131// destructed before deallocation. The handle records the blocks so that deallocate() can destruct them.
132template <typename LhsScalar, typename RhsScalar>
133struct TensorContractionBlockMemAllocator<LhsScalar, RhsScalar, true> {
134 using RawAllocator = TensorContractionBlockMemAllocator<LhsScalar, RhsScalar, false>;
135
136 struct BlockMemHandle {
137 void* mem = nullptr;
138 Index lhs_block_size = 0;
139 Index rhs_block_size = 0;
140 std::vector<LhsScalar*> lhs_blocks;
141 std::vector<RhsScalar*> rhs_blocks;
142 };
143
144 template <typename Device>
145 static BlockMemHandle allocate(Device& d, const Index bm, const Index bk, const Index bn, LhsScalar** lhs_block,
146 RhsScalar** rhs_block) {
147 BlockMemHandle handle;
148 handle.mem = RawAllocator::allocate(d, bm, bk, bn, lhs_block, rhs_block);
149 EIGEN_TRY {
150 handle.lhs_blocks.push_back(*lhs_block);
151 handle.rhs_blocks.push_back(*rhs_block);
152 construct(handle, bm * bk, bn * bk);
153 }
154 EIGEN_CATCH(...) {
155 RawAllocator::deallocate(d, handle.mem);
156 EIGEN_THROW;
157 }
158 return handle;
159 }
160
161 template <typename Device>
162 static BlockMemHandle allocateSlices(Device& d, const Index bm, const Index bk, const Index bn, const Index num_lhs,
163 const Index num_rhs, const Index num_slices, std::vector<LhsScalar*>* lhs_blocks,
164 std::vector<RhsScalar*>* rhs_blocks) {
165 BlockMemHandle handle;
166 handle.mem = RawAllocator::allocateSlices(d, bm, bk, bn, num_lhs, num_rhs, num_slices, lhs_blocks, rhs_blocks);
167 EIGEN_TRY {
168 for (Index x = 0; x < num_slices; x++) {
169 if (num_lhs > 0) handle.lhs_blocks.insert(handle.lhs_blocks.end(), lhs_blocks[x].begin(), lhs_blocks[x].end());
170 if (num_rhs > 0) handle.rhs_blocks.insert(handle.rhs_blocks.end(), rhs_blocks[x].begin(), rhs_blocks[x].end());
171 }
172 construct(handle, bm * bk, bn * bk);
173 }
174 EIGEN_CATCH(...) {
175 RawAllocator::deallocate(d, handle.mem);
176 EIGEN_THROW;
177 }
178 return handle;
179 }
180
181 template <typename Device>
182 static void deallocate(Device& d, const BlockMemHandle& handle) {
183 for (LhsScalar* block : handle.lhs_blocks) destruct_elements_of_array(block, handle.lhs_block_size);
184 for (RhsScalar* block : handle.rhs_blocks) destruct_elements_of_array(block, handle.rhs_block_size);
185 RawAllocator::deallocate(d, handle.mem);
186 }
187
188 private:
189 // If a constructor throws, the blocks constructed so far are destructed before the exception propagates.
190 static void construct(BlockMemHandle& handle, const Index lhs_block_size, const Index rhs_block_size) {
191 std::size_t lhs_done = 0;
192 std::size_t rhs_done = 0;
193 EIGEN_TRY {
194 for (; lhs_done < handle.lhs_blocks.size(); ++lhs_done) {
195 default_construct_elements_of_array(handle.lhs_blocks[lhs_done], lhs_block_size);
196 }
197 for (; rhs_done < handle.rhs_blocks.size(); ++rhs_done) {
198 default_construct_elements_of_array(handle.rhs_blocks[rhs_done], rhs_block_size);
199 }
200 }
201 EIGEN_CATCH(...) {
202 while (rhs_done > 0) destruct_elements_of_array(handle.rhs_blocks[--rhs_done], rhs_block_size);
203 while (lhs_done > 0) destruct_elements_of_array(handle.lhs_blocks[--lhs_done], lhs_block_size);
204 EIGEN_THROW;
205 }
206 handle.lhs_block_size = lhs_block_size;
207 handle.rhs_block_size = rhs_block_size;
208 }
209};
210
211// WARNING: In this code we assume that Lhs and Rhs tensor expressions are in
212// ColMajor storage order. This property is guaranteed by the
213// TensorContractionOp evaluator. TensorContractionKernel specifies how we pack
214// blocks of Lhs and Rhs tensor expressions, and how we invoke matrix
215// multiplication for these blocks. Default tensor contraction uses
216// gemm_pack_rhs, gemm_pack_lhs and gebp_kernel from Eigen Core (see
217// GeneralBlockPanelKernel.h for details).
218//
219// By specializing contraction kernels we can use other low level libraries to
220// perform matrix multiplication, and still rely on Eigen contraction evaluator.
221// This also includes full support in TensorContractionThreadPool, assuming that
222// underlying gemm does not use its own threading.
223//
224// - ResScalar/LhsScalar/RhsScalar - scalar type for the result of
225// multiplication, lhs tensor and rhs tensor respectively.
226//
227// - StorageIndex - index type for the tensor expressions. In practice almost
228// always is Eigen::Index.
229//
230// - OutputMapper provides access to the memory of the output matrix. In
231// practice it's always column major blas_data_mapper (it must be of ResScalar
232// type).
233//
234// - LhsMapper/RhsMapper similarly to blas_data_mapper provide a two dimensional
235// view into the Lhs/Rhs tensor expressions. In practice it's
236// TensorContractionInputMapper, or some specialization of it based on the
237// type of tensor expression (e.g. TensorImagePatchOp has optimized input
238// mapper).
239template <typename ResScalar, typename LhsScalar, typename RhsScalar, typename StorageIndex, typename OutputMapper,
240 typename LhsMapper, typename RhsMapper>
241struct TensorContractionKernel {
242 // True if `invoke()` supports `beta` in `C <- alpha * A * B + beta * C`
243 // (otherwise beta should be always equal to 1).
244 static constexpr bool HasBeta = false;
245
246 EIGEN_DEVICE_FUNC TensorContractionKernel(StorageIndex m_, StorageIndex k_, StorageIndex n_, StorageIndex bm_,
247 StorageIndex bk_, StorageIndex bn_)
248 : m(m_), k(k_), n(n_), bm(bm_), bk(bk_), bn(bn_) {}
249
250 // Pack blocks of Lhs and Rhs into contiguous blocks in memory.
251 using LhsBlock = LhsScalar*;
252 using RhsBlock = RhsScalar*;
253
254 // Packed Lhs/Rhs block memory allocator.
255 using BlockMemAllocator = TensorContractionBlockMemAllocator<LhsScalar, RhsScalar>;
256 using BlockMemHandle = typename BlockMemAllocator::BlockMemHandle;
257
258 using Traits = typename internal::gebp_traits<LhsScalar, RhsScalar>;
259
260 using LhsPacker = internal::gemm_pack_lhs<LhsScalar, StorageIndex, typename LhsMapper::SubMapper, Traits::mr,
261 Traits::LhsProgress, typename Traits::LhsPacket4Packing, ColMajor>;
262
263 using RhsPacker =
264 internal::gemm_pack_rhs<RhsScalar, StorageIndex, typename RhsMapper::SubMapper, Traits::nr, ColMajor>;
265
266 using GebpKernel = internal::gebp_kernel<LhsScalar, RhsScalar, StorageIndex, OutputMapper, Traits::mr, Traits::nr,
267 /*ConjugateLhs*/ false, /*ConjugateRhs*/ false>;
268
269 template <typename Device>
270 EIGEN_DEVICE_FUNC BlockMemHandle allocate(Device& d, LhsBlock* lhs_block, RhsBlock* rhs_block) {
271 return BlockMemAllocator::allocate(d, bm, bk, bn, lhs_block, rhs_block);
272 }
273
274 template <typename Device>
275 EIGEN_DEVICE_FUNC BlockMemHandle allocateSlices(Device& d, const StorageIndex num_lhs, const StorageIndex num_rhs,
276 const StorageIndex num_slices, std::vector<LhsBlock>* lhs_blocks,
277 std::vector<RhsBlock>* rhs_blocks) {
278 return BlockMemAllocator::allocateSlices(d, bm, bk, bn, num_lhs, num_rhs, num_slices, lhs_blocks, rhs_blocks);
279 }
280
281 template <typename Device>
282 EIGEN_DEVICE_FUNC static void deallocate(Device& d, const BlockMemHandle& handle) {
283 BlockMemAllocator::deallocate(d, handle);
284 }
285
286 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void packLhs(LhsBlock* lhsBlock, const typename LhsMapper::SubMapper& data_mapper,
287 const StorageIndex depth, const StorageIndex rows) {
288 LhsPacker()(*lhsBlock, data_mapper, depth, rows, /*stride*/ 0,
289 /*offset*/ 0);
290 }
291
292 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void packRhs(RhsBlock* rhsBlock, const typename RhsMapper::SubMapper& data_mapper,
293 const StorageIndex depth, const StorageIndex cols) {
294 RhsPacker()(*rhsBlock, data_mapper, depth, cols);
295 }
296
297 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void invoke(const OutputMapper& output_mapper, const LhsBlock& lhsBlock,
298 const RhsBlock& rhsBlock, const StorageIndex rows,
299 const StorageIndex depth, const StorageIndex cols,
300 const ResScalar alpha, const ResScalar beta) {
301 // Default GEBP kernel does not support beta.
302 EIGEN_ONLY_USED_FOR_DEBUG(beta);
303 eigen_assert(beta == ResScalar(1));
304 static constexpr int kComputeStrideFromBlockDimensions = -1;
305 GebpKernel()(output_mapper, lhsBlock, rhsBlock, rows, depth, cols, alpha,
306 /*strideA*/ kComputeStrideFromBlockDimensions,
307 /*strideB*/ kComputeStrideFromBlockDimensions,
308 /*offsetA*/ 0, /*offsetB*/ 0);
309 }
310
311 private:
312 // These are dimensions of the original Tensors, and selected block sizes. The
313 // actual block sizes passed to all function above might be smaller because of
314 // the partial blocks at the end.
315 const StorageIndex m;
316 const StorageIndex k;
317 const StorageIndex n;
318 const StorageIndex bm;
319 const StorageIndex bk;
320 const StorageIndex bn;
321};
322
323// Dispatches a contraction operation over all 8 combinations of the three
324// runtime boolean parameters (lhs_inner_dim_contiguous, rhs_inner_dim_contiguous,
325// rhs_inner_dim_reordered), passing them as compile-time bool_constant
326// tags to the callable `fn`.
327template <typename Func>
328EIGEN_STRONG_INLINE void tensor_contraction_dispatch(Func&& fn, bool lhs_inner_dim_contiguous,
329 bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered) {
330 if (lhs_inner_dim_contiguous) {
331 if (rhs_inner_dim_contiguous) {
332 if (rhs_inner_dim_reordered)
333 fn(bool_constant<true>{}, bool_constant<true>{}, bool_constant<true>{});
334 else
335 fn(bool_constant<true>{}, bool_constant<true>{}, bool_constant<false>{});
336 } else {
337 if (rhs_inner_dim_reordered)
338 fn(bool_constant<true>{}, bool_constant<false>{}, bool_constant<true>{});
339 else
340 fn(bool_constant<true>{}, bool_constant<false>{}, bool_constant<false>{});
341 }
342 } else {
343 if (rhs_inner_dim_contiguous) {
344 if (rhs_inner_dim_reordered)
345 fn(bool_constant<false>{}, bool_constant<true>{}, bool_constant<true>{});
346 else
347 fn(bool_constant<false>{}, bool_constant<true>{}, bool_constant<false>{});
348 } else {
349 if (rhs_inner_dim_reordered)
350 fn(bool_constant<false>{}, bool_constant<false>{}, bool_constant<true>{});
351 else
352 fn(bool_constant<false>{}, bool_constant<false>{}, bool_constant<false>{});
353 }
354 }
355}
356
357} // end namespace internal
358
359// Legacy macros kept for backward compatibility with code that overrides them
360// (e.g. TensorFlow Lite restricts template instantiations for binary size).
361// New Eigen code should use internal::tensor_contraction_dispatch() instead.
362#ifndef TENSOR_CONTRACTION_DISPATCH
363#define TENSOR_CONTRACTION_DISPATCH(METHOD, ALIGNMENT, ARGS) \
364 ::Eigen::internal::tensor_contraction_dispatch( \
365 [&](auto lhs_c, auto rhs_c, auto rhs_r) { METHOD<lhs_c(), rhs_c(), rhs_r(), ALIGNMENT> ARGS; }, \
366 this->m_lhs_inner_dim_contiguous, this->m_rhs_inner_dim_contiguous, this->m_rhs_inner_dim_reordered)
367#endif
368
369#ifndef TENSOR_CONTRACTION_ASYNC_DISPATCH
370#define TENSOR_CONTRACTION_ASYNC_DISPATCH(METHOD, DONE, ALIGNMENT, ARGS, FN) \
371 ::Eigen::internal::tensor_contraction_dispatch( \
372 [&](auto lhs_c, auto rhs_c, auto rhs_r) { (new METHOD<DONE, lhs_c(), rhs_c(), rhs_r(), ALIGNMENT> ARGS)->FN; }, \
373 this->m_lhs_inner_dim_contiguous, this->m_rhs_inner_dim_contiguous, this->m_rhs_inner_dim_reordered)
374#endif
375
376// Tensor contraction params that should enable to get from output matrix
377// 2-dimensional coordinates to the output tensor dimensions.
378struct TensorContractionParams {
379 // TensorContraction evaluator assumes that both tensors are in ColMajor
380 // layout, if tensors are in RowMajor evaluator swap lhs with rhs.
381 bool swapped_arguments;
382};
383
384// Output kernel allows to fuse operations into the tensor contraction.
385//
386// Examples:
387// 1. Elementwise Relu transformation following Conv2D.
388// 2. AddBias to the Conv2D output channels dimension.
389//
390// The NoOpOutputKernel implements an output kernel that does absolutely nothing.
391struct NoOpOutputKernel {
407 template <typename Index, typename Scalar>
408 EIGEN_ALWAYS_INLINE void operator()(const internal::blas_data_mapper<Scalar, Index, ColMajor>&,
409 const TensorContractionParams&, Index, Index, Index, Index) const {}
410};
411
415template <typename Indices, typename LhsXprType, typename RhsXprType,
416 typename OutputKernelType = const NoOpOutputKernel>
417class TensorContractionOp
418 : public TensorBase<TensorContractionOp<Indices, LhsXprType, RhsXprType, OutputKernelType>, ReadOnlyAccessors> {
419 public:
420 using Scalar = typename Eigen::internal::traits<TensorContractionOp>::Scalar;
421 using CoeffReturnType = typename internal::gebp_traits<typename LhsXprType::CoeffReturnType,
422 typename RhsXprType::CoeffReturnType>::ResScalar;
423 using Nested = typename Eigen::internal::ref_selector<TensorContractionOp>::type;
424 using StorageKind = typename Eigen::internal::traits<TensorContractionOp>::StorageKind;
425 using Index = typename Eigen::internal::traits<TensorContractionOp>::Index;
426
427 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorContractionOp(const LhsXprType& lhs, const RhsXprType& rhs,
428 const Indices& dims,
429 const OutputKernelType& output_kernel = OutputKernelType())
430 : m_lhs_xpr(lhs), m_rhs_xpr(rhs), m_indices(dims), m_output_kernel(output_kernel) {}
431
432 EIGEN_DEVICE_FUNC const Indices& indices() const { return m_indices; }
433
435 EIGEN_DEVICE_FUNC const internal::remove_all_t<typename LhsXprType::Nested>& lhsExpression() const {
436 return m_lhs_xpr;
437 }
438
439 EIGEN_DEVICE_FUNC const internal::remove_all_t<typename RhsXprType::Nested>& rhsExpression() const {
440 return m_rhs_xpr;
441 }
442
443 EIGEN_DEVICE_FUNC const OutputKernelType& outputKernel() const { return m_output_kernel; }
444
445 protected:
446 typename LhsXprType::Nested m_lhs_xpr;
447 typename RhsXprType::Nested m_rhs_xpr;
448 const Indices m_indices;
449 const OutputKernelType m_output_kernel;
450};
451
452namespace internal {
453template <bool TypesMatch, int StorageOrder, bool MatrixIsRight>
454struct GemvDirectDispatcher {
455 template <typename Evaluator, typename Scalar>
456 static bool run(const Evaluator* self, Scalar* buffer) {
457 self->template evalGemvDirect<StorageOrder, MatrixIsRight>(buffer);
458 return true;
459 }
460};
461
462template <int StorageOrder, bool MatrixIsRight>
463struct GemvDirectDispatcher<false, StorageOrder, MatrixIsRight> {
464 template <typename Evaluator, typename Scalar>
465 static bool run(const Evaluator*, Scalar*) {
466 return false;
467 }
468};
469} // namespace internal
470
471template <typename Derived>
472struct TensorContractionEvaluatorBase {
473 using Indices = typename internal::traits<Derived>::Indices;
474 using LeftArgType = typename internal::traits<Derived>::LeftArgType;
475 using RightArgType = typename internal::traits<Derived>::RightArgType;
476 using OutputKernelType = typename internal::traits<Derived>::OutputKernelType;
477 using Device = typename internal::traits<Derived>::Device;
478
479 using XprType = TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>;
480 using Scalar = std::remove_const_t<typename XprType::Scalar>;
481 using Index = typename XprType::Index;
482 using CoeffReturnType = typename XprType::CoeffReturnType;
483 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
484 using Storage = StorageMemory<Scalar, Device>;
485 using EvaluatorPointerType = typename Storage::Type;
486
487 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
488 static constexpr bool IsAligned = true;
489 static constexpr bool PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1);
490 static constexpr bool BlockAccess = false;
491 static constexpr bool PreferBlockAccess = false;
492 static constexpr bool CoordAccess = false; // to be implemented
493 static constexpr bool RawAccess = true;
494
495 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
496 using TensorBlock = internal::TensorBlockNotImplemented;
497 //===--------------------------------------------------------------------===//
498
499 // Most of the code is assuming that both input tensors are ColMajor. If the
500 // inputs are RowMajor, we will "cheat" by swapping the LHS and RHS:
501 // If we want to compute A * B = C, where A is LHS and B is RHS, the code
502 // will pretend B is LHS and A is RHS.
503 using EvalLeftArgType =
504 std::conditional_t<static_cast<int>(Layout) == static_cast<int>(ColMajor), LeftArgType, RightArgType>;
505 using EvalRightArgType =
506 std::conditional_t<static_cast<int>(Layout) == static_cast<int>(ColMajor), RightArgType, LeftArgType>;
507
508 static constexpr int LDims =
509 internal::array_size<typename TensorEvaluator<EvalLeftArgType, Device>::Dimensions>::value;
510 static constexpr int RDims =
511 internal::array_size<typename TensorEvaluator<EvalRightArgType, Device>::Dimensions>::value;
512 static constexpr int ContractDims = internal::array_size<Indices>::value;
513 static constexpr int NumDims = LDims + RDims - 2 * ContractDims;
514
515 using contract_t = array<Index, ContractDims>;
516 using left_nocontract_t = array<Index, LDims - ContractDims>;
517 using right_nocontract_t = array<Index, RDims - ContractDims>;
518
519 using Dimensions = DSizes<Index, NumDims>;
520
521 EIGEN_STRONG_INLINE TensorContractionEvaluatorBase(const XprType& op, const Device& device)
522 : m_leftImpl(choose(Cond<static_cast<int>(Layout) == static_cast<int>(ColMajor)>(), op.lhsExpression(),
523 op.rhsExpression()),
524 device),
525 m_rightImpl(choose(Cond<static_cast<int>(Layout) == static_cast<int>(ColMajor)>(), op.rhsExpression(),
526 op.lhsExpression()),
527 device),
528 m_device(device),
529 m_output_kernel(op.outputKernel()),
530 m_result(nullptr) {
531 EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<LeftArgType, Device>::Layout) ==
532 static_cast<int>(TensorEvaluator<RightArgType, Device>::Layout)),
533 YOU_MADE_A_PROGRAMMING_MISTAKE);
534
535 DSizes<Index, LDims> eval_left_dims;
536 DSizes<Index, RDims> eval_right_dims;
537 array<IndexPair<Index>, ContractDims> eval_op_indices;
538 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
539 // For ColMajor, we keep using the existing dimensions
540 for (int i = 0; i < LDims; i++) {
541 eval_left_dims[i] = m_leftImpl.dimensions()[i];
542 }
543 for (int i = 0; i < RDims; i++) {
544 eval_right_dims[i] = m_rightImpl.dimensions()[i];
545 }
546 // We keep the pairs of contracting indices.
547 for (int i = 0; i < ContractDims; i++) {
548 eval_op_indices[i].first = op.indices()[i].first;
549 eval_op_indices[i].second = op.indices()[i].second;
550 }
551 } else {
552 // For RowMajor, we need to reverse the existing dimensions
553 for (int i = 0; i < LDims; i++) {
554 eval_left_dims[i] = m_leftImpl.dimensions()[LDims - i - 1];
555 }
556 for (int i = 0; i < RDims; i++) {
557 eval_right_dims[i] = m_rightImpl.dimensions()[RDims - i - 1];
558 }
559 // We need to flip all the pairs of contracting indices as well as
560 // reversing the dimensions.
561 for (int i = 0; i < ContractDims; i++) {
562 eval_op_indices[i].first = LDims - 1 - op.indices()[ContractDims - 1 - i].second;
563 eval_op_indices[i].second = RDims - 1 - op.indices()[ContractDims - 1 - i].first;
564 }
565 }
566
567 // Check for duplicate axes and make sure the first index in eval_op_indices
568 // is increasing. Using O(n^2) sorting is OK since ContractDims is small
569 for (int i = 0; i < ContractDims; i++) {
570 for (int j = i + 1; j < ContractDims; j++) {
571 eigen_assert(eval_op_indices[j].first != eval_op_indices[i].first &&
572 eval_op_indices[j].second != eval_op_indices[i].second && "contraction axes should be unique");
573 if (eval_op_indices[j].first < eval_op_indices[i].first) {
574 numext::swap(eval_op_indices[j], eval_op_indices[i]);
575 }
576 }
577 }
578
579 array<Index, LDims> lhs_strides;
580 lhs_strides[0] = 1;
581 for (int i = 0; i < LDims - 1; ++i) {
582 lhs_strides[i + 1] = lhs_strides[i] * eval_left_dims[i];
583 }
584
585 array<Index, RDims> rhs_strides;
586 rhs_strides[0] = 1;
587 for (int i = 0; i < RDims - 1; ++i) {
588 rhs_strides[i + 1] = rhs_strides[i] * eval_right_dims[i];
589 }
590
591 m_i_size = 1;
592 m_j_size = 1;
593 m_k_size = 1;
594
595 // To compute the dimension, we simply concatenate the non-contracting
596 // dimensions of the left and then the right tensor. Additionally, we also
597 // compute the strides corresponding to the left non-contracting
598 // dimensions and right non-contracting dimensions.
599 m_lhs_inner_dim_contiguous = true;
600 int dim_idx = 0;
601 Index nocontract_idx = 0;
602
603 for (int i = 0; i < LDims; i++) {
604 // find if we are contracting on index i of left tensor
605 bool contracting = false;
606 for (int j = 0; j < ContractDims; j++) {
607 if (eval_op_indices[j].first == i) {
608 contracting = true;
609 break;
610 }
611 }
612 if (!contracting) {
613 // add dimension size to output dimensions
614 m_dimensions[dim_idx] = eval_left_dims[i];
615 m_left_nocontract_strides[nocontract_idx] = lhs_strides[i];
616 if (dim_idx != i) {
617 m_lhs_inner_dim_contiguous = false;
618 }
619 m_i_strides[nocontract_idx] = m_i_size;
620 m_i_size *= eval_left_dims[i];
621 dim_idx++;
622 nocontract_idx++;
623 }
624 }
625
626 nocontract_idx = 0;
627 for (int i = 0; i < RDims; i++) {
628 bool contracting = false;
629 // find if we are contracting on index i of right tensor
630 for (int j = 0; j < ContractDims; j++) {
631 if (eval_op_indices[j].second == i) {
632 contracting = true;
633 break;
634 }
635 }
636 if (!contracting) {
637 m_dimensions[dim_idx] = eval_right_dims[i];
638 m_j_strides[nocontract_idx] = m_j_size;
639 m_j_size *= eval_right_dims[i];
640 m_right_nocontract_strides[nocontract_idx] = rhs_strides[i];
641 dim_idx++;
642 nocontract_idx++;
643 }
644 }
645
646 // Now compute the strides corresponding to the contracting dimensions. We
647 // assumed above that non-contracting axes are represented in the same order
648 // in the matrix as they are in the tensor. This is not the case for
649 // contracting axes. As the contracting axes must be of the same size in
650 // each tensor, we'll only look at the first tensor here.
651 m_rhs_inner_dim_contiguous = true;
652 m_rhs_inner_dim_reordered = false;
653 for (int i = 0; i < ContractDims; i++) {
654 Index left = eval_op_indices[i].first;
655 Index right = eval_op_indices[i].second;
656
657 Index size = eval_left_dims[left];
658 eigen_assert(size == eval_right_dims[right] && "Contraction axes must be same size");
659
660 m_k_strides[i] = m_k_size;
661 m_k_size *= size;
662 m_left_contracting_strides[i] = lhs_strides[left];
663 m_right_contracting_strides[i] = rhs_strides[right];
664
665 if (i > 0 && right < eval_op_indices[i - 1].second) {
666 m_rhs_inner_dim_reordered = true;
667 }
668 if (right != i) {
669 m_rhs_inner_dim_contiguous = false;
670 }
671 }
672 eigen_assert(!m_rhs_inner_dim_reordered || (ContractDims > 1 && !m_rhs_inner_dim_contiguous));
673
674 // After the sort above, eval_op_indices[*].first is in ascending order. The
675 // contracted LHS dims are the contiguous prefix of storage iff that prefix
676 // is exactly {0, 1, ..., ContractDims-1}. Similarly for the RHS — and we
677 // also detect the "contracted dims at the trailing end" case, which
678 // matters when LHS/RHS were swapped at the type level for RowMajor inputs.
679 m_lhs_contracted_dims_leading = true;
680 m_rhs_contracted_dims_leading = true;
681 m_rhs_contracted_dims_trailing = true;
682 const int rhs_trail_start = RDims - ContractDims;
683 for (int i = 0; i < ContractDims; i++) {
684 if (eval_op_indices[i].first != i) m_lhs_contracted_dims_leading = false;
685 if (eval_op_indices[i].second != i) m_rhs_contracted_dims_leading = false;
686 if (eval_op_indices[i].second != rhs_trail_start + i) m_rhs_contracted_dims_trailing = false;
687 }
688
689 // If the layout is RowMajor, we need to reverse the m_dimensions
690 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(RowMajor)) {
691 for (int i = 0, j = NumDims - 1; i < j; i++, j--) {
692 numext::swap(m_dimensions[i], m_dimensions[j]);
693 }
694 }
695
696 // A set of parameters that will allow output kernel to get from output
697 // tensor dimensions (i, j) into the original tensor dimensions.
698 // TODO(ezhulenev): Add parameters required to infer output tensor index for
699 // more complex contractions than 2x2 on internal dimension.
700 m_tensor_contraction_params.swapped_arguments = static_cast<int>(Layout) == RowMajor;
701 }
702
703 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; }
704
705 EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(EvaluatorPointerType data) {
706 m_leftImpl.evalSubExprsIfNeeded(nullptr);
707 // The executor does not call cleanup() when evaluation throws, so release the operands and the result here.
708 EIGEN_TRY { m_rightImpl.evalSubExprsIfNeeded(nullptr); }
709 EIGEN_CATCH(...) {
710 m_leftImpl.cleanup();
711 EIGEN_THROW;
712 }
713 EIGEN_TRY {
714 if (data) {
715 evalTo(data);
716 } else {
717 m_result = static_cast<EvaluatorPointerType>(m_device.allocate(dimensions().TotalSize() * sizeof(Scalar)));
718 constructResult();
719 evalTo(m_result);
720 }
721 }
722 EIGEN_CATCH(...) {
723 cleanup();
724 EIGEN_THROW;
725 }
726 return data == nullptr;
727 }
728
729#ifdef EIGEN_USE_THREADS
730 template <typename EvalSubExprsCallback>
731 EIGEN_STRONG_INLINE void evalSubExprsIfNeededAsync(EvaluatorPointerType dest, EvalSubExprsCallback done) {
732 m_leftImpl.evalSubExprsIfNeededAsync(nullptr, [this, done, dest](bool) {
733 m_rightImpl.evalSubExprsIfNeededAsync(nullptr, [this, done, dest](bool) {
734 const bool need_assign = (dest == nullptr);
735 EvaluatorPointerType buffer = dest;
736 if (need_assign) {
737 m_result = static_cast<EvaluatorPointerType>(m_device.allocate(dimensions().TotalSize() * sizeof(Scalar)));
738 constructResult();
739 buffer = m_result;
740 }
741 // One callback type for both branches: each distinct closure type instantiates the whole evalProductAsync tree.
742 evalToAsync(buffer, [done, need_assign]() { done(need_assign); });
743 });
744 });
745 }
746#endif // EIGEN_USE_THREADS
747
748 EIGEN_DEVICE_FUNC void evalTo(Scalar* buffer) const {
749 static_cast<const Derived*>(this)->template evalProduct<Unaligned>(buffer);
750 }
751
752#ifdef EIGEN_USE_THREADS
753 template <typename EvalToCallback>
754 void evalToAsync(Scalar* buffer, EvalToCallback done) const {
755 static_cast<const Derived*>(this)->template evalProductAsync<EvalToCallback, Unaligned>(buffer, std::move(done));
756 }
757#endif // EIGEN_USE_THREADS
758
759 template <typename Func>
760 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void dispatchContraction(Func&& fn, bool lhs_inner_dim_contiguous,
761 bool rhs_inner_dim_contiguous,
762 bool rhs_inner_dim_reordered) {
763 // m_rhs_inner_dim_reordered is only ever set when ContractDims > 1 and
764 // !m_rhs_inner_dim_contiguous. Canonicalizing rhs_r at compile time avoids
765 // instantiating unreachable contractions (reducing 8 combinations to 4 when
766 // ContractDims <= 1, and to 6 when ContractDims > 1).
767 internal::tensor_contraction_dispatch(
768 [&](auto lhs_c, auto rhs_c, auto rhs_r) {
769 constexpr bool kRhsReordered = (ContractDims > 1) && !rhs_c() && rhs_r();
770 fn(lhs_c, rhs_c, internal::bool_constant<kRhsReordered>{});
771 },
772 lhs_inner_dim_contiguous, rhs_inner_dim_contiguous,
773 (ContractDims > 1) && !rhs_inner_dim_contiguous && rhs_inner_dim_reordered);
774 }
775
776 template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment>
777 void evalProductSequential(Scalar* buffer) const {
778 if (this->m_i_size == 0 || this->m_j_size == 0) {
779 return;
780 }
781 if (this->m_k_size == 0) {
782 // Contraction over dimension of size zero results in an all-zero output.
783 this->m_device.fill(buffer, buffer + this->m_i_size * this->m_j_size, Scalar(0));
784 using OutputMapper = internal::blas_data_mapper<Scalar, Index, ColMajor>;
785 this->m_output_kernel(OutputMapper(buffer, this->m_i_size), this->m_tensor_contraction_params,
786 static_cast<Index>(0), static_cast<Index>(0), this->m_i_size, this->m_j_size);
787 return;
788 }
789 // Gemv-shape contractions (output is a vector) get a direct GEMV kernel
790 // call when the operand layout admits one. There are four shapes:
791 //
792 // m_j_size == 1: ColMajor inputs. m_leftImpl is the matrix.
793 // (A) lhs_inner_dim_contiguous: contracted dims at non-contiguous end
794 // of LHS. Existing evalGemv via TensorContractionInputMapper.
795 // (B) m_lhs_contracted_dims_leading: contracted dims at contiguous end.
796 // RowMajor view of LHS via const_blas_data_mapper.
797 //
798 // m_i_size == 1: RowMajor inputs (lines 431-434 swap LHS/RHS at the type
799 // level, so the matrix is m_rightImpl).
800 // (C) m_rhs_contracted_dims_leading: contracted dims at contiguous end
801 // of (eval-)RHS. RowMajor view of m_rightImpl.
802 // (D) m_rhs_contracted_dims_trailing: contracted dims at non-contiguous
803 // end. ColMajor view of m_rightImpl.
804 //
805 // Cases B/C/D require direct memory (data() != nullptr) on both impls.
806 // Anything else falls back to evalGemm.
807 constexpr bool types_match = std::is_same<typename EvalLeftArgType::Scalar, Scalar>::value &&
808 std::is_same<typename EvalRightArgType::Scalar, Scalar>::value;
809 if (this->m_j_size == 1) {
810 if (lhs_inner_dim_contiguous) {
811 this->template evalGemv<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Alignment>(
812 buffer);
813 return;
814 }
815 EIGEN_IF_CONSTEXPR (types_match) {
816 // The vector is read with unit stride, which needs the RHS contracted dims in LHS order.
817 if (m_lhs_contracted_dims_leading && !rhs_inner_dim_reordered && m_leftImpl.data() != nullptr &&
818 m_rightImpl.data() != nullptr) {
819 if (internal::GemvDirectDispatcher<types_match, RowMajor, false>::run(this, buffer)) {
820 return;
821 }
822 }
823 }
824 } else if (this->m_i_size == 1 && m_leftImpl.data() != nullptr && m_rightImpl.data() != nullptr) {
825 EIGEN_IF_CONSTEXPR (types_match) {
826 if (m_rhs_contracted_dims_leading) {
827 if (internal::GemvDirectDispatcher<types_match, RowMajor, true>::run(this, buffer)) {
828 return;
829 }
830 }
831 if (m_rhs_contracted_dims_trailing) {
832 if (internal::GemvDirectDispatcher<types_match, ColMajor, true>::run(this, buffer)) {
833 return;
834 }
835 }
836 }
837 }
838 this->template evalGemm<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Alignment>(
839 buffer);
840 }
841
842 template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment>
843#if !defined(EIGEN_HIPCC)
844 EIGEN_DEVICE_FUNC
845#endif
846 void
847 evalGemv(Scalar* buffer) const {
848 const Index rows = m_i_size;
849 const Index cols = m_k_size;
850
851 using LhsScalar = std::remove_const_t<typename EvalLeftArgType::Scalar>;
852 using RhsScalar = std::remove_const_t<typename EvalRightArgType::Scalar>;
853 using LeftEvaluator = TensorEvaluator<EvalLeftArgType, Device>;
854 using RightEvaluator = TensorEvaluator<EvalRightArgType, Device>;
855 const int lhs_packet_size = internal::unpacket_traits<typename LeftEvaluator::PacketReturnType>::size;
856 const int rhs_packet_size = internal::unpacket_traits<typename RightEvaluator::PacketReturnType>::size;
857 constexpr int lhs_alignment = LeftEvaluator::IsAligned ? Aligned : Unaligned;
858 constexpr int rhs_alignment = RightEvaluator::IsAligned ? Aligned : Unaligned;
859 using LhsMapper = internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs, LeftEvaluator,
860 left_nocontract_t, contract_t, lhs_packet_size,
861 lhs_inner_dim_contiguous, false, lhs_alignment>;
862
863 using RhsMapper =
864 internal::TensorContractionInputMapper<RhsScalar, Index, internal::Rhs, RightEvaluator, right_nocontract_t,
865 contract_t, rhs_packet_size, rhs_inner_dim_contiguous,
866 rhs_inner_dim_reordered, rhs_alignment>;
867
868 LhsMapper lhs(m_leftImpl, m_left_nocontract_strides, m_i_strides, m_left_contracting_strides, m_k_strides);
869 RhsMapper rhs(m_rightImpl, m_right_nocontract_strides, m_j_strides, m_right_contracting_strides, m_k_strides);
870
871 const Scalar alpha(1);
872 const Index resIncr(1);
873
874 // zero out the result buffer (which must be of size at least rows * sizeof(Scalar)
875 m_device.fill(buffer, buffer + rows, Scalar(0));
876
877 internal::general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, false, RhsScalar, RhsMapper,
878 false>::run(rows, cols, lhs, rhs, buffer, resIncr, alpha);
879
880 using OutputMapper = internal::blas_data_mapper<Scalar, Index, ColMajor>;
881 m_output_kernel(OutputMapper(buffer, rows), m_tensor_contraction_params, static_cast<Index>(0),
882 static_cast<Index>(0), rows, static_cast<Index>(1));
883 }
884
885 // Direct gemv fast path: drives the GEMV kernel against raw tensor memory via
886 // const_blas_data_mapper, bypassing TensorContractionInputMapper (which only
887 // delivers vectorized loads along the free dim) and the GEBP packing overhead
888 // of evalGemm.
889 //
890 // StorageOrder is the layout of the (rows × cols) matrix view in memory.
891 // MatrixIsRight selects which TensorEvaluator holds the matrix and which
892 // holds the vector — needed because the constructor swaps LHS/RHS for
893 // RowMajor inputs (lines 431-434), so the gemv shape lives in m_rightImpl
894 // there. Caller guarantees both impls expose direct memory.
895 template <int StorageOrder, bool MatrixIsRight>
896#if !defined(EIGEN_HIPCC)
897 EIGEN_DEVICE_FUNC
898#endif
899 void
900 evalGemvDirect(Scalar* buffer) const {
901 using MatScalar = std::remove_const_t<
902 std::conditional_t<MatrixIsRight, typename EvalRightArgType::Scalar, typename EvalLeftArgType::Scalar>>;
903 using VecScalar = std::remove_const_t<
904 std::conditional_t<MatrixIsRight, typename EvalLeftArgType::Scalar, typename EvalRightArgType::Scalar>>;
905
906 const Index rows = MatrixIsRight ? m_j_size : m_i_size;
907 const Index cols = m_k_size;
908 // For a row-major (rows × cols) view the row stride is cols; for a
909 // column-major view the column stride is rows.
910 const Index mat_stride = (StorageOrder == RowMajor) ? cols : rows;
911
912 const MatScalar* mat_data = MatrixIsRight ? m_rightImpl.data() : m_leftImpl.data();
913 const VecScalar* vec_data = MatrixIsRight ? m_leftImpl.data() : m_rightImpl.data();
914 eigen_assert(mat_data != nullptr && vec_data != nullptr);
915
916 using LhsMapper = internal::const_blas_data_mapper<MatScalar, Index, StorageOrder>;
917 using RhsMapper = internal::const_blas_data_mapper<VecScalar, Index, ColMajor>;
918 LhsMapper lhs(mat_data, mat_stride);
919 RhsMapper rhs(vec_data, /*stride=*/1);
920
921 m_device.fill(buffer, buffer + rows, Scalar(0));
922
923 internal::general_matrix_vector_product<Index, MatScalar, LhsMapper, StorageOrder, false, VecScalar, RhsMapper,
924 false>::run(rows, cols, lhs, rhs, buffer, /*resIncr=*/1, Scalar(1));
925
926 using OutputMapper = internal::blas_data_mapper<Scalar, Index, ColMajor>;
927 m_output_kernel(OutputMapper(buffer, rows), m_tensor_contraction_params, static_cast<Index>(0),
928 static_cast<Index>(0), rows, static_cast<Index>(1));
929 }
930
931 template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment>
932#if !defined(EIGEN_HIPCC)
933 EIGEN_DEVICE_FUNC
934#endif
935 void
936 evalGemm(Scalar* buffer) const {
937 // columns in left side, rows in right side
938 const Index k = this->m_k_size;
939 this->template evalGemmPartial<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered,
940 Alignment, true>(buffer, 0, k, 1);
941 }
942
943 template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment>
944 EIGEN_DEVICE_FUNC void evalGemmPartialWithoutOutputKernel(Scalar* buffer, Index k_start, Index k_end,
945 int num_threads) const {
946 evalGemmPartial<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Alignment,
947 /*use_output_kernel*/ false>(buffer, k_start, k_end, num_threads);
948 }
949
950 template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment,
951 bool use_output_kernel>
952 EIGEN_DEVICE_FUNC void evalGemmPartial(Scalar* buffer, Index k_start, Index k_end, int num_threads) const {
953 eigen_assert(k_end >= k_start && k_start >= 0 && k_end <= this->m_k_size);
954 // columns in slice on left side, rows on right side
955 const Index k_slice = k_end - k_start;
956
957 // rows in left side
958 const Index m = this->m_i_size;
959
960 // columns in right side
961 const Index n = this->m_j_size;
962
963 if (m == 0 || n == 0) return;
964 if (k_slice == 0) {
965 this->m_device.fill(buffer, buffer + m * n, Scalar(0));
966 if (use_output_kernel) {
967 using OutputMapper = internal::blas_data_mapper<Scalar, Index, ColMajor>;
968 m_output_kernel(OutputMapper(buffer, m), m_tensor_contraction_params, static_cast<Index>(0),
969 static_cast<Index>(0), m, n);
970 }
971 return;
972 }
973
974 // define data mappers for Lhs and Rhs
975 using LhsScalar = std::remove_const_t<typename EvalLeftArgType::Scalar>;
976 using RhsScalar = std::remove_const_t<typename EvalRightArgType::Scalar>;
977
978 using LeftEvaluator = TensorEvaluator<EvalLeftArgType, Device>;
979 using RightEvaluator = TensorEvaluator<EvalRightArgType, Device>;
980
981 const int lhs_packet_size = internal::unpacket_traits<typename LeftEvaluator::PacketReturnType>::size;
982 const int rhs_packet_size = internal::unpacket_traits<typename RightEvaluator::PacketReturnType>::size;
983
984 using LhsMapper =
985 internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs, LeftEvaluator, left_nocontract_t,
986 contract_t, lhs_packet_size, lhs_inner_dim_contiguous, false, Unaligned>;
987
988 using RhsMapper =
989 internal::TensorContractionInputMapper<RhsScalar, Index, internal::Rhs, RightEvaluator, right_nocontract_t,
990 contract_t, rhs_packet_size, rhs_inner_dim_contiguous,
991 rhs_inner_dim_reordered, Unaligned>;
992
993 using OutputMapper = internal::blas_data_mapper<Scalar, Index, ColMajor>;
994
995 using TensorContractionKernel =
996 internal::TensorContractionKernel<Scalar, LhsScalar, RhsScalar, Index, OutputMapper, LhsMapper, RhsMapper>;
997
998 // initialize data mappers
999 LhsMapper lhs(this->m_leftImpl, this->m_left_nocontract_strides, this->m_i_strides,
1000 this->m_left_contracting_strides, this->m_k_strides);
1001
1002 RhsMapper rhs(this->m_rightImpl, this->m_right_nocontract_strides, this->m_j_strides,
1003 this->m_right_contracting_strides, this->m_k_strides);
1004
1005 OutputMapper output(buffer, m);
1006
1007 // Sizes of the blocks to load in cache. See the Goto paper for details.
1008 internal::TensorContractionBlocking<Scalar, LhsScalar, RhsScalar, Index, internal::ShardByCol> blocking(
1009 k_slice, m, n, num_threads);
1010 const Index kc = blocking.kc();
1011 const Index mc = numext::mini(m, blocking.mc());
1012 const Index nc = numext::mini(n, blocking.nc());
1013
1014 using LhsBlock = typename TensorContractionKernel::LhsBlock;
1015 using RhsBlock = typename TensorContractionKernel::RhsBlock;
1016
1017 LhsBlock blockA;
1018 RhsBlock blockB;
1019
1020 TensorContractionKernel kernel(m, k_slice, n, mc, kc, nc);
1021
1022 using BlockMemHandle = typename TensorContractionKernel::BlockMemHandle;
1023 const BlockMemHandle packed_mem = kernel.allocate(this->m_device, &blockA, &blockB);
1024 // Releases the packed blocks also when packing, gebp or the output kernel throws.
1025 struct PackedMemGuard {
1026 TensorContractionKernel& kernel;
1027 const Device& device;
1028 const BlockMemHandle& mem;
1029 EIGEN_DEVICE_FUNC ~PackedMemGuard() { kernel.deallocate(device, mem); }
1030 } packed_mem_guard{kernel, this->m_device, packed_mem};
1031
1032 // If a contraction kernel does not support beta, explicitly initialize
1033 // output buffer with zeroes.
1034 EIGEN_IF_CONSTEXPR (!TensorContractionKernel::HasBeta) {
1035 this->m_device.fill(buffer, buffer + m * n, Scalar(0));
1036 }
1037
1038 for (Index i2 = 0; i2 < m; i2 += mc) {
1039 const Index actual_mc = numext::mini(i2 + mc, m) - i2;
1040 for (Index k2 = k_start; k2 < k_end; k2 += kc) {
1041 // make sure we don't overshoot right edge of left matrix, then pack vertical panel
1042 const Index actual_kc = numext::mini(k2 + kc, k_end) - k2;
1043 kernel.packLhs(&blockA, lhs.getSubMapper(i2, k2), actual_kc, actual_mc);
1044
1045 // If kernel supports beta, there is no need to initialize output
1046 // buffer with zeroes.
1047 const Scalar alpha = Scalar(1);
1048 const Scalar beta = (TensorContractionKernel::HasBeta && k2 == k_start) ? Scalar(0) : Scalar(1);
1049
1050 // series of horizontal blocks
1051 for (Index j2 = 0; j2 < n; j2 += nc) {
1052 // make sure we don't overshoot right edge of right matrix, then pack block
1053 const Index actual_nc = numext::mini(j2 + nc, n) - j2;
1054 kernel.packRhs(&blockB, rhs.getSubMapper(k2, j2), actual_kc, actual_nc);
1055
1056 // call gebp (matrix kernel)
1057 // The parameters here are copied from Eigen's GEMM implementation
1058 const OutputMapper output_mapper = output.getSubMapper(i2, j2);
1059 kernel.invoke(output_mapper, blockA, blockB, actual_mc, actual_kc, actual_nc, alpha, beta);
1060
1061 // We are done with this [i2, j2] output block.
1062 if (use_output_kernel && k2 + kc >= k_end) {
1063 m_output_kernel(output_mapper, m_tensor_contraction_params, i2, j2, actual_mc, actual_nc);
1064 }
1065 }
1066 }
1067 }
1068 }
1069
1070 EIGEN_STRONG_INLINE void cleanup() {
1071 m_leftImpl.cleanup();
1072 m_rightImpl.cleanup();
1073 releaseResult();
1074 }
1075
1076 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_result[index]; }
1077
1078 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool) const {
1079 return TensorOpCost(sizeof(CoeffReturnType), 0, 0);
1080 }
1081
1082 template <int LoadMode>
1083 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
1084 return internal::ploadt<PacketReturnType, LoadMode>(m_result + index);
1085 }
1086
1087 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EvaluatorPointerType data() const { return m_result; }
1088
1089 // Required so a contraction can be composed with operators whose own
1090 // getResourceRequirements() forwards into m_impl (TensorPaddingOp,
1091 // TensorBroadcastingOp, etc.). Without this, e.g. an expression like
1092 // `Tensor B = A.contract(C, dims).pad(p)` fails to compile because
1093 // Pad's BlockAccess is gated on m_impl.RawAccess (which is true here)
1094 // and instantiating Pad's getResourceRequirements then requires this
1095 // method on the operand evaluator.
1096 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
1097 return internal::TensorBlockResourceRequirements::any();
1098 }
1099
1100 protected:
1101 // The temporary result is written by assignment (fill, gebp), so scalars that require initialization must be
1102 // constructed first. The GPU evaluator allocates m_result itself without constructing it, so cleanup() destructs only
1103 // what was constructed here.
1104 EIGEN_STRONG_INLINE void constructResult() {
1105 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
1106 EIGEN_TRY { internal::default_construct_elements_of_array(m_result, dimensions().TotalSize()); }
1107 EIGEN_CATCH(...) {
1108 m_device.deallocate(m_result);
1109 m_result = nullptr;
1110 EIGEN_THROW;
1111 }
1112 m_result_constructed = true;
1113 }
1114 }
1115
1116 EIGEN_STRONG_INLINE void releaseResult() {
1117 if (m_result != nullptr) {
1118 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
1119 if (m_result_constructed) internal::destruct_elements_of_array(m_result, dimensions().TotalSize());
1120 m_result_constructed = false;
1121 }
1122 m_device.deallocate(m_result);
1123 m_result = nullptr;
1124 }
1125 }
1126
1127 Dimensions m_dimensions;
1128
1129 contract_t m_k_strides{};
1130 contract_t m_left_contracting_strides{};
1131 contract_t m_right_contracting_strides{};
1132
1133 bool m_lhs_inner_dim_contiguous;
1134 bool m_rhs_inner_dim_contiguous;
1135 bool m_rhs_inner_dim_reordered;
1136 // Set in the constructor; describe whether the contracted LHS/RHS dims
1137 // form a contiguous block at the leading or trailing end of storage. Used
1138 // by evalProductSequential to pick a direct GEMV path when applicable.
1139 bool m_lhs_contracted_dims_leading;
1140 bool m_rhs_contracted_dims_leading;
1141 bool m_rhs_contracted_dims_trailing;
1142
1143 left_nocontract_t m_i_strides{};
1144 right_nocontract_t m_j_strides{};
1145 left_nocontract_t m_left_nocontract_strides{};
1146 right_nocontract_t m_right_nocontract_strides{};
1147
1148 Index m_i_size;
1149 Index m_j_size;
1150 Index m_k_size;
1151
1152 TensorContractionParams m_tensor_contraction_params;
1153
1154 TensorEvaluator<EvalLeftArgType, Device> m_leftImpl;
1155 TensorEvaluator<EvalRightArgType, Device> m_rightImpl;
1156 const Device EIGEN_DEVICE_REF m_device;
1157 OutputKernelType m_output_kernel;
1158 EvaluatorPointerType m_result;
1159 bool m_result_constructed = false;
1160};
1161
1162// evaluator for default device
1163template <typename Indices, typename LeftArgType, typename RightArgType, typename OutputKernelType, typename Device>
1164struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Device>
1165 : public TensorContractionEvaluatorBase<
1166 TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Device>> {
1167 using Self = TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Device>;
1168 using Base = TensorContractionEvaluatorBase<Self>;
1169
1170 using XprType = TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>;
1171 using Scalar = std::remove_const_t<typename XprType::Scalar>;
1172 using Index = typename XprType::Index;
1173 using CoeffReturnType = typename XprType::CoeffReturnType;
1174 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
1175
1176 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
1177
1178 // Most of the code is assuming that both input tensors are ColMajor. If the
1179 // inputs are RowMajor, we will "cheat" by swapping the LHS and RHS:
1180 // If we want to compute A * B = C, where A is LHS and B is RHS, the code
1181 // will pretend B is LHS and A is RHS.
1182 using EvalLeftArgType = std::conditional_t<Layout == static_cast<int>(ColMajor), LeftArgType, RightArgType>;
1183 using EvalRightArgType = std::conditional_t<Layout == static_cast<int>(ColMajor), RightArgType, LeftArgType>;
1184
1185 static constexpr int LDims =
1186 internal::array_size<typename TensorEvaluator<EvalLeftArgType, Device>::Dimensions>::value;
1187 static constexpr int RDims =
1188 internal::array_size<typename TensorEvaluator<EvalRightArgType, Device>::Dimensions>::value;
1189 static constexpr int ContractDims = internal::array_size<Indices>::value;
1190
1191 using contract_t = array<Index, ContractDims>;
1192 using left_nocontract_t = array<Index, LDims - ContractDims>;
1193 using right_nocontract_t = array<Index, RDims - ContractDims>;
1194
1195 static constexpr int NumDims = LDims + RDims - 2 * ContractDims;
1196
1197 // Could we use NumDimensions here?
1198 using Dimensions = DSizes<Index, NumDims>;
1199
1200 TensorEvaluator(const XprType& op, const Device& device) : Base(op, device) {}
1201
1202 template <int Alignment>
1203 void evalProduct(Scalar* buffer) const {
1204 Base::dispatchContraction(
1205 [&](auto lhs_c, auto rhs_c, auto rhs_r) {
1206 this->template evalProductSequential<lhs_c(), rhs_c(), rhs_r(), Alignment>(buffer);
1207 },
1208 this->m_lhs_inner_dim_contiguous, this->m_rhs_inner_dim_contiguous, this->m_rhs_inner_dim_reordered);
1209 }
1210};
1211
1212} // end namespace Eigen
1213
1214#endif // EIGEN_TENSOR_TENSOR_CONTRACTION_H
The tensor base class.
Definition TensorForwardDeclarations.h:69
Definition TensorContraction.h:418
const internal::remove_all_t< typename LhsXprType::Nested > & lhsExpression() const
Definition TensorContraction.h:435
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47