11#ifndef EIGEN_TENSOR_TENSOR_CONTRACTION_THREAD_POOL_H
12#define EIGEN_TENSOR_TENSOR_CONTRACTION_THREAD_POOL_H
15#ifdef EIGEN_USE_THREADS
18#include "./InternalHeaderCheck.h"
22template <
typename Indices,
typename LeftArgType,
typename RightArgType,
typename OutputKernelType>
25 :
public TensorContractionEvaluatorBase<TensorEvaluator<
26 const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, ThreadPoolDevice>> {
27 using Device = ThreadPoolDevice;
29 using Self = TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>, Device>;
30 using Base = TensorContractionEvaluatorBase<Self>;
32 using XprType = TensorContractionOp<Indices, LeftArgType, RightArgType, OutputKernelType>;
33 using Scalar = std::remove_const_t<typename XprType::Scalar>;
34 using Index =
typename XprType::Index;
35 using CoeffReturnType =
typename XprType::CoeffReturnType;
36 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
38 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
44 using EvalLeftArgType =
45 std::conditional_t<static_cast<int>(Layout) ==
static_cast<int>(
ColMajor), LeftArgType, RightArgType>;
46 using EvalRightArgType =
47 std::conditional_t<static_cast<int>(Layout) ==
static_cast<int>(
ColMajor), RightArgType, LeftArgType>;
49 static constexpr int LDims =
50 internal::array_size<typename TensorEvaluator<EvalLeftArgType, Device>::Dimensions>::value;
51 static constexpr int RDims =
52 internal::array_size<typename TensorEvaluator<EvalRightArgType, Device>::Dimensions>::value;
53 static constexpr int ContractDims = internal::array_size<Indices>::value;
55 using left_dim_mapper_t = array<Index, LDims>;
56 using right_dim_mapper_t = array<Index, RDims>;
58 using contract_t = array<Index, ContractDims>;
59 using left_nocontract_t = array<Index, LDims - ContractDims>;
60 using right_nocontract_t = array<Index, RDims - ContractDims>;
62 static constexpr int NumDims = LDims + RDims - 2 * ContractDims;
64 using Dimensions = DSizes<Index, NumDims>;
67 using LhsScalar = std::remove_const_t<typename EvalLeftArgType::Scalar>;
68 using RhsScalar = std::remove_const_t<typename EvalRightArgType::Scalar>;
69 using Traits =
typename internal::gebp_traits<LhsScalar, RhsScalar>;
71 using LeftEvaluator = TensorEvaluator<EvalLeftArgType, Device>;
72 using RightEvaluator = TensorEvaluator<EvalRightArgType, Device>;
74 TensorEvaluator(
const XprType& op,
const Device& device) : Base(op, device) {}
76 template <
int Alignment>
77 void evalProduct(Scalar* buffer)
const {
78 evalProductImpl<NoCallback, Alignment>(buffer, NoCallback());
81 template <
typename EvalToCallback,
int Alignment>
82 void evalProductAsync(Scalar* buffer, EvalToCallback done)
const {
83 evalProductImpl<EvalToCallback, Alignment>(buffer, std::move(done));
86 using Base::dispatchContraction;
88 template <
typename DoneCallback,
int Alignment>
89 void evalProductImpl(Scalar* buffer, DoneCallback done)
const {
105 static constexpr bool IsEvalInSyncMode = std::is_same<DoneCallback, NoCallback>::value;
107 const Index m = this->m_i_size;
108 const Index n = this->m_j_size;
109 const Index k = this->m_k_size;
110 if (m == 0 || n == 0) {
111 EIGEN_IF_CONSTEXPR (!IsEvalInSyncMode) done();
116 [&](
auto lhs_c,
auto rhs_c,
auto rhs_r) {
117 this->
template evalProductSequential<lhs_c(), rhs_c(), rhs_r(), Unaligned>(buffer);
119 this->m_lhs_inner_dim_contiguous, this->m_rhs_inner_dim_contiguous, this->m_rhs_inner_dim_reordered);
120 EIGEN_IF_CONSTEXPR (!IsEvalInSyncMode) done();
147 bool shard_by_col = shardByCol(m, n, 2);
153 internal::TensorContractionBlocking<Scalar, LhsScalar, RhsScalar, Index, internal::ShardByCol> blocking(k, m, n,
159 internal::TensorContractionBlocking<Scalar, LhsScalar, RhsScalar, Index, internal::ShardByRow> blocking(k, m, n,
170 const TensorOpCost cost = contractionCost(m, n, bm, bn, bk, shard_by_col,
false);
172 TensorCostModel<ThreadPoolDevice>::numThreads(
static_cast<double>(n) * m, cost, this->m_device.numThreads());
173 int num_threads_by_k = numThreadsInnerDim(m, n, k);
174 if (shardByInnerDim(m, n, k, num_threads, num_threads_by_k)) {
177 EIGEN_IF_CONSTEXPR (IsEvalInSyncMode) {
178 EvalShardedByInnerDimContext<DoneCallback> ctx(
this, num_threads_by_k, buffer, m, n, k, std::move(done));
179 ctx.template run<Alignment>();
182 new EvalShardedByInnerDimContext<DoneCallback>(
this, num_threads_by_k, buffer, m, n, k, std::move(done));
183 ctx->template runAsync<Alignment>();
191 if (n == 1) num_threads = 1;
193 if (num_threads == 1) {
195 [&](
auto lhs_c,
auto rhs_c,
auto rhs_r) {
196 this->
template evalProductSequential<lhs_c(), rhs_c(), rhs_r(), Unaligned>(buffer);
198 this->m_lhs_inner_dim_contiguous, this->m_rhs_inner_dim_contiguous, this->m_rhs_inner_dim_reordered);
199 EIGEN_IF_CONSTEXPR (!IsEvalInSyncMode) done();
204 shard_by_col = shardByCol(m, n, num_threads);
206 internal::TensorContractionBlocking<Scalar, LhsScalar, RhsScalar, Index, internal::ShardByCol> blocking(
207 k, m, n, num_threads);
212 internal::TensorContractionBlocking<Scalar, LhsScalar, RhsScalar, Index, internal::ShardByRow> blocking(
213 k, m, n, num_threads);
220 Index nm0 = numext::div_ceil(m, bm);
221 Index nn0 = numext::div_ceil(n, bn);
222 Index nk = numext::div_ceil(k, bk);
233 gm = coarsenM(m, n, bm, bn, bk, gn, num_threads, shard_by_col);
234 gn = coarsenN(m, n, bm, bn, bk, gm, num_threads, shard_by_col);
236 gn = coarsenN(m, n, bm, bn, bk, gm, num_threads, shard_by_col);
237 gm = coarsenM(m, n, bm, bn, bk, gn, num_threads, shard_by_col);
240 Index nm = numext::div_ceil(nm0, gm);
241 Index nn = numext::div_ceil(nn0, gn);
247 const Index sharding_dim_tasks = shard_by_col ? nn : nm;
248 const int num_worker_threads = this->m_device.numThreadsInPool();
253 const float oversharding_factor = num_worker_threads <= 4 ? 8.0
254 : num_worker_threads <= 8 ? 4.0
255 : num_worker_threads <= 16 ? 2.0
256 : num_worker_threads <= 32 ? 1.0
257 : num_worker_threads <= 64 ? 0.8
260 const bool parallelize_by_sharding_dim_only = sharding_dim_tasks >= oversharding_factor * num_worker_threads;
269 bool parallel_pack = num_threads >= nm * nn;
271 if (m * bk * Index(
sizeof(LhsScalar)) + n * bk * Index(
sizeof(RhsScalar)) <= l2CacheSize() * num_threads)
272 parallel_pack =
true;
275 if ((shard_by_col ? nm : nn) == 1) parallel_pack =
false;
278 if (parallelize_by_sharding_dim_only) parallel_pack =
false;
281 [&](
auto lhs_c,
auto rhs_c,
auto rhs_r) {
282 EIGEN_IF_CONSTEXPR (IsEvalInSyncMode) {
283 EvalParallelContext<NoCallback, lhs_c(), rhs_c(), rhs_r()> ctx(
284 this, num_threads, buffer, m, n, k, bm, bn, bk, nm, nn, nk, gm, gn, nm0, nn0, shard_by_col,
285 parallel_pack, parallelize_by_sharding_dim_only, NoCallback());
288 auto* ctx =
new EvalParallelContext<DoneCallback, lhs_c(), rhs_c(), rhs_r()>(
289 this, num_threads, buffer, m, n, k, bm, bn, bk, nm, nn, nk, gm, gn, nm0, nn0, shard_by_col,
290 parallel_pack, parallelize_by_sharding_dim_only, std::move(done));
294 this->m_lhs_inner_dim_contiguous, this->m_rhs_inner_dim_contiguous, this->m_rhs_inner_dim_reordered);
302 void operator()()
const { eigen_assert(
false &&
"NoCallback should never be called"); }
307 template <
typename DoneCallback,
typename Context>
308 class EvalParallelNotification;
311 template <
typename Context>
312 class EvalParallelNotification<NoCallback, Context> {
314 EvalParallelNotification(Context*, NoCallback) {}
315 void Notify() { done_.Notify(); }
316 void Wait() { done_.Wait(); }
319 Eigen::Notification done_;
323 template <
typename DoneCallback,
typename Context>
324 class EvalParallelNotification {
326 EvalParallelNotification(Context* ctx, DoneCallback done) : ctx_(ctx), done_(std::move(done)) {}
332 DoneCallback done_copy = std::move(done_);
352 template <
typename DoneCallback,
bool lhs_inner_dim_contiguous,
bool rhs_inner_dim_contiguous,
353 bool rhs_inner_dim_reordered>
354 class EvalParallelContext {
357 internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs, LeftEvaluator, left_nocontract_t,
358 contract_t, internal::packet_traits<LhsScalar>::size,
359 lhs_inner_dim_contiguous,
false,
Unaligned>;
361 internal::TensorContractionInputMapper<RhsScalar, Index, internal::Rhs, RightEvaluator, right_nocontract_t,
362 contract_t, internal::packet_traits<RhsScalar>::size,
363 rhs_inner_dim_contiguous, rhs_inner_dim_reordered,
Unaligned>;
365 using OutputMapper = internal::blas_data_mapper<Scalar, Index, ColMajor>;
367 using TensorContractionKernel =
368 internal::TensorContractionKernel<Scalar, LhsScalar, RhsScalar, Index, OutputMapper, LhsMapper, RhsMapper>;
370 using LhsBlock =
typename TensorContractionKernel::LhsBlock;
371 using RhsBlock =
typename TensorContractionKernel::RhsBlock;
372 using BlockMemHandle =
typename TensorContractionKernel::BlockMemHandle;
374 EvalParallelContext(
const Self* self,
int num_threads, Scalar* buffer, Index tm, Index tn, Index tk, Index bm,
375 Index bn, Index bk, Index nm, Index nn, Index nk, Index gm, Index gn, Index nm0, Index nn0,
376 bool shard_by_col,
bool parallel_pack,
bool parallelize_by_sharding_dim_only, DoneCallback done)
377 : created_by_thread_id_(std::this_thread::get_id()),
378 done_(this, std::move(done)),
379 device_(self->m_device),
380 lhs_(self->m_leftImpl, self->m_left_nocontract_strides, self->m_i_strides, self->m_left_contracting_strides,
382 rhs_(self->m_rightImpl, self->m_right_nocontract_strides, self->m_j_strides,
383 self->m_right_contracting_strides, self->m_k_strides),
386 output_kernel_(self->m_output_kernel),
387 tensor_contraction_params_(self->m_tensor_contraction_params),
388 num_threads_(num_threads),
389 shard_by_col_(shard_by_col),
390 parallel_pack_(parallel_pack),
391 parallelize_by_sharding_dim_only_(parallelize_by_sharding_dim_only),
405 kernel_(m_, k_, n_, bm_, bk_, bn_),
406 num_thread_local_allocations_(0),
410 thread_local_capacity(2 * (parallelize_by_sharding_dim_only_ ? device_.numThreadsInPool() : 0)),
413 lhs_thread_local_blocks_(shard_by_col_ ? 0 : thread_local_capacity, {*
this}, {*
this}),
414 rhs_thread_local_blocks_(shard_by_col_ ? thread_local_capacity : 0, {*
this}, {*
this}) {
416 eigen_assert(!(parallel_pack && parallelize_by_sharding_dim_only));
418 for (Index x = 0; x < P; x++) {
424 x == 0 ? 1 : (parallel_pack_ ? nn_ + nm_ : (shard_by_col_ ? nn_ : nm_)) + (x == P - 1 ? nm_ * nn_ : 0);
425 state_packing_ready_[x] = parallel_pack_ ? 0 : (shard_by_col_ ? nm_ : nn_);
426 state_kernel_[x] =
new std::atomic<uint8_t>*[nm_];
427 for (Index m = 0; m < nm_; m++) {
428 state_kernel_[x][m] =
new std::atomic<uint8_t>[nn_];
432 for (Index n = 0; n < nn_; n++)
433 state_kernel_[x][m][n].store((x == 0 ? 0 : 1) + (parallel_pack_ ? 2 : 1), std::memory_order_relaxed);
439 bool packed_mem_allocated =
false;
440 can_use_thread_local_packed_ =
nullptr;
444 const bool skip_sharded_global_slice = parallelize_by_sharding_dim_only_ && nk_ == 1;
445 const Index num_global_lhs = (skip_sharded_global_slice && !shard_by_col_) ? 0 : nm0_;
446 const Index num_global_rhs = (skip_sharded_global_slice && shard_by_col_) ? 0 : nn0_;
447 packed_mem_ = kernel_.allocateSlices(
451 std::min<Index>(nk_, P - 1),
452 packed_lhs_, packed_rhs_);
453 packed_mem_allocated =
true;
455 if (parallelize_by_sharding_dim_only_) {
456 const int num_worker_threads = device_.numThreadsInPool();
459 can_use_thread_local_packed_ =
new std::atomic<bool>[nn_];
460 for (
int i = 0; i < nn_; ++i) can_use_thread_local_packed_[i].store(
true, std::memory_order_relaxed);
462 Index num_blocks = num_worker_threads * gn_;
463 thread_local_pre_allocated_mem_ = kernel_.allocateSlices(
468 nullptr, &rhs_thread_local_pre_allocated_);
471 can_use_thread_local_packed_ =
new std::atomic<bool>[nm_];
472 for (
int i = 0; i < nm_; ++i) can_use_thread_local_packed_[i].store(
true, std::memory_order_relaxed);
474 Index num_blocks = num_worker_threads * gm_;
475 thread_local_pre_allocated_mem_ = kernel_.allocateSlices(
479 1, &lhs_thread_local_pre_allocated_,
485 if (packed_mem_allocated) kernel_.deallocate(device_, packed_mem_);
486 delete[] can_use_thread_local_packed_;
487 for (Index x = 0; x < P; x++) {
488 for (Index m = 0; m < nm_; m++)
delete[] state_kernel_[x][m];
489 delete[] state_kernel_[x];
491 EIGEN_IF_CONSTEXPR (std::is_same<DoneCallback, NoCallback>::value) done_.Notify();
496 ~EvalParallelContext() {
497 for (Index x = 0; x < P; x++) {
498 for (Index m = 0; m < nm_; m++)
delete[] state_kernel_[x][m];
499 delete[] state_kernel_[x];
501 kernel_.deallocate(device_, packed_mem_);
502 if (parallelize_by_sharding_dim_only_) {
503 kernel_.deallocate(device_, thread_local_pre_allocated_mem_);
504 delete[] can_use_thread_local_packed_;
510 void run() noexcept {
531 std::thread::id created_by_thread_id_;
535 EvalParallelNotification<DoneCallback, EvalParallelContext> done_;
537 const Device& device_;
540 Scalar*
const buffer_;
541 OutputMapper output_;
542 OutputKernelType output_kernel_;
543 TensorContractionParams tensor_contraction_params_;
544 const int num_threads_;
545 const bool shard_by_col_;
546 const bool parallel_pack_;
547 const bool parallelize_by_sharding_dim_only_;
568 TensorContractionKernel kernel_;
604 static constexpr Index P = 3;
607 BlockMemHandle packed_mem_;
608 std::vector<LhsBlock> packed_lhs_[P - 1];
609 std::vector<RhsBlock> packed_rhs_[P - 1];
629 BlockMemHandle thread_local_pre_allocated_mem_;
633 std::vector<LhsBlock> lhs_thread_local_pre_allocated_;
634 std::vector<RhsBlock> rhs_thread_local_pre_allocated_;
637 std::atomic<int> num_thread_local_allocations_;
638 const int thread_local_capacity;
646 template <
typename BlockType>
647 class ThreadLocalBlocks {
649 ThreadLocalBlocks() =
default;
651 ThreadLocalBlocks(BlockType* base,
size_t grain_size)
652 : is_pre_allocated_(true), thread_local_pre_allocated_base_(base), grain_size_(grain_size) {}
654 ThreadLocalBlocks(BlockMemHandle mem_handle, std::vector<BlockType> blocks)
655 : is_pre_allocated_(false), mem_handle_(std::move(mem_handle)), blocks_(std::move(blocks)) {}
657 BlockType& block(
int grain_index) {
658 eigen_assert(grain_index >= 0);
659 eigen_assert(
static_cast<size_t>(grain_index) < size());
660 return is_pre_allocated_ ? thread_local_pre_allocated_base_[grain_index] : blocks_[grain_index];
663 void Release(EvalParallelContext& ctx)
const {
664 if (!is_pre_allocated_) {
665 ctx.kernel_.deallocate(ctx.device_, mem_handle_);
669 size_t size()
const {
return is_pre_allocated_ ? grain_size_ : blocks_.size(); }
672 bool is_pre_allocated_;
675 BlockType* thread_local_pre_allocated_base_ =
nullptr;
676 size_t grain_size_ = 0;
679 BlockMemHandle mem_handle_{};
680 std::vector<BlockType> blocks_;
689 template <
typename BlockType,
bool is_rhs>
690 class ThreadLocalBlocksInitialize {
691 static constexpr bool kIsLhs = !is_rhs && std::is_same<BlockType, LhsBlock>::value;
692 static constexpr bool kIsRhs = is_rhs && std::is_same<BlockType, RhsBlock>::value;
693 static_assert(kIsLhs || kIsRhs,
"Unknown block type");
695 using Blocks = ThreadLocalBlocks<BlockType>;
698 ThreadLocalBlocksInitialize(EvalParallelContext& ctx)
699 : ctx_(ctx), num_worker_threads_(ctx_.device_.numThreadsInPool()) {}
701 void operator()(Blocks& blocks) {
702 const int n = ctx_.num_thread_local_allocations_.fetch_add(1, std::memory_order_relaxed);
704 if (n >= num_worker_threads_) {
705 ThreadLocalBlocksAllocator<is_rhs>::allocate(ctx_, blocks);
707 ThreadLocalBlocksAllocator<is_rhs>::reuse(ctx_, n, blocks);
714 template <
bool pack_rhs,
typename EvalCtx = EvalParallelContext>
715 struct ThreadLocalBlocksAllocator;
717 template <
typename EvalCtx>
718 struct ThreadLocalBlocksAllocator<true, EvalCtx> {
719 static void allocate(EvalCtx& ctx, Blocks& blocks) {
720 std::vector<RhsBlock> rhs_blocks;
721 BlockMemHandle mem_handle = ctx.kernel_.allocateSlices(ctx.device_,
725 nullptr, &rhs_blocks);
726 blocks = ThreadLocalBlocks<RhsBlock>(std::move(mem_handle), std::move(rhs_blocks));
729 static void reuse(EvalCtx& ctx,
int index, Blocks& blocks) {
730 RhsBlock* ptr = &ctx.rhs_thread_local_pre_allocated_[ctx.gn_ * index];
731 blocks = ThreadLocalBlocks<RhsBlock>(ptr, ctx.gn_);
735 template <
typename EvalCtx>
736 struct ThreadLocalBlocksAllocator<false, EvalCtx> {
737 static void allocate(EvalCtx& ctx, Blocks& blocks) {
738 std::vector<LhsBlock> lhs_blocks;
739 BlockMemHandle mem_handle = ctx.kernel_.allocateSlices(ctx.device_,
743 &lhs_blocks,
nullptr);
744 blocks = ThreadLocalBlocks<LhsBlock>(std::move(mem_handle), std::move(lhs_blocks));
747 static void reuse(EvalCtx& ctx,
int index, Blocks& blocks) {
748 LhsBlock* ptr = &ctx.lhs_thread_local_pre_allocated_[ctx.gm_ * index];
749 blocks = ThreadLocalBlocks<LhsBlock>(ptr, ctx.gm_);
753 EvalParallelContext& ctx_;
754 const int num_worker_threads_;
757 template <
typename BlockType>
758 class ThreadLocalBlocksRelease {
760 using Blocks = ThreadLocalBlocks<BlockType>;
761 ThreadLocalBlocksRelease(EvalParallelContext& ctx) : ctx_(ctx) {}
762 void operator()(Blocks& blocks) { blocks.Release(ctx_); }
765 EvalParallelContext& ctx_;
769 using ThreadLocalLhsInit = ThreadLocalBlocksInitialize<LhsBlock,
false>;
770 using ThreadLocalRhsInit = ThreadLocalBlocksInitialize<RhsBlock,
true>;
773 using ThreadLocalLhsRelease = ThreadLocalBlocksRelease<LhsBlock>;
774 using ThreadLocalRhsRelease = ThreadLocalBlocksRelease<RhsBlock>;
778 Eigen::ThreadLocal<ThreadLocalBlocks<LhsBlock>, ThreadLocalLhsInit, ThreadLocalLhsRelease> lhs_thread_local_blocks_;
779 Eigen::ThreadLocal<ThreadLocalBlocks<RhsBlock>, ThreadLocalRhsInit, ThreadLocalRhsRelease> rhs_thread_local_blocks_;
786 std::atomic<bool>* can_use_thread_local_packed_;
788 std::atomic<uint8_t>** state_kernel_[P];
793 std::atomic<Index> state_packing_ready_[P];
794 std::atomic<Index> state_switch_[P];
796 LhsBlock& packed_lhs(Index m, Index k, Index m1,
bool use_thread_local) {
797 if (use_thread_local) {
798 eigen_assert(!shard_by_col_);
799 ThreadLocalBlocks<LhsBlock>& blocks = lhs_thread_local_blocks_.local();
801 Index grain_index = m1 - m * gm_;
803 internal::convert_index<int>(grain_index));
805 return packed_lhs_[k % (P - 1)][m1];
809 RhsBlock& packed_rhs(Index n, Index k, Index n1,
bool use_thread_local) {
810 if (use_thread_local) {
811 eigen_assert(shard_by_col_);
812 ThreadLocalBlocks<RhsBlock>& blocks = rhs_thread_local_blocks_.local();
814 Index grain_index = n1 - n * gn_;
816 internal::convert_index<int>(grain_index));
818 return packed_rhs_[k % (P - 1)][n1];
832 void pack_lhs(Index m, Index k) {
833 bool use_thread_local =
false;
835 if (parallelize_by_sharding_dim_only_ && !shard_by_col_ &&
836 can_use_thread_local_packed_[m].load(std::memory_order_relaxed)) {
837 if (state_kernel_[k % P][m][0].load(std::memory_order_relaxed) == 1) {
838 use_thread_local =
true;
844 can_use_thread_local_packed_[m].store(
false, std::memory_order_relaxed);
848 const Index mend = m * gm_ + gm(m);
849 for (Index m1 = m * gm_; m1 < mend; m1++)
850 kernel_.packLhs(&packed_lhs(m, k, m1, use_thread_local), lhs_.getSubMapper(m1 * bm_, k * bk_), bk(k), bm(m1));
852 if (!parallel_pack_ && shard_by_col_) {
853 eigen_assert(!use_thread_local);
856 signal_switch(k + 1);
857 for (Index n = nn_ - 1; n >= 0; n--) {
858 bool sync = parallelize_by_sharding_dim_only_ || n == 0;
859 signal_kernel(m, n, k, sync, use_thread_local);
864 void pack_rhs(Index n, Index k) {
865 bool use_thread_local =
false;
867 if (parallelize_by_sharding_dim_only_ && shard_by_col_ &&
868 can_use_thread_local_packed_[n].load(std::memory_order_relaxed)) {
869 if (state_kernel_[k % P][0][n].load(std::memory_order_relaxed) == 1) {
870 use_thread_local =
true;
876 can_use_thread_local_packed_[n].store(
false, std::memory_order_relaxed);
880 const Index nend = n * gn_ + gn(n);
881 for (Index n1 = n * gn_; n1 < nend; n1++) {
882 EIGEN_IF_CONSTEXPR (!TensorContractionKernel::HasBeta) {
893 std::fill_n(buffer_ + n1 * bn_ * m_, bn(n1) * m_, Scalar(0));
896 kernel_.packRhs(&packed_rhs(n, k, n1, use_thread_local), rhs_.getSubMapper(k * bk_, n1 * bn_), bk(k), bn(n1));
899 if (parallel_pack_ || shard_by_col_) {
900 signal_switch(k + 1);
901 for (Index m = nm_ - 1; m >= 0; m--) {
902 bool sync = parallelize_by_sharding_dim_only_ || m == 0;
903 signal_kernel(m, n, k, sync, use_thread_local);
906 eigen_assert(!use_thread_local);
911 void kernel(Index m, Index n, Index k,
bool use_thread_local) {
915 const Index nend = n * gn_ + gn(n);
916 const Index mend = m * gm_ + gm(m);
919 const Scalar alpha = Scalar(1);
920 const Scalar beta = (TensorContractionKernel::HasBeta && k == 0) ? Scalar(0) : Scalar(1);
923 for (Index n1 = n * gn_; n1 < nend; n1++) {
924 for (Index m1 = m * gm_; m1 < mend; m1++) {
925 const auto output_mapper = output_.getSubMapper(m1 * bm_, n1 * bn_);
926 kernel_.invoke(output_mapper, packed_lhs(m, k, m1, !shard_by_col_ && use_thread_local),
927 packed_rhs(n, k, n1, shard_by_col_ && use_thread_local), bm(m1), bk(k), bn(n1), alpha, beta);
931 output_kernel_(output_mapper, tensor_contraction_params_, m1 * bm_, n1 * bn_, bm(m1), bn(n1));
936 for (Index m1 = m * gm_; m1 < mend; m1++)
937 for (Index n1 = n * gn_; n1 < nend; n1++) {
938 const auto output_mapper = output_.getSubMapper(m1 * bm_, n1 * bn_);
939 kernel_.invoke(output_mapper, packed_lhs(m, k, m1, !shard_by_col_ && use_thread_local),
940 packed_rhs(n, k, n1, shard_by_col_ && use_thread_local), bm(m1), bk(k), bn(n1), alpha, beta);
944 output_kernel_(output_mapper, tensor_contraction_params_, m1 * bm_, n1 * bn_, bm(m1), bn(n1));
948 signal_kernel(m, n, k + 1,
false,
false);
949 signal_switch(k + 2);
952 void signal_packing(Index k) {
953 eigen_assert(!parallel_pack_);
954 Index s = state_packing_ready_[k % P].fetch_sub(1);
957 state_packing_ready_[k % P] = shard_by_col_ ? nm_ : nn_;
958 enqueue_packing(k, shard_by_col_);
961 void signal_kernel(Index m, Index n, Index k,
bool sync,
bool use_thread_local) {
962 std::atomic<uint8_t>* state = &state_kernel_[k % P][m][n];
963 Index s = state->load();
965 if (s != 1 && state->fetch_sub(1) != 1) {
966 eigen_assert(!use_thread_local);
969 state->store(parallel_pack_ ? 3 : 2, std::memory_order_relaxed);
971 kernel(m, n, k, use_thread_local);
973 eigen_assert(!use_thread_local);
974 device_.enqueue([
this, m, n, k, use_thread_local]() { kernel(m, n, k, use_thread_local); });
978 void signal_switch(Index k, Index v = 1) {
979 Index s = state_switch_[k % P].fetch_sub(v);
980 eigen_assert(s >= v);
985 state_switch_[k % P] = (parallel_pack_ ? nm_ + nn_ : (shard_by_col_ ? nn_ : nm_)) + nm_ * nn_;
989 if (parallel_pack_) {
990 enqueue_packing(k, !shard_by_col_);
991 enqueue_packing(k, shard_by_col_);
992 }
else if (shard_by_col_) {
993 enqueue_packing(k,
false);
995 enqueue_packing(k,
true);
1003 }
else if (k == nk_) {
1004 signal_switch(k + 1, parallel_pack_ ? nm_ + nn_ : (shard_by_col_ ? nn_ : nm_));
1011 void enqueue_packing(Index k,
bool rhs) { enqueue_packing_helper(0, rhs ? nn_ : nm_, k, rhs); }
1013 void enqueue_packing_helper(Index start, Index end, Index k,
bool rhs) {
1014 if (end - start == 1) {
1020 while (end - start > 1) {
1021 Index mid = (start + end) / 2;
1022 device_.enqueue([
this, mid, end, k, rhs]() { enqueue_packing_helper(mid, end, k, rhs); });
1034 bool pack_async = (start == 0) && (parallelize_by_sharding_dim_only_ && shard_by_col_ == rhs) &&
1035 (k > 0 || std::this_thread::get_id() == created_by_thread_id_);
1038 device_.enqueue([
this, start, end, k, rhs]() { enqueue_packing_helper(start, end, k, rhs); });
1040 enqueue_packing_helper(start, end, k, rhs);
1046 Index bm(Index m)
const {
return m + 1 < nm0_ ? bm_ : m_ + bm_ - bm_ * nm0_; }
1047 Index bn(Index n)
const {
return n + 1 < nn0_ ? bn_ : n_ + bn_ - bn_ * nn0_; }
1048 Index bk(Index k)
const {
return k + 1 < nk_ ? bk_ : k_ + bk_ - bk_ * nk_; }
1050 Index gm(Index m)
const {
return m + 1 < nm_ ? gm_ : nm0_ + gm_ - gm_ * nm_; }
1051 Index gn(Index n)
const {
return n + 1 < nn_ ? gn_ : nn0_ + gn_ - gn_ * nn_; }
1053 EvalParallelContext(
const EvalParallelContext&) =
delete;
1054 void operator=(
const EvalParallelContext&) =
delete;
1064 template <
typename DoneCallback>
1065 struct EvalShardedByInnerDimContext {
1066 EvalShardedByInnerDimContext(
const Self* self,
int num_threads, Scalar* result_buffer, Index m_size, Index n_size,
1067 Index k_size, DoneCallback done_callback)
1069 m_lhs_inner_dim_contiguous(evaluator->m_lhs_inner_dim_contiguous),
1070 m_rhs_inner_dim_contiguous(evaluator->m_rhs_inner_dim_contiguous),
1071 m_rhs_inner_dim_reordered(evaluator->m_rhs_inner_dim_reordered),
1072 result(result_buffer),
1076 done(std::move(done_callback)),
1077 buffer_size_bytes(m * n * sizeof(Scalar)),
1078 block_size(innerDimBlockSize(k, num_threads)),
1079 num_blocks(numext::div_ceil<Index>(k, block_size)),
1080 num_pending_blocks(internal::convert_index<int>(num_blocks)),
1081 l0_ranges(numext::div_ceil<Index>(num_blocks, l0_size)),
1082 l0_state(l0_ranges),
1083 block_buffers(num_blocks) {
1085 for (
int i = 0; i < l0_ranges; ++i) {
1086 const Index num_pending_tasks = actualRangeSize(l0_ranges, l0_size, i);
1087 l0_state.emplace_back(internal::convert_index<int>(num_pending_tasks));
1092 for (Index block_idx = 0; block_idx < num_blocks; ++block_idx) {
1093 block_buffers.emplace_back(block_idx == 0 ? result : allocateBlockBuffer());
1097 releaseBlockBuffers();
1102 ~EvalShardedByInnerDimContext() { releaseBlockBuffers(); }
1104 Scalar* allocateBlockBuffer()
const {
1105 Scalar* buf =
static_cast<Scalar*
>(evaluator->m_device.allocate(buffer_size_bytes));
1106 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
1107 EIGEN_TRY { internal::default_construct_elements_of_array(buf, m * n); }
1109 evaluator->m_device.deallocate(buf);
1116 void releaseBlockBuffers() {
1117 for (
size_t i = 1; i < block_buffers.size(); ++i) {
1118 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
1119 internal::destruct_elements_of_array(block_buffers[i], m * n);
1121 evaluator->m_device.deallocate(block_buffers[i]);
1125 template <
int Alignment>
1127 Barrier barrier(internal::convert_index<int>(num_blocks));
1128 evalBlocks<Alignment>(barrier);
1131 aggregateL0Blocks<Alignment>();
1134 applyOutputKernel();
1139 template <
int Alignment>
1140 void evalBlocks(Barrier& barrier)
noexcept {
1141 eval<Alignment>(barrier, 0, num_blocks);
1145 template <
int Alignment>
1146 void runAsync() noexcept {
1147 evalAsync<Alignment>(0, num_blocks);
1151 const Self* evaluator;
1154 bool m_lhs_inner_dim_contiguous;
1155 bool m_rhs_inner_dim_contiguous;
1156 bool m_rhs_inner_dim_reordered;
1170 Index buffer_size_bytes;
1176 std::atomic<int> num_pending_blocks;
1194 static constexpr Index l0_size = 4;
1198 MaxSizeVector<std::atomic<int>> l0_state;
1201 MaxSizeVector<Scalar*> block_buffers;
1203 template <
int Alignment>
1204 void processBlock(Index block_idx, Index begin, Index end) {
1205 Scalar* buf = block_buffers[block_idx];
1207 dispatchContraction(
1208 [&](
auto lhs_c,
auto rhs_c,
auto rhs_r) {
1209 evaluator->template evalGemmPartialWithoutOutputKernel<lhs_c(), rhs_c(), rhs_r(), Alignment>(
1210 buf, begin, end, internal::convert_index<int>(num_blocks));
1212 m_lhs_inner_dim_contiguous, m_rhs_inner_dim_contiguous, m_rhs_inner_dim_reordered);
1215 const Index l0_index = block_idx / l0_size;
1216 const int v = l0_state[l0_index].fetch_sub(1);
1217 eigen_assert(v >= 1);
1222 const Index rng_size = actualRangeSize(l0_ranges, l0_size, l0_index);
1223 const Index dst_block_idx = l0_index * l0_size;
1225 if (rng_size == l0_size) {
1226 addAllToBuffer<Alignment>(m * n,
1227 block_buffers[dst_block_idx + 1],
1228 block_buffers[dst_block_idx + 2],
1229 block_buffers[dst_block_idx + 3],
1230 block_buffers[dst_block_idx]);
1233 for (
int i = 1; i < rng_size; ++i) {
1234 addToBuffer<Alignment>(m * n,
1235 block_buffers[dst_block_idx + i],
1236 block_buffers[dst_block_idx]);
1243 template <
int Alignment>
1244 void aggregateL0Blocks()
const {
1247 for (; l0_index + 2 < l0_ranges; l0_index += 3) {
1248 addAllToBuffer<Alignment>(m * n,
1249 block_buffers[(l0_index + 0) * l0_size],
1250 block_buffers[(l0_index + 1) * l0_size],
1251 block_buffers[(l0_index + 2) * l0_size],
1255 for (; l0_index < l0_ranges; ++l0_index) {
1256 addToBuffer<Alignment>(m * n, block_buffers[l0_index * l0_size], block_buffers[0]);
1260 void applyOutputKernel()
const {
1261 using OutputMapper = internal::blas_data_mapper<Scalar, Index, ColMajor>;
1262 evaluator->m_output_kernel(OutputMapper(result, m), evaluator->m_tensor_contraction_params,
1263 static_cast<Eigen::Index
>(0),
static_cast<Eigen::Index
>(0), m, n);
1267 Index actualBlockSize(Index block_idx)
const {
1268 return block_idx + 1 < num_blocks ? block_size : k + block_size - block_size * num_blocks;
1272 Index actualRangeSize(Index num_ranges, Index range_size, Index range_idx)
const {
1273 eigen_assert(range_idx < num_ranges);
1274 return range_idx + 1 < num_ranges ? range_size : num_blocks + range_size - range_size * num_ranges;
1277 template <
int Alignment>
1278 EIGEN_STRONG_INLINE
static void addToBuffer(
size_t n,
const Scalar* src_buf, Scalar* tgt_buf) {
1279 const int output_packet_size = internal::unpacket_traits<PacketReturnType>::size;
1281 const size_t num_packets = n / output_packet_size;
1282 for (; i < output_packet_size * num_packets; i += output_packet_size) {
1283 const PacketReturnType src_val = internal::pload<PacketReturnType>(src_buf + i);
1284 const PacketReturnType tgt_val = internal::ploadt<PacketReturnType, Alignment>(tgt_buf + i);
1285 const PacketReturnType sum = internal::padd(src_val, tgt_val);
1286 internal::pstoret<Scalar, PacketReturnType, Alignment>(tgt_buf + i, sum);
1288 for (; i < n; ++i) {
1289 tgt_buf[i] += src_buf[i];
1293 template <
int Alignment>
1294 EIGEN_STRONG_INLINE
static void addAllToBuffer(
size_t n,
const Scalar* src_buf0,
const Scalar* src_buf1,
1295 const Scalar* src_buf2, Scalar* dst_buf) {
1296 using ::Eigen::internal::padd;
1297 using ::Eigen::internal::pload;
1298 using ::Eigen::internal::ploadt;
1299 using ::Eigen::internal::pstoret;
1301 const int output_packet_size = internal::unpacket_traits<PacketReturnType>::size;
1304 const size_t num_packets = n / output_packet_size;
1305 for (; i < output_packet_size * num_packets; i += output_packet_size) {
1306 const auto src_val0 = pload<PacketReturnType>(src_buf0 + i);
1307 const auto src_val1 = pload<PacketReturnType>(src_buf1 + i);
1308 const auto src_val2 = pload<PacketReturnType>(src_buf2 + i);
1310 const auto dst_val = ploadt<PacketReturnType, Alignment>(dst_buf + i);
1311 const auto sum = padd(padd(dst_val, src_val0), padd(src_val1, src_val2));
1313 pstoret<Scalar, PacketReturnType, Alignment>(dst_buf + i, sum);
1315 for (; i < n; ++i) {
1316 dst_buf[i] += src_buf0[i] + src_buf1[i] + src_buf2[i];
1320 template <
int Alignment>
1321 void eval(Barrier& barrier, Index start_block_idx, Index end_block_idx) {
1322 while (end_block_idx - start_block_idx > 1) {
1323 Index mid_block_idx = (start_block_idx + end_block_idx) / 2;
1324 evaluator->m_device.enqueue([
this, &barrier, mid_block_idx, end_block_idx]() {
1325 eval<Alignment>(barrier, mid_block_idx, end_block_idx);
1327 end_block_idx = mid_block_idx;
1330 Index block_idx = start_block_idx;
1331 Index block_start = block_idx * block_size;
1332 Index block_end = block_start + actualBlockSize(block_idx);
1334 processBlock<Alignment>(block_idx, block_start, block_end);
1338 template <
int Alignment>
1339 void evalAsync(Index start_block_idx, Index end_block_idx) {
1340 while (end_block_idx - start_block_idx > 1) {
1341 Index mid_block_idx = (start_block_idx + end_block_idx) / 2;
1342 evaluator->m_device.enqueue(
1343 [
this, mid_block_idx, end_block_idx]() { evalAsync<Alignment>(mid_block_idx, end_block_idx); });
1344 end_block_idx = mid_block_idx;
1347 Index block_idx = start_block_idx;
1349 Index block_start = block_idx * block_size;
1350 Index block_end = block_start + actualBlockSize(block_idx);
1352 processBlock<Alignment>(block_idx, block_start, block_end);
1354 int v = num_pending_blocks.fetch_sub(1);
1355 eigen_assert(v >= 1);
1359 aggregateL0Blocks<Alignment>();
1362 applyOutputKernel();
1369 DoneCallback done_copy = std::move(done);
1379 EvalShardedByInnerDimContext(
const EvalShardedByInnerDimContext&) =
delete;
1380 void operator=(
const EvalShardedByInnerDimContext&) =
delete;
1389 static bool shardByCol(Index m, Index n, Index num_threads) {
1396 if (m / num_threads >= Traits::nr &&
1398 (n / num_threads < Traits::nr ||
1401 (n / num_threads < 4 * Traits::nr && (n % (num_threads * Traits::nr)) != 0 &&
1403 ((m % (num_threads * Traits::nr)) == 0 ||
1411 if (n / num_threads < 16 * Traits::nr && m > n * 32)
return false;
1415 Index coarsenM(Index m, Index n, Index bm, Index bn, Index bk, Index gn,
int num_threads,
bool shard_by_col)
const {
1418 Index nm0 = numext::div_ceil(m, bm);
1424 while (gm1 <= nm0 && nm1 == numext::div_ceil(nm0, gm1)) gm1++;
1425 if (gm1 > nm0)
break;
1427 int res = checkGrain(m, n, bm, bn, bk, gm1, gn, gm, gn, num_threads, shard_by_col);
1429 nm1 = numext::div_ceil(nm0, gm1);
1430 if (res == 0)
continue;
1437 Index coarsenN(Index m, Index n, Index bm, Index bn, Index bk, Index gm,
int num_threads,
bool shard_by_col)
const {
1440 Index nn0 = numext::div_ceil(n, bn);
1443 while (gn1 <= nn0 && nn1 == numext::div_ceil(nn0, gn1)) gn1++;
1444 if (gn1 > nn0)
break;
1445 int res = checkGrain(m, n, bm, bn, bk, gm, gn1, gm, gn, num_threads, shard_by_col);
1447 nn1 = numext::div_ceil(nn0, gn1);
1448 if (res == 0)
continue;
1456 int checkGrain(Index m, Index n, Index bm, Index bn, Index bk, Index gm, Index gn, Index oldgm, Index oldgn,
1457 int num_threads,
bool shard_by_col)
const {
1458 const TensorOpCost cost = contractionCost(bm * gm, bn * gn, bm, bn, bk, shard_by_col,
true);
1459 double taskSize = TensorCostModel<ThreadPoolDevice>::taskSize(
static_cast<double>(bm) * gm * bn * gn, cost);
1462 if (taskSize < 1)
return 1;
1464 if (taskSize > 2)
return -1;
1471 Index nm0 = numext::div_ceil(m, bm);
1472 Index nn0 = numext::div_ceil(n, bn);
1473 Index new_tasks = numext::div_ceil(nm0, gm) * numext::div_ceil(nn0, gn);
1474 double new_parallelism =
1475 static_cast<double>(new_tasks) / (numext::div_ceil<Index>(new_tasks, num_threads) * num_threads);
1476 Index old_tasks = numext::div_ceil(nm0, oldgm) * numext::div_ceil(nn0, oldgn);
1477 double old_parallelism =
1478 static_cast<double>(old_tasks) / (numext::div_ceil<Index>(old_tasks, num_threads) * num_threads);
1479 if (new_parallelism > old_parallelism || new_parallelism == 1)
return 1;
1483 TensorOpCost contractionCost(Index m, Index n, Index bm, Index bn, Index bk,
bool shard_by_col,
1484 bool prepacked)
const {
1485 const int packed_size = std::min<int>(PacketType<LhsScalar, Device>::size, PacketType<RhsScalar, Device>::size);
1486 const int output_packet_size = internal::unpacket_traits<PacketReturnType>::size;
1487 const double kd =
static_cast<double>(bk);
1488 double compute_bandwidth = computeBandwidth(bm, bn, bk);
1490 TensorOpCost cost = TensorOpCost(0, 0, kd * compute_bandwidth,
true, packed_size);
1492 cost += TensorOpCost(0,
sizeof(CoeffReturnType), 0,
true, output_packet_size);
1500 TensorOpCost lhsCost = this->m_leftImpl.costPerCoeff(
true) * (kd / n);
1501 TensorOpCost rhsCost = this->m_rightImpl.costPerCoeff(
true) * (kd / m);
1505 lhsCost.dropMemoryCost();
1507 rhsCost.dropMemoryCost();
1508 return cost + lhsCost + rhsCost;
1513 static bool shardByInnerDim(Index m, Index n, Index k,
int num_threads,
int num_threads_by_k) {
1514 std::ptrdiff_t bufsize = m * n *
sizeof(Scalar);
1515 bool shard_by_k =
false;
1517 num_threads_by_k < 2 ||
1518 num_threads_by_k < num_threads ||
1519 bufsize > l3CacheSize() / num_threads_by_k ||
1521 k / num_threads_by_k < 2 * Traits::nr) {
1523 }
else if (numext::maxi(m, n) / num_threads < Traits::nr ||
1525 (k / num_threads_by_k > 8 * Traits::nr &&
1528 (numext::mini(m, n) < 2 * Traits::nr || num_threads_by_k > num_threads))) {
1537 static Index innerDimBlockSize(Index k,
int num_threads) {
1540 const Index packet_size = internal::packet_traits<RhsScalar>::size;
1541 const auto round_up = [=](Index index) -> Index {
1542 const Index kmultiple = packet_size <= 8 ? 8 : packet_size;
1543 return numext::div_ceil<Index>(index, kmultiple) * kmultiple;
1546 const Index target_block_size = round_up(numext::div_ceil<Index>(k, num_threads));
1547 const Index desired_min_block_size = 12 * packet_size;
1549 return numext::mini<Index>(k, numext::maxi<Index>(desired_min_block_size, target_block_size));
1554 double contractionCostPerInnerDim(Index m, Index n)
const {
1555 using CostModel = TensorCostModel<ThreadPoolDevice>;
1556 const int output_packet_size = internal::unpacket_traits<PacketReturnType>::size;
1559 const double bandwidth = (m < output_packet_size) ? shortBlockComputeBandwidth() : peakComputeBandwidth();
1560 const TensorOpCost kernel_cost(0, 0, (bandwidth * m) * n,
true, output_packet_size);
1563 TensorOpCost pack_cost = this->m_leftImpl.costPerCoeff(
true) * m + this->m_rightImpl.costPerCoeff(
true) * n;
1565 TensorOpCost(0,
static_cast<double>(m) *
sizeof(LhsScalar) +
static_cast<double>(n) *
sizeof(RhsScalar), 0);
1567 return CostModel::totalCost(1, kernel_cost) + CostModel::totalCost(1, pack_cost);
1572 int numThreadsInnerDim(Index m, Index n, Index k)
const {
1573 using CostModel = TensorCostModel<ThreadPoolDevice>;
1574 const int output_packet_size = internal::unpacket_traits<PacketReturnType>::size;
1575 const double total_parallel_cost = k * contractionCostPerInnerDim(m, n);
1578 const double reduction_cost = CostModel::totalCost(m * n, TensorOpCost(2, 1, 1,
true, output_packet_size));
1581 const double max_threads_by_work = total_parallel_cost / CostModel::kPerThreadCycles;
1582 const double max_threads_by_steps =
static_cast<double>(k / (8 * Traits::nr + 1));
1583 const int max_threads =
static_cast<int>(numext::mini<double>(
1584 this->m_device.numThreads(), numext::mini<double>(max_threads_by_work, max_threads_by_steps)));
1585 int num_threads = 1;
1586 double min_cost = total_parallel_cost;
1587 for (
int nt = 2; nt <= max_threads; nt += 2) {
1590 const int num_blocks = internal::convert_index<int>(numext::div_ceil<Index>(k, innerDimBlockSize(k, nt)));
1591 const double parallel_cost =
1592 total_parallel_cost / num_blocks + CostModel::kStartupCycles + num_blocks * reduction_cost;
1593 if (parallel_cost < min_cost) {
1594 num_threads = num_blocks;
1595 min_cost = parallel_cost;
1601 static double peakComputeBandwidth() {
1602#ifdef EIGEN_VECTORIZE_FMA
1614 static double shortBlockComputeBandwidth() {
return 2.0; }
1616 double computeBandwidth(Index bm, Index bn, Index bk)
const {
1620 if (bk == 1)
return 4.0;
1621 if (bn < Traits::nr || bm < Traits::mr)
return shortBlockComputeBandwidth();
1622 return peakComputeBandwidth();
Definition TensorContraction.h:418
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47