11#ifndef EIGEN_TENSOR_TENSOR_EXECUTOR_H
12#define EIGEN_TENSOR_TENSOR_EXECUTOR_H
15#include "./InternalHeaderCheck.h"
37template <
typename Expression,
typename Device,
bool Vectorizable, TiledEvaluation Tiling>
40 using StorageIndex =
typename Expression::Index;
47 static_assert(std::is_same<Device, DefaultDevice>::value,
48 "Default executor instantiated with non-default device. "
49 "You must #define EIGEN_USE_THREADS, EIGEN_USE_GPU or "
50 "EIGEN_USE_SYCL before including Eigen headers.");
52 static EIGEN_STRONG_INLINE
void run(
const Expression& expr,
const Device& device = DefaultDevice()) {
54 const bool needs_assign = evaluator.evalSubExprsIfNeeded(
nullptr);
59 const StorageIndex size =
static_cast<StorageIndex
>(array_prod(evaluator.dimensions()));
60 for (StorageIndex i = 0; i < size; ++i) {
61 evaluator.evalScalar(i);
77template <
typename Expression,
typename Device,
typename DoneCallback,
bool Vectorizable, TiledEvaluation Tiling>
83template <
typename Expression>
85 TiledEvaluation::Off> {
87 using StorageIndex =
typename Expression::Index;
89 static EIGEN_STRONG_INLINE
void run(
const Expression& expr,
const DefaultDevice& device = DefaultDevice()) {
91 const bool needs_assign = evaluator.evalSubExprsIfNeeded(
nullptr);
93 const StorageIndex size =
static_cast<StorageIndex
>(array_prod(evaluator.dimensions()));
94 const int PacketSize =
95 unpacket_traits<typename TensorEvaluator<Expression, DefaultDevice>::PacketReturnType>::size;
100 const StorageIndex UnrolledSize = (size / (4 * PacketSize)) * 4 * PacketSize;
101 for (StorageIndex i = 0; i < UnrolledSize; i += 4 * PacketSize) {
102 for (StorageIndex j = 0; j < 4; j++) {
103 evaluator.evalPacket(i + j * PacketSize);
106 const StorageIndex VectorizedSize = (size / PacketSize) * PacketSize;
107 for (StorageIndex i = UnrolledSize; i < VectorizedSize; i += PacketSize) {
108 evaluator.evalPacket(i);
110 for (StorageIndex i = VectorizedSize; i < size; ++i) {
111 evaluator.evalScalar(i);
122template <
typename Expression,
bool Vectorizable>
124 TiledEvaluation::On> {
126 using Scalar =
typename traits<Expression>::Scalar;
127 using ScalarNoConst = std::remove_const_t<Scalar>;
130 using StorageIndex =
typename traits<Expression>::Index;
132 static constexpr int NumDims = traits<Expression>::NumDimensions;
134 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(
const Expression& expr,
135 const DefaultDevice& device = DefaultDevice()) {
136 using TensorBlockMapper = TensorBlockMapper<NumDims, Evaluator::Layout, StorageIndex>;
138 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, StorageIndex>;
139 using TensorBlockScratch = internal::TensorBlockScratchAllocator<DefaultDevice>;
141 Evaluator evaluator(expr, device);
144 const bool needs_assign = evaluator.evalSubExprsIfNeeded(
nullptr);
148 const TensorBlockResourceRequirements requirements = evaluator.getResourceRequirements();
150 const TensorBlockMapper block_mapper(
typename TensorBlockDesc::Dimensions(evaluator.dimensions()), requirements);
153 TensorBlockScratch scratch(device);
155 const StorageIndex total_block_count = block_mapper.blockCount();
156 for (StorageIndex i = 0; i < total_block_count; ++i) {
157 TensorBlockDesc desc = block_mapper.blockDescriptor(i);
158 evaluator.evalBlock(desc, scratch);
177#ifdef EIGEN_USE_THREADS
179template <
typename TensorBlockMapper>
180struct TensorExecutorTilingContext {
181 TensorExecutorTilingContext() =
default;
182 TensorExecutorTilingContext(
const TensorBlockMapper& b_mapper,
const TensorOpCost& b_cost,
size_t b_aligned_size)
183 : block_mapper(b_mapper), cost(b_cost), aligned_blocksize(b_aligned_size) {}
185 TensorBlockMapper block_mapper;
187 size_t aligned_blocksize;
192template <
typename Evaluator,
typename TensorBlockMapper,
bool Vectorizable>
193TensorExecutorTilingContext<TensorBlockMapper> GetTensorExecutorTilingContext(
const Evaluator& evaluator) {
195 TensorBlockResourceRequirements requirements = evaluator.getResourceRequirements();
199 double task_inverse_ratio = TensorCostModel<ThreadPoolDevice>::taskSize(1, requirements.cost_per_coeff);
200 const size_t task_size = numext::maxi<size_t>(1,
static_cast<size_t>(1.0 / task_inverse_ratio));
201 requirements.size = (requirements.size > 1) ? numext::mini(requirements.size, task_size) : task_size;
203 TensorBlockMapper block_mapper(
typename TensorBlockMapper::Dimensions(evaluator.dimensions()), requirements);
205 size_t block_size = block_mapper.blockTotalSize();
206 const size_t align = numext::maxi(EIGEN_MAX_ALIGN_BYTES, 1);
207 const size_t aligned_blocksize =
208 align * numext::div_ceil<size_t>(block_size *
sizeof(
typename Evaluator::Scalar), align);
210 return {block_mapper, requirements.cost_per_coeff * block_size, aligned_blocksize};
213template <
typename Evaluator,
typename StorageIndex,
bool Vectorizable>
215 static void run(Evaluator* evaluator_in,
const StorageIndex firstIdx,
const StorageIndex lastIdx) {
216 Evaluator evaluator = *evaluator_in;
217 eigen_assert(lastIdx >= firstIdx);
218 for (StorageIndex i = firstIdx; i < lastIdx; ++i) {
219 evaluator.evalScalar(i);
223 static StorageIndex alignBlockSize(StorageIndex size) {
return size; }
226template <
typename Evaluator,
typename StorageIndex>
227struct EvalRange<Evaluator, StorageIndex, true> {
228 static constexpr int PacketSize = unpacket_traits<typename Evaluator::PacketReturnType>::size;
230 static void run(Evaluator* evaluator_in,
const StorageIndex firstIdx,
const StorageIndex lastIdx) {
231 Evaluator evaluator = *evaluator_in;
232 eigen_assert(lastIdx >= firstIdx);
233 StorageIndex i = firstIdx;
234 if (lastIdx - firstIdx >= PacketSize) {
235 eigen_assert(firstIdx % PacketSize == 0);
236 StorageIndex last_chunk_offset = lastIdx - 4 * PacketSize;
240 for (; i <= last_chunk_offset; i += 4 * PacketSize) {
241 for (StorageIndex j = 0; j < 4; j++) {
242 evaluator.evalPacket(i + j * PacketSize);
245 last_chunk_offset = lastIdx - PacketSize;
246 for (; i <= last_chunk_offset; i += PacketSize) {
247 evaluator.evalPacket(i);
250 for (; i < lastIdx; ++i) {
251 evaluator.evalScalar(i);
255 static StorageIndex alignBlockSize(StorageIndex size) {
257 if (size >= 16 * PacketSize) {
258 return (size + 4 * PacketSize - 1) & ~(4 * PacketSize - 1);
261 return (size + PacketSize - 1) & ~(PacketSize - 1);
269template <
typename Evaluator,
typename BlockMapper,
typename IndexType>
270struct EvalBlockRange {
271 static void run(Evaluator* evaluator_in,
const BlockMapper& block_mapper,
const ThreadPoolDevice& device,
272 IndexType firstBlockIdx, IndexType lastBlockIdx) {
273 using TensorBlockScratch = TensorBlockScratchAllocator<ThreadPoolDevice>;
274 Evaluator evaluator = *evaluator_in;
275 eigen_assert(lastBlockIdx >= firstBlockIdx);
276 TensorBlockScratch scratch(device);
278 for (IndexType block_idx = firstBlockIdx; block_idx < lastBlockIdx; ++block_idx) {
279 auto desc = block_mapper.blockDescriptor(block_idx);
280 evaluator.evalBlock(desc, scratch);
286template <
typename Expression,
bool Vectorizable, TiledEvaluation Tiling>
287class TensorExecutor<Expression, ThreadPoolDevice, Vectorizable, Tiling> {
289 using StorageIndex =
typename Expression::Index;
291 static EIGEN_STRONG_INLINE
void run(
const Expression& expr,
const ThreadPoolDevice& device) {
292 using Evaluator = TensorEvaluator<Expression, ThreadPoolDevice>;
293 using EvalRange = EvalRange<Evaluator, StorageIndex, Vectorizable>;
295 Evaluator evaluator(expr, device);
296 const bool needs_assign = evaluator.evalSubExprsIfNeeded(
nullptr);
298 const StorageIndex size =
static_cast<StorageIndex
>(array_prod(evaluator.dimensions()));
300 size, evaluator.costPerCoeff(Vectorizable), EvalRange::alignBlockSize,
301 [&evaluator](StorageIndex firstIdx, StorageIndex lastIdx) { EvalRange::run(&evaluator, firstIdx, lastIdx); });
307template <
typename Expression,
bool Vectorizable>
309 TiledEvaluation::On> {
311 using IndexType =
typename traits<Expression>::Index;
312 using Scalar =
typename traits<Expression>::Scalar;
313 using ScalarNoConst = std::remove_const_t<Scalar>;
315 static constexpr int NumDims = traits<Expression>::NumDimensions;
317 using Evaluator = TensorEvaluator<Expression, ThreadPoolDevice>;
318 using BlockMapper = TensorBlockMapper<NumDims, Evaluator::Layout, IndexType>;
319 using TilingContext = TensorExecutorTilingContext<BlockMapper>;
321 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, IndexType>;
322 using TensorBlockScratch = internal::TensorBlockScratchAllocator<ThreadPoolDevice>;
324 static EIGEN_STRONG_INLINE
void run(
const Expression& expr,
const ThreadPoolDevice& device) {
325 Evaluator evaluator(expr, device);
327 const bool needs_assign = evaluator.evalSubExprsIfNeeded(
nullptr);
329 const TilingContext tiling =
330 internal::GetTensorExecutorTilingContext<Evaluator, BlockMapper, Vectorizable>(evaluator);
332 auto eval_block = [&device, &evaluator, &tiling](IndexType firstBlockIdx, IndexType lastBlockIdx) {
333 EvalBlockRange<Evaluator, BlockMapper, IndexType>::run(&evaluator, tiling.block_mapper, device, firstBlockIdx,
338 if (tiling.block_mapper.blockCount() == 1) {
339 TensorBlockScratch scratch(device);
340 TensorBlockDesc desc(0, tiling.block_mapper.blockDimensions());
341 evaluator.evalBlock(desc, scratch);
343 device.parallelFor(tiling.block_mapper.blockCount(), tiling.cost, std::move(eval_block));
350template <
typename Expression,
typename DoneCallback,
bool Vectorizable, TiledEvaluation Tiling>
351class TensorAsyncExecutor<Expression, ThreadPoolDevice, DoneCallback, Vectorizable, Tiling> {
353 using StorageIndex =
typename Expression::Index;
354 using Evaluator = TensorEvaluator<Expression, ThreadPoolDevice>;
356 static EIGEN_STRONG_INLINE
void runAsync(
const Expression& expr,
const ThreadPoolDevice& device, DoneCallback done) {
357 TensorAsyncExecutorContext*
const ctx =
new TensorAsyncExecutorContext(expr, device, std::move(done));
359 const auto on_eval_subexprs = [ctx, &device](
bool need_assign) ->
void {
365 using EvalRange = EvalRange<Evaluator, StorageIndex, Vectorizable>;
366 const StorageIndex size =
static_cast<StorageIndex
>(array_prod(ctx->evaluator.dimensions()));
367 device.parallelForAsync(
368 size, ctx->evaluator.costPerCoeff(Vectorizable), EvalRange::alignBlockSize,
369 [ctx](StorageIndex firstIdx, StorageIndex lastIdx) { EvalRange::run(&ctx->evaluator, firstIdx, lastIdx); },
370 [ctx]() { delete ctx; });
373 ctx->evaluator.evalSubExprsIfNeededAsync(
nullptr, on_eval_subexprs);
377 struct TensorAsyncExecutorContext {
378 TensorAsyncExecutorContext(
const Expression& expr,
const ThreadPoolDevice& thread_pool, DoneCallback done)
379 : evaluator(expr, thread_pool), on_done(std::move(done)) {}
381 ~TensorAsyncExecutorContext() {
389 DoneCallback on_done;
393template <
typename Expression,
typename DoneCallback,
bool Vectorizable>
394class TensorAsyncExecutor<Expression, ThreadPoolDevice, DoneCallback, Vectorizable, TiledEvaluation::On> {
396 using IndexType =
typename traits<Expression>::Index;
397 using Scalar =
typename traits<Expression>::Scalar;
398 using ScalarNoConst = std::remove_const_t<Scalar>;
400 static constexpr int NumDims = traits<Expression>::NumDimensions;
402 using Evaluator = TensorEvaluator<Expression, ThreadPoolDevice>;
403 using BlockMapper = TensorBlockMapper<NumDims, Evaluator::Layout, IndexType>;
404 using TilingContext = TensorExecutorTilingContext<BlockMapper>;
406 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, IndexType>;
407 using TensorBlockScratch = internal::TensorBlockScratchAllocator<ThreadPoolDevice>;
409 static EIGEN_STRONG_INLINE
void runAsync(
const Expression& expr,
const ThreadPoolDevice& device, DoneCallback done) {
410 TensorAsyncExecutorContext*
const ctx =
new TensorAsyncExecutorContext(expr, device, std::move(done));
412 const auto on_eval_subexprs = [ctx](
bool need_assign) ->
void {
418 ctx->tiling = internal::GetTensorExecutorTilingContext<Evaluator, BlockMapper, Vectorizable>(ctx->evaluator);
420 auto eval_block = [ctx](IndexType firstBlockIdx, IndexType lastBlockIdx) {
421 EvalBlockRange<Evaluator, BlockMapper, IndexType>::run(&ctx->evaluator, ctx->tiling.block_mapper, ctx->device,
422 firstBlockIdx, lastBlockIdx);
426 if (ctx->tiling.block_mapper.blockCount() == 1) {
429 TensorBlockScratch scratch(ctx->device);
430 TensorBlockDesc desc(0, ctx->tiling.block_mapper.blockDimensions());
431 ctx->evaluator.evalBlock(desc, scratch);
435 ctx->device.parallelForAsync(ctx->tiling.block_mapper.blockCount(), ctx->tiling.cost, eval_block,
436 [ctx]() { delete ctx; });
440 ctx->evaluator.evalSubExprsIfNeededAsync(
nullptr, on_eval_subexprs);
444 struct TensorAsyncExecutorContext {
445 TensorAsyncExecutorContext(
const Expression& expr,
const ThreadPoolDevice& thread_pool, DoneCallback done)
446 : device(thread_pool), evaluator(expr, thread_pool), on_done(std::move(done)) {}
448 ~TensorAsyncExecutorContext() {
453 const ThreadPoolDevice& device;
455 TilingContext tiling;
458 DoneCallback on_done;
465#if defined(EIGEN_USE_GPU)
467template <
typename Expression,
bool Vectorizable, TiledEvaluation Tiling>
468class TensorExecutor<Expression, GpuDevice, Vectorizable, Tiling> {
470 using StorageIndex =
typename Expression::Index;
471 static void run(
const Expression& expr,
const GpuDevice& device);
474#if defined(EIGEN_GPUCC)
476template <
typename Index>
477EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
int sum_will_overflow(Index lhs, Index rhs) {
478 const Index highest = NumTraits<Index>::highest();
479 const Index lowest = NumTraits<Index>::lowest();
480 if (lhs > 0 && rhs > 0) {
481 return lhs > highest - rhs ? 1 : 0;
482 }
else if (lhs < 0 && rhs < 0) {
483 return lhs < lowest - rhs ? -1 : 0;
491template <
typename Index>
492EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index saturate_add(Index lhs, Index rhs) {
493 const Index highest = NumTraits<Index>::highest();
494 const Index lowest = NumTraits<Index>::lowest();
495 int overflow = sum_will_overflow(lhs, rhs);
496 return overflow == 1 ? highest : overflow == -1 ? lowest : lhs + rhs;
502template <
typename Index>
507 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SafeStep(Index lastIdx, Index step_size)
508 : can_overflow_(sum_will_overflow(lastIdx, step_size)), step_size_(step_size) {}
511 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index operator()(Index index)
const {
512 return can_overflow_ ? saturate_add(index, step_size_) : index + step_size_;
516 const bool can_overflow_;
517 const Index step_size_;
520template <
typename Evaluator,
typename StorageIndex,
bool Vectorizable>
521struct EigenMetaKernelEval {
522 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void run(Evaluator& eval, StorageIndex firstIdx, StorageIndex lastIdx,
523 StorageIndex step_size) {
524 SafeStep<StorageIndex> safe_step(lastIdx, step_size);
525 for (StorageIndex i = firstIdx; i < lastIdx; i = safe_step(i)) {
531template <
typename Evaluator,
typename StorageIndex>
532struct EigenMetaKernelEval<Evaluator, StorageIndex, true> {
533 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void run(Evaluator& eval, StorageIndex firstIdx, StorageIndex lastIdx,
534 StorageIndex step_size) {
535 const StorageIndex PacketSize = unpacket_traits<typename Evaluator::PacketReturnType>::size;
536 const StorageIndex vectorized_size = (lastIdx / PacketSize) * PacketSize;
537 const StorageIndex vectorized_step_size = step_size * PacketSize;
539 SafeStep<StorageIndex> safe_vectorized_step(vectorized_size, vectorized_step_size);
541 for (StorageIndex i = firstIdx * PacketSize; i < vectorized_size; i = safe_vectorized_step(i)) {
544 SafeStep<StorageIndex> safe_step(lastIdx, step_size);
545 for (StorageIndex i = saturate_add(vectorized_size, firstIdx); i < lastIdx; i = safe_step(i)) {
551template <
typename Evaluator,
typename StorageIndex>
552__global__
void __launch_bounds__(1024) EigenMetaKernel(Evaluator eval, StorageIndex size) {
553 const StorageIndex first_index = blockIdx.x * blockDim.x + threadIdx.x;
554 const StorageIndex step_size = blockDim.x * gridDim.x;
556 const bool vectorizable = Evaluator::PacketAccess & Evaluator::IsAligned;
557 EigenMetaKernelEval<Evaluator, StorageIndex, vectorizable>::run(eval, first_index, size, step_size);
561template <
typename Expression,
bool Vectorizable, TiledEvaluation Tiling>
562EIGEN_STRONG_INLINE
void TensorExecutor<Expression, GpuDevice, Vectorizable, Tiling>::run(
const Expression& expr,
563 const GpuDevice& device) {
564 TensorEvaluator<Expression, GpuDevice> evaluator(expr, device);
565 const bool needs_assign = evaluator.evalSubExprsIfNeeded(
nullptr);
567 const int block_size = device.maxGpuThreadsPerBlock();
568 const int max_blocks =
static_cast<int>(
569 numext::mini<int64_t>(device.getNumGpuMultiProcessors() * device.maxGpuThreadsPerMultiProcessor(),
570 NumTraits<StorageIndex>::highest()) /
572 const StorageIndex size =
static_cast<StorageIndex
>(array_prod(evaluator.dimensions()));
574 const int num_blocks = numext::maxi<int>(
575 numext::mini<int>(max_blocks,
static_cast<int>(numext::div_ceil<StorageIndex>(size, block_size))), 1);
577 LAUNCH_GPU_KERNEL((EigenMetaKernel<TensorEvaluator<Expression, GpuDevice>, StorageIndex>), num_blocks, block_size,
578 0, device, evaluator, size);
589template <
typename Evaluator>
590struct ExecExprFunctorKernel {
591 using Index =
typename Evaluator::Index;
594 template <
typename Scratch>
595 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE ExecExprFunctorKernel(
const Scratch, Evaluator evaluator_,
const Index range_)
596 : evaluator(evaluator_), range(range_) {}
598 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void operator()(cl::sycl::nd_item<1> itemID)
const { compute(itemID); }
599 template <
bool is_vec = Evaluator::PacketAccess>
600 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t<!is_vec> compute(
const cl::sycl::nd_item<1>& itemID)
const {
601 Index gId =
static_cast<Index
>(itemID.get_global_linear_id());
602 Index total_threads = itemID.get_global_range(0);
604 for (Index i = gId; i < range; i += total_threads) {
605 evaluator.evalScalar(i);
608 template <
bool is_vec = Evaluator::PacketAccess>
609 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t<is_vec> compute(
const cl::sycl::nd_item<1>& itemID)
const {
610 const Index vectorizedRange = (range / Evaluator::PacketSize) * Evaluator::PacketSize;
611 Index gId =
static_cast<Index
>(itemID.get_global_linear_id());
612 const Index step = Evaluator::PacketSize * itemID.get_global_range(0);
613 const Index start = Evaluator::PacketSize * gId;
614 for (Index i = start; i < vectorizedRange; i += step) {
615 evaluator.evalPacket(i);
617 gId += vectorizedRange;
618 for (Index i = gId; i < range; i += itemID.get_global_range(0)) {
619 evaluator.evalScalar(i);
624template <
typename Expression,
bool Vectorizable, TiledEvaluation Tiling>
625class TensorExecutor<Expression, Eigen::SyclDevice, Vectorizable, Tiling> {
627 using Index =
typename Expression::Index;
628 static EIGEN_STRONG_INLINE
void run(
const Expression& expr,
const Eigen::SyclDevice& dev) {
629 using Evaluator = Eigen::TensorEvaluator<Expression, Eigen::SyclDevice>;
630 Evaluator evaluator(expr, dev);
631 const bool needs_assign = evaluator.evalSubExprsIfNeeded(
nullptr);
633 Index range, GRange, tileSize;
634 Index total_size = ::Eigen::internal::array_prod(evaluator.dimensions());
635 total_size = (total_size == 0) ? 1 : total_size;
636 const int PacketSize = Eigen::PacketType<typename Evaluator::CoeffReturnType, Eigen::SyclDevice>::size;
637 Index vectorizable_threads =
static_cast<Index
>(total_size / PacketSize);
638 dev.parallel_for_setup(vectorizable_threads, tileSize, range, GRange);
641 dev.template nullary_kernel_launcher<typename Evaluator::CoeffReturnType, ExecExprFunctorKernel<Evaluator> >(
642 evaluator, cl::sycl::nd_range<1>(cl::sycl::range<1>(GRange), cl::sycl::range<1>(tileSize)), Index(1),
Definition TensorExecutor.h:78
The tensor executor class.
Definition TensorExecutor.h:38
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47