Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TensorExecutor.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_TENSOR_TENSOR_EXECUTOR_H
12#define EIGEN_TENSOR_TENSOR_EXECUTOR_H
13
14// IWYU pragma: private
15#include "./InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
37template <typename Expression, typename Device, bool Vectorizable, TiledEvaluation Tiling>
39 public:
40 using StorageIndex = typename Expression::Index;
41
42 // Including `contrib/Eigen/Tensor` in different translation units
43 // with/without `EIGEN_USE_THREADS` or `EIGEN_USE_GPU` is a potential ODR
44 // violation. If this template is instantiated with a non-default device, it
45 // means that this header file was included without defining
46 // `EIGEN_USE_THREADS`, `EIGEN_USE_GPU` or `EIGEN_USE_SYCL`.
47 static_assert(std::is_same<Device, DefaultDevice>::value,
48 "Default executor instantiated with non-default device. "
49 "You must #define EIGEN_USE_THREADS, EIGEN_USE_GPU or "
50 "EIGEN_USE_SYCL before including Eigen headers.");
51
52 static EIGEN_STRONG_INLINE void run(const Expression& expr, const Device& device = DefaultDevice()) {
53 TensorEvaluator<Expression, Device> evaluator(expr, device);
54 const bool needs_assign = evaluator.evalSubExprsIfNeeded(nullptr);
55 if (needs_assign) {
56 // Scalars that require initialization take this path, and their operations may throw. Release the temporaries
57 // that evalSubExprsIfNeeded allocated before the exception propagates.
58 EIGEN_TRY {
59 const StorageIndex size = static_cast<StorageIndex>(array_prod(evaluator.dimensions()));
60 for (StorageIndex i = 0; i < size; ++i) {
61 evaluator.evalScalar(i);
62 }
63 }
64 EIGEN_CATCH(...) {
65 evaluator.cleanup();
66 EIGEN_THROW;
67 }
68 }
69 evaluator.cleanup();
70 }
71};
72
77template <typename Expression, typename Device, typename DoneCallback, bool Vectorizable, TiledEvaluation Tiling>
79
83template <typename Expression>
84class TensorExecutor<Expression, DefaultDevice, /*Vectorizable=*/true,
85 /*Tiling=*/TiledEvaluation::Off> {
86 public:
87 using StorageIndex = typename Expression::Index;
88
89 static EIGEN_STRONG_INLINE void run(const Expression& expr, const DefaultDevice& device = DefaultDevice()) {
90 TensorEvaluator<Expression, DefaultDevice> evaluator(expr, device);
91 const bool needs_assign = evaluator.evalSubExprsIfNeeded(nullptr);
92 if (needs_assign) {
93 const StorageIndex size = static_cast<StorageIndex>(array_prod(evaluator.dimensions()));
94 const int PacketSize =
95 unpacket_traits<typename TensorEvaluator<Expression, DefaultDevice>::PacketReturnType>::size;
96
97 // Give compiler a strong possibility to unroll the loop. But don't insist
98 // on unrolling, because if the function is expensive compiler should not
99 // unroll the loop at the expense of inlining.
100 const StorageIndex UnrolledSize = (size / (4 * PacketSize)) * 4 * PacketSize;
101 for (StorageIndex i = 0; i < UnrolledSize; i += 4 * PacketSize) {
102 for (StorageIndex j = 0; j < 4; j++) {
103 evaluator.evalPacket(i + j * PacketSize);
104 }
105 }
106 const StorageIndex VectorizedSize = (size / PacketSize) * PacketSize;
107 for (StorageIndex i = UnrolledSize; i < VectorizedSize; i += PacketSize) {
108 evaluator.evalPacket(i);
109 }
110 for (StorageIndex i = VectorizedSize; i < size; ++i) {
111 evaluator.evalScalar(i);
112 }
113 }
114 evaluator.cleanup();
115 }
116};
117
122template <typename Expression, bool Vectorizable>
123class TensorExecutor<Expression, DefaultDevice, Vectorizable,
124 /*Tiling=*/TiledEvaluation::On> {
125 public:
126 using Scalar = typename traits<Expression>::Scalar;
127 using ScalarNoConst = std::remove_const_t<Scalar>;
128
130 using StorageIndex = typename traits<Expression>::Index;
131
132 static constexpr int NumDims = traits<Expression>::NumDimensions;
133
134 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(const Expression& expr,
135 const DefaultDevice& device = DefaultDevice()) {
136 using TensorBlockMapper = TensorBlockMapper<NumDims, Evaluator::Layout, StorageIndex>;
137
138 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, StorageIndex>;
139 using TensorBlockScratch = internal::TensorBlockScratchAllocator<DefaultDevice>;
140
141 Evaluator evaluator(expr, device);
142
143 // TODO(ezhulenev): Do not use tiling for small tensors?
144 const bool needs_assign = evaluator.evalSubExprsIfNeeded(nullptr);
145
146 if (needs_assign) {
147 // Query expression tree for desired block size/shape.
148 const TensorBlockResourceRequirements requirements = evaluator.getResourceRequirements();
149
150 const TensorBlockMapper block_mapper(typename TensorBlockDesc::Dimensions(evaluator.dimensions()), requirements);
151
152 // Share scratch memory allocator between all blocks.
153 TensorBlockScratch scratch(device);
154
155 const StorageIndex total_block_count = block_mapper.blockCount();
156 for (StorageIndex i = 0; i < total_block_count; ++i) {
157 TensorBlockDesc desc = block_mapper.blockDescriptor(i);
158 evaluator.evalBlock(desc, scratch);
159 scratch.reset();
160 }
161 }
162 evaluator.cleanup();
163 }
164};
165
177#ifdef EIGEN_USE_THREADS
178
179template <typename TensorBlockMapper>
180struct TensorExecutorTilingContext {
181 TensorExecutorTilingContext() = default;
182 TensorExecutorTilingContext(const TensorBlockMapper& b_mapper, const TensorOpCost& b_cost, size_t b_aligned_size)
183 : block_mapper(b_mapper), cost(b_cost), aligned_blocksize(b_aligned_size) {}
184
185 TensorBlockMapper block_mapper; // navigate through blocks
186 TensorOpCost cost; // cost of computing a single block
187 size_t aligned_blocksize; // block size after memory alignment
188};
189
190// Computes block evaluation parameters, and allocates temporary memory buffer
191// for blocks. See TensorExecutor/TensorAsyncExecutor (Tiling=On) below.
192template <typename Evaluator, typename TensorBlockMapper, bool Vectorizable>
193TensorExecutorTilingContext<TensorBlockMapper> GetTensorExecutorTilingContext(const Evaluator& evaluator) {
194 // Query expression tree for desired block size/shape.
195 TensorBlockResourceRequirements requirements = evaluator.getResourceRequirements();
196
197 // Update target block size based on cost model, preserving tighter
198 // cache-targeted block size bounds requested by subexpressions.
199 double task_inverse_ratio = TensorCostModel<ThreadPoolDevice>::taskSize(1, requirements.cost_per_coeff);
200 const size_t task_size = numext::maxi<size_t>(1, static_cast<size_t>(1.0 / task_inverse_ratio));
201 requirements.size = (requirements.size > 1) ? numext::mini(requirements.size, task_size) : task_size;
202
203 TensorBlockMapper block_mapper(typename TensorBlockMapper::Dimensions(evaluator.dimensions()), requirements);
204
205 size_t block_size = block_mapper.blockTotalSize();
206 const size_t align = numext::maxi(EIGEN_MAX_ALIGN_BYTES, 1);
207 const size_t aligned_blocksize =
208 align * numext::div_ceil<size_t>(block_size * sizeof(typename Evaluator::Scalar), align);
209
210 return {block_mapper, requirements.cost_per_coeff * block_size, aligned_blocksize};
211}
212
213template <typename Evaluator, typename StorageIndex, bool Vectorizable>
214struct EvalRange {
215 static void run(Evaluator* evaluator_in, const StorageIndex firstIdx, const StorageIndex lastIdx) {
216 Evaluator evaluator = *evaluator_in;
217 eigen_assert(lastIdx >= firstIdx);
218 for (StorageIndex i = firstIdx; i < lastIdx; ++i) {
219 evaluator.evalScalar(i);
220 }
221 }
222
223 static StorageIndex alignBlockSize(StorageIndex size) { return size; }
224};
225
226template <typename Evaluator, typename StorageIndex>
227struct EvalRange<Evaluator, StorageIndex, /*Vectorizable*/ true> {
228 static constexpr int PacketSize = unpacket_traits<typename Evaluator::PacketReturnType>::size;
229
230 static void run(Evaluator* evaluator_in, const StorageIndex firstIdx, const StorageIndex lastIdx) {
231 Evaluator evaluator = *evaluator_in;
232 eigen_assert(lastIdx >= firstIdx);
233 StorageIndex i = firstIdx;
234 if (lastIdx - firstIdx >= PacketSize) {
235 eigen_assert(firstIdx % PacketSize == 0);
236 StorageIndex last_chunk_offset = lastIdx - 4 * PacketSize;
237 // Give compiler a strong possibility to unroll the loop. But don't insist
238 // on unrolling, because if the function is expensive compiler should not
239 // unroll the loop at the expense of inlining.
240 for (; i <= last_chunk_offset; i += 4 * PacketSize) {
241 for (StorageIndex j = 0; j < 4; j++) {
242 evaluator.evalPacket(i + j * PacketSize);
243 }
244 }
245 last_chunk_offset = lastIdx - PacketSize;
246 for (; i <= last_chunk_offset; i += PacketSize) {
247 evaluator.evalPacket(i);
248 }
249 }
250 for (; i < lastIdx; ++i) {
251 evaluator.evalScalar(i);
252 }
253 }
254
255 static StorageIndex alignBlockSize(StorageIndex size) {
256 // Align block size to packet size and account for unrolling in run above.
257 if (size >= 16 * PacketSize) {
258 return (size + 4 * PacketSize - 1) & ~(4 * PacketSize - 1);
259 }
260 // Aligning to 4 * PacketSize would increase block size by more than 25%.
261 return (size + PacketSize - 1) & ~(PacketSize - 1);
262 }
263};
264
265// Evaluates a range of blocks for the tiled executors below. Each task copies
266// the evaluator so that concurrent tasks never share per-instance state (e.g.
267// stateful functors reached through coeff()), matching EvalRange for the
268// non-tiled path.
269template <typename Evaluator, typename BlockMapper, typename IndexType>
270struct EvalBlockRange {
271 static void run(Evaluator* evaluator_in, const BlockMapper& block_mapper, const ThreadPoolDevice& device,
272 IndexType firstBlockIdx, IndexType lastBlockIdx) {
273 using TensorBlockScratch = TensorBlockScratchAllocator<ThreadPoolDevice>;
274 Evaluator evaluator = *evaluator_in;
275 eigen_assert(lastBlockIdx >= firstBlockIdx);
276 TensorBlockScratch scratch(device);
277
278 for (IndexType block_idx = firstBlockIdx; block_idx < lastBlockIdx; ++block_idx) {
279 auto desc = block_mapper.blockDescriptor(block_idx);
280 evaluator.evalBlock(desc, scratch);
281 scratch.reset();
282 }
283 }
284};
285
286template <typename Expression, bool Vectorizable, TiledEvaluation Tiling>
287class TensorExecutor<Expression, ThreadPoolDevice, Vectorizable, Tiling> {
288 public:
289 using StorageIndex = typename Expression::Index;
290
291 static EIGEN_STRONG_INLINE void run(const Expression& expr, const ThreadPoolDevice& device) {
292 using Evaluator = TensorEvaluator<Expression, ThreadPoolDevice>;
293 using EvalRange = EvalRange<Evaluator, StorageIndex, Vectorizable>;
294
295 Evaluator evaluator(expr, device);
296 const bool needs_assign = evaluator.evalSubExprsIfNeeded(nullptr);
297 if (needs_assign) {
298 const StorageIndex size = static_cast<StorageIndex>(array_prod(evaluator.dimensions()));
299 device.parallelFor(
300 size, evaluator.costPerCoeff(Vectorizable), EvalRange::alignBlockSize,
301 [&evaluator](StorageIndex firstIdx, StorageIndex lastIdx) { EvalRange::run(&evaluator, firstIdx, lastIdx); });
302 }
303 evaluator.cleanup();
304 }
305};
306
307template <typename Expression, bool Vectorizable>
308class TensorExecutor<Expression, ThreadPoolDevice, Vectorizable,
309 /*Tiling=*/TiledEvaluation::On> {
310 public:
311 using IndexType = typename traits<Expression>::Index;
312 using Scalar = typename traits<Expression>::Scalar;
313 using ScalarNoConst = std::remove_const_t<Scalar>;
314
315 static constexpr int NumDims = traits<Expression>::NumDimensions;
316
317 using Evaluator = TensorEvaluator<Expression, ThreadPoolDevice>;
318 using BlockMapper = TensorBlockMapper<NumDims, Evaluator::Layout, IndexType>;
319 using TilingContext = TensorExecutorTilingContext<BlockMapper>;
320
321 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, IndexType>;
322 using TensorBlockScratch = internal::TensorBlockScratchAllocator<ThreadPoolDevice>;
323
324 static EIGEN_STRONG_INLINE void run(const Expression& expr, const ThreadPoolDevice& device) {
325 Evaluator evaluator(expr, device);
326
327 const bool needs_assign = evaluator.evalSubExprsIfNeeded(nullptr);
328 if (needs_assign) {
329 const TilingContext tiling =
330 internal::GetTensorExecutorTilingContext<Evaluator, BlockMapper, Vectorizable>(evaluator);
331
332 auto eval_block = [&device, &evaluator, &tiling](IndexType firstBlockIdx, IndexType lastBlockIdx) {
333 EvalBlockRange<Evaluator, BlockMapper, IndexType>::run(&evaluator, tiling.block_mapper, device, firstBlockIdx,
334 lastBlockIdx);
335 };
336
337 // Evaluate small expressions directly as a single block.
338 if (tiling.block_mapper.blockCount() == 1) {
339 TensorBlockScratch scratch(device);
340 TensorBlockDesc desc(0, tiling.block_mapper.blockDimensions());
341 evaluator.evalBlock(desc, scratch);
342 } else {
343 device.parallelFor(tiling.block_mapper.blockCount(), tiling.cost, std::move(eval_block));
344 }
345 }
346 evaluator.cleanup();
347 }
348};
349
350template <typename Expression, typename DoneCallback, bool Vectorizable, TiledEvaluation Tiling>
351class TensorAsyncExecutor<Expression, ThreadPoolDevice, DoneCallback, Vectorizable, Tiling> {
352 public:
353 using StorageIndex = typename Expression::Index;
354 using Evaluator = TensorEvaluator<Expression, ThreadPoolDevice>;
355
356 static EIGEN_STRONG_INLINE void runAsync(const Expression& expr, const ThreadPoolDevice& device, DoneCallback done) {
357 TensorAsyncExecutorContext* const ctx = new TensorAsyncExecutorContext(expr, device, std::move(done));
358
359 const auto on_eval_subexprs = [ctx, &device](bool need_assign) -> void {
360 if (!need_assign) {
361 delete ctx;
362 return;
363 }
364
365 using EvalRange = EvalRange<Evaluator, StorageIndex, Vectorizable>;
366 const StorageIndex size = static_cast<StorageIndex>(array_prod(ctx->evaluator.dimensions()));
367 device.parallelForAsync(
368 size, ctx->evaluator.costPerCoeff(Vectorizable), EvalRange::alignBlockSize,
369 [ctx](StorageIndex firstIdx, StorageIndex lastIdx) { EvalRange::run(&ctx->evaluator, firstIdx, lastIdx); },
370 [ctx]() { delete ctx; });
371 };
372
373 ctx->evaluator.evalSubExprsIfNeededAsync(nullptr, on_eval_subexprs);
374 }
375
376 private:
377 struct TensorAsyncExecutorContext {
378 TensorAsyncExecutorContext(const Expression& expr, const ThreadPoolDevice& thread_pool, DoneCallback done)
379 : evaluator(expr, thread_pool), on_done(std::move(done)) {}
380
381 ~TensorAsyncExecutorContext() {
382 evaluator.cleanup();
383 on_done();
384 }
385
386 Evaluator evaluator;
387
388 private:
389 DoneCallback on_done;
390 };
391};
392
393template <typename Expression, typename DoneCallback, bool Vectorizable>
394class TensorAsyncExecutor<Expression, ThreadPoolDevice, DoneCallback, Vectorizable, /*Tileable*/ TiledEvaluation::On> {
395 public:
396 using IndexType = typename traits<Expression>::Index;
397 using Scalar = typename traits<Expression>::Scalar;
398 using ScalarNoConst = std::remove_const_t<Scalar>;
399
400 static constexpr int NumDims = traits<Expression>::NumDimensions;
401
402 using Evaluator = TensorEvaluator<Expression, ThreadPoolDevice>;
403 using BlockMapper = TensorBlockMapper<NumDims, Evaluator::Layout, IndexType>;
404 using TilingContext = TensorExecutorTilingContext<BlockMapper>;
405
406 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, IndexType>;
407 using TensorBlockScratch = internal::TensorBlockScratchAllocator<ThreadPoolDevice>;
408
409 static EIGEN_STRONG_INLINE void runAsync(const Expression& expr, const ThreadPoolDevice& device, DoneCallback done) {
410 TensorAsyncExecutorContext* const ctx = new TensorAsyncExecutorContext(expr, device, std::move(done));
411
412 const auto on_eval_subexprs = [ctx](bool need_assign) -> void {
413 if (!need_assign) {
414 delete ctx;
415 return;
416 }
417
418 ctx->tiling = internal::GetTensorExecutorTilingContext<Evaluator, BlockMapper, Vectorizable>(ctx->evaluator);
419
420 auto eval_block = [ctx](IndexType firstBlockIdx, IndexType lastBlockIdx) {
421 EvalBlockRange<Evaluator, BlockMapper, IndexType>::run(&ctx->evaluator, ctx->tiling.block_mapper, ctx->device,
422 firstBlockIdx, lastBlockIdx);
423 };
424
425 // Evaluate small expressions directly as a single block.
426 if (ctx->tiling.block_mapper.blockCount() == 1) {
427 {
428 // Release the scratch buffers before `delete ctx` signals completion and the device may go away.
429 TensorBlockScratch scratch(ctx->device);
430 TensorBlockDesc desc(0, ctx->tiling.block_mapper.blockDimensions());
431 ctx->evaluator.evalBlock(desc, scratch);
432 }
433 delete ctx;
434 } else {
435 ctx->device.parallelForAsync(ctx->tiling.block_mapper.blockCount(), ctx->tiling.cost, eval_block,
436 [ctx]() { delete ctx; });
437 }
438 };
439
440 ctx->evaluator.evalSubExprsIfNeededAsync(nullptr, on_eval_subexprs);
441 }
442
443 private:
444 struct TensorAsyncExecutorContext {
445 TensorAsyncExecutorContext(const Expression& expr, const ThreadPoolDevice& thread_pool, DoneCallback done)
446 : device(thread_pool), evaluator(expr, thread_pool), on_done(std::move(done)) {}
447
448 ~TensorAsyncExecutorContext() {
449 evaluator.cleanup();
450 on_done();
451 }
452
453 const ThreadPoolDevice& device;
454 Evaluator evaluator;
455 TilingContext tiling;
456
457 private:
458 DoneCallback on_done;
459 };
460};
461
462#endif // EIGEN_USE_THREADS
463
464// GPU: the evaluation of the expression is offloaded to a GPU.
465#if defined(EIGEN_USE_GPU)
466
467template <typename Expression, bool Vectorizable, TiledEvaluation Tiling>
468class TensorExecutor<Expression, GpuDevice, Vectorizable, Tiling> {
469 public:
470 using StorageIndex = typename Expression::Index;
471 static void run(const Expression& expr, const GpuDevice& device);
472};
473
474#if defined(EIGEN_GPUCC)
475// Returns 1 if lhs + rhs would overflow, -1 if it would underflow, otherwise 0.
476template <typename Index>
477EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int sum_will_overflow(Index lhs, Index rhs) {
478 const Index highest = NumTraits<Index>::highest();
479 const Index lowest = NumTraits<Index>::lowest();
480 if (lhs > 0 && rhs > 0) {
481 return lhs > highest - rhs ? 1 : 0;
482 } else if (lhs < 0 && rhs < 0) {
483 return lhs < lowest - rhs ? -1 : 0;
484 } else {
485 return 0;
486 }
487}
488
489// Returns lhs + rhs, saturating to the highest/lowest representable value on
490// overflow/underflow respectively.
491template <typename Index>
492EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index saturate_add(Index lhs, Index rhs) {
493 const Index highest = NumTraits<Index>::highest();
494 const Index lowest = NumTraits<Index>::lowest();
495 int overflow = sum_will_overflow(lhs, rhs);
496 return overflow == 1 ? highest : overflow == -1 ? lowest : lhs + rhs;
497}
498
499// A functor that adds step_size to a given index, saturating to avoid
500// overflow/underflow. If overflow/underflow is not possible, regular addition
501// is used (for efficiency).
502template <typename Index>
503struct SafeStep {
504 // lastIdx is one past the end of the possible indexes.
505 // step_size is the value that will be added to the given index when the
506 // functor is called.
507 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SafeStep(Index lastIdx, Index step_size)
508 : can_overflow_(sum_will_overflow(lastIdx, step_size)), step_size_(step_size) {}
509
510 // Adds step_size to index, saturating on overflow (if overflow is possible).
511 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index operator()(Index index) const {
512 return can_overflow_ ? saturate_add(index, step_size_) : index + step_size_;
513 }
514
515 private:
516 const bool can_overflow_;
517 const Index step_size_;
518};
519
520template <typename Evaluator, typename StorageIndex, bool Vectorizable>
521struct EigenMetaKernelEval {
522 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void run(Evaluator& eval, StorageIndex firstIdx, StorageIndex lastIdx,
523 StorageIndex step_size) {
524 SafeStep<StorageIndex> safe_step(lastIdx, step_size);
525 for (StorageIndex i = firstIdx; i < lastIdx; i = safe_step(i)) {
526 eval.evalScalar(i);
527 }
528 }
529};
530
531template <typename Evaluator, typename StorageIndex>
532struct EigenMetaKernelEval<Evaluator, StorageIndex, true> {
533 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void run(Evaluator& eval, StorageIndex firstIdx, StorageIndex lastIdx,
534 StorageIndex step_size) {
535 const StorageIndex PacketSize = unpacket_traits<typename Evaluator::PacketReturnType>::size;
536 const StorageIndex vectorized_size = (lastIdx / PacketSize) * PacketSize;
537 const StorageIndex vectorized_step_size = step_size * PacketSize;
538
539 SafeStep<StorageIndex> safe_vectorized_step(vectorized_size, vectorized_step_size);
540 // Use the vector path
541 for (StorageIndex i = firstIdx * PacketSize; i < vectorized_size; i = safe_vectorized_step(i)) {
542 eval.evalPacket(i);
543 }
544 SafeStep<StorageIndex> safe_step(lastIdx, step_size);
545 for (StorageIndex i = saturate_add(vectorized_size, firstIdx); i < lastIdx; i = safe_step(i)) {
546 eval.evalScalar(i);
547 }
548 }
549};
550
551template <typename Evaluator, typename StorageIndex>
552__global__ void __launch_bounds__(1024) EigenMetaKernel(Evaluator eval, StorageIndex size) {
553 const StorageIndex first_index = blockIdx.x * blockDim.x + threadIdx.x;
554 const StorageIndex step_size = blockDim.x * gridDim.x;
555
556 const bool vectorizable = Evaluator::PacketAccess & Evaluator::IsAligned;
557 EigenMetaKernelEval<Evaluator, StorageIndex, vectorizable>::run(eval, first_index, size, step_size);
558}
559
560/*static*/
561template <typename Expression, bool Vectorizable, TiledEvaluation Tiling>
562EIGEN_STRONG_INLINE void TensorExecutor<Expression, GpuDevice, Vectorizable, Tiling>::run(const Expression& expr,
563 const GpuDevice& device) {
564 TensorEvaluator<Expression, GpuDevice> evaluator(expr, device);
565 const bool needs_assign = evaluator.evalSubExprsIfNeeded(nullptr);
566 if (needs_assign) {
567 const int block_size = device.maxGpuThreadsPerBlock();
568 const int max_blocks = static_cast<int>(
569 numext::mini<int64_t>(device.getNumGpuMultiProcessors() * device.maxGpuThreadsPerMultiProcessor(),
570 NumTraits<StorageIndex>::highest()) /
571 block_size);
572 const StorageIndex size = static_cast<StorageIndex>(array_prod(evaluator.dimensions()));
573 // Create at least one block to ensure we don't crash with tensors of size 0.
574 const int num_blocks = numext::maxi<int>(
575 numext::mini<int>(max_blocks, static_cast<int>(numext::div_ceil<StorageIndex>(size, block_size))), 1);
576
577 LAUNCH_GPU_KERNEL((EigenMetaKernel<TensorEvaluator<Expression, GpuDevice>, StorageIndex>), num_blocks, block_size,
578 0, device, evaluator, size);
579 }
580 evaluator.cleanup();
581}
582
583#endif // EIGEN_GPUCC
584#endif // EIGEN_USE_GPU
585
586// SYCL Executor policy
587#ifdef EIGEN_USE_SYCL
588
589template <typename Evaluator>
590struct ExecExprFunctorKernel {
591 using Index = typename Evaluator::Index;
592 Evaluator evaluator;
593 const Index range;
594 template <typename Scratch>
595 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE ExecExprFunctorKernel(const Scratch, Evaluator evaluator_, const Index range_)
596 : evaluator(evaluator_), range(range_) {}
597
598 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void operator()(cl::sycl::nd_item<1> itemID) const { compute(itemID); }
599 template <bool is_vec = Evaluator::PacketAccess>
600 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t<!is_vec> compute(const cl::sycl::nd_item<1>& itemID) const {
601 Index gId = static_cast<Index>(itemID.get_global_linear_id());
602 Index total_threads = itemID.get_global_range(0);
603
604 for (Index i = gId; i < range; i += total_threads) {
605 evaluator.evalScalar(i);
606 }
607 }
608 template <bool is_vec = Evaluator::PacketAccess>
609 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t<is_vec> compute(const cl::sycl::nd_item<1>& itemID) const {
610 const Index vectorizedRange = (range / Evaluator::PacketSize) * Evaluator::PacketSize;
611 Index gId = static_cast<Index>(itemID.get_global_linear_id());
612 const Index step = Evaluator::PacketSize * itemID.get_global_range(0);
613 const Index start = Evaluator::PacketSize * gId;
614 for (Index i = start; i < vectorizedRange; i += step) {
615 evaluator.evalPacket(i);
616 }
617 gId += vectorizedRange;
618 for (Index i = gId; i < range; i += itemID.get_global_range(0)) {
619 evaluator.evalScalar(i);
620 }
621 }
622};
623
624template <typename Expression, bool Vectorizable, TiledEvaluation Tiling>
625class TensorExecutor<Expression, Eigen::SyclDevice, Vectorizable, Tiling> {
626 public:
627 using Index = typename Expression::Index;
628 static EIGEN_STRONG_INLINE void run(const Expression& expr, const Eigen::SyclDevice& dev) {
629 using Evaluator = Eigen::TensorEvaluator<Expression, Eigen::SyclDevice>;
630 Evaluator evaluator(expr, dev);
631 const bool needs_assign = evaluator.evalSubExprsIfNeeded(nullptr);
632 if (needs_assign) {
633 Index range, GRange, tileSize;
634 Index total_size = ::Eigen::internal::array_prod(evaluator.dimensions());
635 total_size = (total_size == 0) ? 1 : total_size;
636 const int PacketSize = Eigen::PacketType<typename Evaluator::CoeffReturnType, Eigen::SyclDevice>::size;
637 Index vectorizable_threads = static_cast<Index>(total_size / PacketSize);
638 dev.parallel_for_setup(vectorizable_threads, tileSize, range, GRange);
639 range = total_size;
640
641 dev.template nullary_kernel_launcher<typename Evaluator::CoeffReturnType, ExecExprFunctorKernel<Evaluator> >(
642 evaluator, cl::sycl::nd_range<1>(cl::sycl::range<1>(GRange), cl::sycl::range<1>(tileSize)), Index(1),
643 range)
644 .wait();
645 }
646 evaluator.cleanup();
647 }
648};
649
650#endif
651
652} // end namespace internal
653
654} // end namespace Eigen
655
656#endif // EIGEN_TENSOR_TENSOR_EXECUTOR_H
Definition TensorExecutor.h:78
The tensor executor class.
Definition TensorExecutor.h:38
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47