11#if defined(EIGEN_USE_THREADS) && !defined(EIGEN_TENSOR_TENSOR_DEVICE_THREAD_POOL_H)
12#define EIGEN_TENSOR_TENSOR_DEVICE_THREAD_POOL_H
15#include "./InternalHeaderCheck.h"
22 virtual ~Allocator() =
default;
23 virtual void* allocate(
size_t num_bytes)
const = 0;
24 virtual void deallocate(
void* buffer)
const = 0;
28struct ThreadPoolDevice {
30 ThreadPoolDevice(ThreadPoolInterface* pool,
int num_cores, Allocator* allocator =
nullptr)
31 : pool_(pool), num_threads_(num_cores), allocator_(allocator) {}
33 EIGEN_STRONG_INLINE
void* allocate(
size_t num_bytes)
const {
34 return allocator_ ? allocator_->allocate(num_bytes) : internal::aligned_malloc(num_bytes);
37 EIGEN_STRONG_INLINE
void deallocate(
void* buffer)
const {
39 allocator_->deallocate(buffer);
41 internal::aligned_free(buffer);
45 EIGEN_STRONG_INLINE
void* allocate_temp(
size_t num_bytes)
const {
return allocate(num_bytes); }
47 EIGEN_STRONG_INLINE
void deallocate_temp(
void* buffer)
const { deallocate(buffer); }
49 template <
typename Type>
50 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Type get(Type data)
const {
54 EIGEN_STRONG_INLINE
void memcpy(
void* dst,
const void* src,
size_t n)
const {
56 ::memcpy(dst, src, n);
59 const size_t kMinBlockSize = 32768;
62 const size_t num_threads = CostModel::numThreads(n, TensorOpCost(1.0, 1.0, 0),
static_cast<int>(numThreads()));
63 if (n <= kMinBlockSize || num_threads < 2) {
64 ::memcpy(dst, src, n);
66 const char* src_ptr =
static_cast<const char*
>(src);
67 char* dst_ptr =
static_cast<char*
>(dst);
68 const size_t blocksize = (n + (num_threads - 1)) / num_threads;
69 Barrier barrier(
static_cast<int>(num_threads - 1));
71 for (
size_t i = 1; i < num_threads; ++i) {
72 pool_->Schedule([n, i, src_ptr, dst_ptr, blocksize, &barrier] {
73 ::memcpy(dst_ptr + i * blocksize, src_ptr + i * blocksize, numext::mini(blocksize, n - (i * blocksize)));
78 ::memcpy(dst_ptr, src_ptr, blocksize);
83 EIGEN_STRONG_INLINE
void memcpyHostToDevice(
void* dst,
const void* src,
size_t n)
const { memcpy(dst, src, n); }
84 EIGEN_STRONG_INLINE
void memcpyDeviceToHost(
void* dst,
const void* src,
size_t n)
const { memcpy(dst, src, n); }
86 EIGEN_STRONG_INLINE
void memset(
void* buffer,
int c,
size_t n)
const { ::memset(buffer, c, n); }
89 EIGEN_STRONG_INLINE
void fill(T* begin, T* end,
const T& value)
const {
90 std::fill(begin, end, value);
93 EIGEN_STRONG_INLINE
int numThreads()
const {
return num_threads_; }
97 EIGEN_STRONG_INLINE
int numThreadsInPool()
const {
return pool_->NumThreads(); }
99 EIGEN_STRONG_INLINE
size_t firstLevelCacheSize()
const {
return l1CacheSize(); }
101 EIGEN_STRONG_INLINE
size_t lastLevelCacheSize()
const {
103 return l3CacheSize() / num_threads_;
106 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void synchronize()
const {
110 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
int majorDeviceVersion()
const {
116 template <
class Function,
class... Args>
117 EIGEN_STRONG_INLINE
void enqueueNoNotification(Function&& f, Args&&... args)
const {
118 enqueue(std::forward<Function>(f), std::forward<Args>(args)...);
121 template <
class Function,
class... Args>
122 EIGEN_STRONG_INLINE
void enqueue(Function&& f, Args&&... args)
const {
123#if EIGEN_COMP_CXXVER >= 20
126 pool_->Schedule([f = std::forward<Function>(f), ... args = std::forward<Args>(args)]()
mutable { f(args...); });
132 pool_->Schedule(std::bind(std::forward<Function>(f), std::forward<Args>(args)...));
138 template <
class Function>
139 EIGEN_STRONG_INLINE
void enqueue(Function&& f)
const {
140 pool_->Schedule(std::forward<Function>(f));
145 EIGEN_STRONG_INLINE
int currentThreadId()
const {
return pool_->CurrentThreadId(); }
154 void parallelFor(Index n,
const TensorOpCost& cost, std::function<Index(Index)> block_align,
155 std::function<
void(Index, Index)> f)
const {
156 if (EIGEN_PREDICT_FALSE(n <= 0)) {
160 if (n == 1 || numThreads() == 1 || CostModel::numThreads(n, cost,
static_cast<int>(numThreads())) == 1) {
166 ParallelForBlock block = CalculateParallelForBlock(n, cost, block_align);
171 Barrier barrier(
static_cast<unsigned int>(block.count));
172 if (block.count <= numThreads()) {
175 handleRange(0, n, block.size, &barrier, pool_, f);
179 pool_->Schedule([
this, n, &block, &barrier, &f]() { handleRange(0, n, block.size, &barrier, pool_, f); });
186 void parallelFor(Index n,
const TensorOpCost& cost, std::function<
void(Index, Index)> f)
const {
187 parallelFor(n, cost,
nullptr, std::move(f));
197 void parallelForAsync(Index n,
const TensorOpCost& cost, std::function<Index(Index)> block_align,
198 std::function<
void(Index, Index)> f, std::function<
void()> done)
const {
200 if (n <= 1 || numThreads() == 1 || CostModel::numThreads(n, cost,
static_cast<int>(numThreads())) == 1) {
207 ParallelForBlock block = CalculateParallelForBlock(n, cost, block_align);
209 ParallelForAsyncContext*
const ctx =
210 new ParallelForAsyncContext(block.count, block.size, pool_, std::move(f), std::move(done));
212 if (block.count <= numThreads()) {
215 handleRangeAsync(ctx, 0, n);
219 pool_->Schedule([ctx, n]() { handleRangeAsync(ctx, 0, n); });
224 void parallelForAsync(Index n,
const TensorOpCost& cost, std::function<
void(Index, Index)> f,
225 std::function<
void()> done)
const {
226 parallelForAsync(n, cost,
nullptr, std::move(f), std::move(done));
230 ThreadPoolInterface* getPool()
const {
return pool_; }
233 Allocator* allocator()
const {
return allocator_; }
236 using CostModel = TensorCostModel<ThreadPoolDevice>;
238 static void handleRange(Index firstIdx, Index lastIdx, Index granularity, Barrier* barrier, ThreadPoolInterface* pool,
239 const std::function<
void(Index, Index)>& f) {
240 while (lastIdx - firstIdx > granularity) {
242 const Index midIdx = firstIdx + numext::div_ceil((lastIdx - firstIdx) / 2, granularity) * granularity;
243 pool->Schedule([=, &f]() { handleRange(midIdx, lastIdx, granularity, barrier, pool, f); });
247 f(firstIdx, lastIdx);
253 struct ParallelForAsyncContext {
254 ParallelForAsyncContext(Index block_count, Index block_size, ThreadPoolInterface* p,
255 std::function<
void(Index, Index)> block_f, std::function<
void()> done_callback)
256 : count(block_count), granularity(block_size), pool(p), f(std::move(block_f)), done(std::move(done_callback)) {}
257 ~ParallelForAsyncContext() { done(); }
259 std::atomic<Index> count;
261 ThreadPoolInterface* pool;
262 std::function<void(Index, Index)> f;
263 std::function<void()> done;
268 static void handleRangeAsync(ParallelForAsyncContext* ctx, Index firstIdx, Index lastIdx) {
269 while (lastIdx - firstIdx > ctx->granularity) {
270 const Index midIdx = firstIdx + numext::div_ceil((lastIdx - firstIdx) / 2, ctx->granularity) * ctx->granularity;
271 ctx->pool->Schedule([ctx, midIdx, lastIdx]() { handleRangeAsync(ctx, midIdx, lastIdx); });
274 ctx->f(firstIdx, lastIdx);
275 if (ctx->count.fetch_sub(1) == 1)
delete ctx;
278 struct ParallelForBlock {
288 ParallelForBlock CalculateParallelForBlock(
const Index n,
const TensorOpCost& cost,
289 const std::function<Index(Index)>& block_align)
const {
290 const double block_size_f = 1.0 / CostModel::taskSize(1, cost);
291 const Index max_oversharding_factor = 4;
292 Index block_size = numext::mini(
293 n, numext::maxi<Index>(numext::div_ceil<Index>(n, max_oversharding_factor * numThreads()), block_size_f));
294 const Index max_block_size = numext::mini(n, 2 * block_size);
297 Index new_block_size = block_align(block_size);
298 eigen_assert(new_block_size >= block_size);
299 block_size = numext::mini(n, new_block_size);
302 Index block_count = numext::div_ceil(n, block_size);
306 double max_efficiency =
307 static_cast<double>(block_count) / (numext::div_ceil<Index>(block_count, numThreads()) * numThreads());
311 for (Index prev_block_count = block_count; max_efficiency < 1.0 && prev_block_count > 1;) {
314 Index coarser_block_size = numext::div_ceil(n, prev_block_count - 1);
316 Index new_block_size = block_align(coarser_block_size);
317 eigen_assert(new_block_size >= coarser_block_size);
318 coarser_block_size = numext::mini(n, new_block_size);
320 if (coarser_block_size > max_block_size) {
324 const Index coarser_block_count = numext::div_ceil(n, coarser_block_size);
325 eigen_assert(coarser_block_count < prev_block_count);
326 prev_block_count = coarser_block_count;
327 const double coarser_efficiency =
static_cast<double>(coarser_block_count) /
328 (numext::div_ceil<Index>(coarser_block_count, numThreads()) * numThreads());
329 if (coarser_efficiency + 0.01 >= max_efficiency) {
331 block_size = coarser_block_size;
332 block_count = coarser_block_count;
333 if (max_efficiency < coarser_efficiency) {
334 max_efficiency = coarser_efficiency;
339 return {block_size, block_count};
342 ThreadPoolInterface* pool_;
344 Allocator* allocator_;
Namespace containing all symbols from the Eigen library.