Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TensorReductionGpu.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_TENSOR_TENSOR_REDUCTION_GPU_H
12#define EIGEN_TENSOR_TENSOR_REDUCTION_GPU_H
13
14// IWYU pragma: private
15#include "./InternalHeaderCheck.h"
16
17namespace Eigen {
18namespace internal {
19
20#if defined(EIGEN_USE_GPU) && defined(EIGEN_GPUCC)
21// Full reducers for GPU, don't vectorize for now
22
23// Reducer function that enables multiple gpu threads to safely accumulate at the same
24// output address. It basically reads the current value of the output variable, and
25// attempts to update it with the new value. If in the meantime another gpu thread
26// updated the content of the output address it will try again.
27template <typename T, typename R>
28__device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer) {
29 EIGEN_IF_CONSTEXPR (sizeof(T) == 4) {
30 unsigned int oldval = *reinterpret_cast<unsigned int*>(output);
31 unsigned int newval = oldval;
32 reducer.reduce(accum, reinterpret_cast<T*>(&newval));
33 if (newval == oldval) {
34 return;
35 }
36 unsigned int readback;
37 while ((readback = atomicCAS((unsigned int*)output, oldval, newval)) != oldval) {
38 oldval = readback;
39 newval = oldval;
40 reducer.reduce(accum, reinterpret_cast<T*>(&newval));
41 if (newval == oldval) {
42 return;
43 }
44 }
45 } else EIGEN_IF_CONSTEXPR (sizeof(T) == 8) {
46 unsigned long long oldval = *reinterpret_cast<unsigned long long*>(output);
47 unsigned long long newval = oldval;
48 reducer.reduce(accum, reinterpret_cast<T*>(&newval));
49 if (newval == oldval) {
50 return;
51 }
52 unsigned long long readback;
53 while ((readback = atomicCAS(reinterpret_cast<unsigned long long*>(output), oldval, newval)) != oldval) {
54 oldval = readback;
55 newval = oldval;
56 reducer.reduce(accum, reinterpret_cast<T*>(&newval));
57 if (newval == oldval) {
58 return;
59 }
60 }
61 } else {
62 gpu_assert(0 && "Wordsize not supported");
63 }
64}
65
66// We extend atomicExch to support extra data types
67template <typename Type>
68__device__ inline Type atomicExchCustom(Type* address, Type val) {
69 return atomicExch(address, val);
70}
71
72template <typename T>
73EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR auto reduction_shuffle_mask() {
74#if defined(EIGEN_HIP_DEVICE_COMPILE)
75 return 0xFFFFFFFFFFFFFFFFull;
76#else
77 return 0xFFFFFFFFu;
78#endif
79}
80
81template <typename T>
82__device__ EIGEN_ALWAYS_INLINE T reduction_shuffle_down(T value, int offset) {
83#if defined(EIGEN_HIPCC)
84 return __shfl_down(value, offset, warpSize);
85#else
86 return __shfl_down_sync(reduction_shuffle_mask<T>(), value, offset, warpSize);
87#endif
88}
89
90template <>
91__device__ EIGEN_ALWAYS_INLINE int reduction_shuffle_down<int>(int value, int offset) {
92#if defined(EIGEN_HIPCC)
93 return __shfl_down(value, offset, warpSize);
94#else
95 return __shfl_down_sync(reduction_shuffle_mask<int>(), value, offset, warpSize);
96#endif
97}
98
99template <>
100__device__ EIGEN_ALWAYS_INLINE float reduction_shuffle_down<float>(float value, int offset) {
101#if defined(EIGEN_HIPCC)
102 return __shfl_down(value, offset, warpSize);
103#else
104 return __shfl_down_sync(reduction_shuffle_mask<float>(), value, offset, warpSize);
105#endif
106}
107
108template <>
109__device__ EIGEN_ALWAYS_INLINE double reduction_shuffle_down<double>(double value, int offset) {
110#if defined(EIGEN_HIPCC)
111 return __shfl_down(value, offset, warpSize);
112#else
113 return __shfl_down_sync(reduction_shuffle_mask<double>(), value, offset, warpSize);
114#endif
115}
116
117template <>
118__device__ inline double atomicExchCustom(double* address, double val) {
119 unsigned long long int* address_as_ull = reinterpret_cast<unsigned long long int*>(address);
120 return __longlong_as_double(atomicExch(address_as_ull, __double_as_longlong(val)));
121}
122
123// Half-float reduction specializations.
124template <typename R>
125__device__ inline void atomicReduce(half2* output, half2 accum, R& reducer) {
126 unsigned int oldval = *reinterpret_cast<unsigned int*>(output);
127 unsigned int newval = oldval;
128 reducer.reducePacket(accum, reinterpret_cast<half2*>(&newval));
129 if (newval == oldval) {
130 return;
131 }
132 unsigned int readback;
133 while ((readback = atomicCAS((unsigned int*)output, oldval, newval)) != oldval) {
134 oldval = readback;
135 newval = oldval;
136 reducer.reducePacket(accum, reinterpret_cast<half2*>(&newval));
137 if (newval == oldval) {
138 return;
139 }
140 }
141}
142#ifdef EIGEN_GPU_COMPILE_PHASE
143// reduction should be associative since reduction is not atomic in wide vector but atomic in half2 operations
144template <typename R>
145__device__ inline void atomicReduce(Packet4h2* output, Packet4h2 accum, R& reducer) {
146 half2* houtput = reinterpret_cast<half2*>(output);
147 half2* haccum = reinterpret_cast<half2*>(&accum);
148 for (int i = 0; i < 4; ++i) {
149 atomicReduce(houtput + i, *(haccum + i), reducer);
150 }
151}
152#endif // EIGEN_GPU_COMPILE_PHASE
153
154template <>
155__device__ inline void atomicReduce(float* output, float accum, SumReducer<float>&) {
156 atomicAdd(output, accum);
157}
158
159#if !defined(EIGEN_CUDA_ARCH) || EIGEN_CUDA_ARCH >= 600
160template <>
161__device__ inline void atomicReduce(double* output, double accum, SumReducer<double>&) {
162 atomicAdd(output, accum);
163}
164#endif
165
166template <typename CoeffType, typename Index>
167__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void ReductionInitKernel(const CoeffType val, Index num_preserved_coeffs,
168 CoeffType* output) {
169 const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x;
170 const Index num_threads = blockDim.x * gridDim.x;
171 for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) {
172 output[i] = val;
173 }
174}
175
176template <int BlockSize, int NumPerThread, typename Self, typename Reducer, typename Index>
177__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void FullReductionKernel(Reducer reducer, const Self input, Index num_coeffs,
178 typename Self::CoeffReturnType* output,
179 unsigned int* semaphore) {
180 // Initialize the output value
181 const Index first_index = static_cast<Index>(blockIdx.x) * BlockSize * NumPerThread + threadIdx.x;
182 if (gridDim.x == 1) {
183 if (first_index == 0) {
184 *output = reducer.initialize();
185 }
186 } else {
187 if (threadIdx.x == 0) {
188 unsigned int block = atomicCAS(semaphore, 0u, 1u);
189 if (block == 0) {
190 // We're the first block to run, initialize the output value
191 atomicExchCustom(output, reducer.initialize());
192 __threadfence();
193 atomicExch(semaphore, 2u);
194 } else {
195 // Wait for the first block to initialize the output value.
196 // Use atomicCAS here to ensure that the reads aren't cached
197 unsigned int val;
198 do {
199 val = atomicCAS(semaphore, 2u, 2u);
200 } while (val < 2u);
201 }
202 }
203 }
204
205 __syncthreads();
206
207 eigen_assert(gridDim.x == 1 || *semaphore >= 2u);
208
209 typename Self::CoeffReturnType accum = reducer.initialize();
210 Index max_iter = numext::mini<Index>(num_coeffs - first_index, NumPerThread * BlockSize);
211 for (Index i = 0; i < max_iter; i += BlockSize) {
212 const Index index = first_index + i;
213 eigen_assert(index < num_coeffs);
214 typename Self::CoeffReturnType val = input.m_impl.coeff(index);
215 reducer.reduce(val, &accum);
216 }
217
218#pragma unroll
219 for (int offset = warpSize / 2; offset > 0; offset /= 2) {
220 reducer.reduce(reduction_shuffle_down(accum, offset), &accum);
221 }
222
223 if ((threadIdx.x & (warpSize - 1)) == 0) {
224 atomicReduce(output, accum, reducer);
225 }
226
227 if (gridDim.x > 1 && threadIdx.x == 0) {
228 // Let the last block reset the semaphore
229 atomicInc(semaphore, gridDim.x + 1);
230#if defined(EIGEN_HIPCC)
231 __threadfence_system();
232#endif
233 }
234}
235
236// Half-float reduction specializations.
237template <typename Self, typename Reducer, typename Index>
238__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void ReductionInitFullReduxKernelHalfFloat(Reducer reducer, const Self input,
239 Index num_coeffs, half* scratch) {
240 eigen_assert(blockDim.x == 1);
241 eigen_assert(gridDim.x == 1);
242 using packet_type = packet_traits<Eigen::half>::type;
243 Index packet_remainder = num_coeffs % Index(unpacket_traits<packet_type>::size);
244 if (packet_remainder != 0) {
245 half2* h2scratch = reinterpret_cast<half2*>(scratch);
246 for (Index i = num_coeffs - packet_remainder; i + 2 <= num_coeffs; i += 2) {
247 *h2scratch = __halves2half2(input.coeff(i), input.coeff(i + 1));
248 h2scratch++;
249 }
250 if ((num_coeffs & 1) != 0) {
251 half lastCoeff = input.coeff(num_coeffs - 1);
252 *h2scratch = __halves2half2(lastCoeff, reducer.initialize());
253 }
254 } else {
255 packet_type reduce = reducer.template initializePacket<packet_type>();
256 internal::pstoreu(scratch, reduce);
257 }
258}
259
260template <typename Self, typename Reducer, typename Index>
261__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void ReductionInitKernelHalfFloat(Reducer reducer, const Self /*input*/,
262 Index num_coeffs, half* output) {
263 const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x;
264 const Index num_threads = blockDim.x * gridDim.x;
265 using PacketType = typename packet_traits<Eigen::half>::type;
266
267 const Index num_packets = num_coeffs / Index(unpacket_traits<PacketType>::size);
268 PacketType* p_output = reinterpret_cast<PacketType*>(output);
269 for (Index i = thread_id; i < num_packets; i += num_threads) {
270 p_output[i] = reducer.template initializePacket<PacketType>();
271 }
272 Index packet_remainder = num_coeffs % Index(unpacket_traits<PacketType>::size);
273 if (thread_id < packet_remainder) {
274 output[num_coeffs - packet_remainder + thread_id] = reducer.initialize();
275 }
276}
277
278template <int BlockSize, int NumPerThread, typename Self, typename Reducer, typename Index>
279__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void FullReductionKernelHalfFloat(Reducer reducer, const Self input,
280 Index num_coeffs, half* output,
281 half* scratch) {
282 using PacketType = typename packet_traits<Eigen::half>::type;
283 const int packet_width = unpacket_traits<PacketType>::size;
284 eigen_assert(NumPerThread % packet_width == 0);
285 const Index first_index = static_cast<Index>(blockIdx.x) * BlockSize * NumPerThread + packet_width * threadIdx.x;
286
287 // Initialize the output value if it wasn't initialized by the ReductionInitKernel
288
289 if (gridDim.x == 1) {
290 if (first_index == 0) {
291 int rem = num_coeffs % packet_width;
292 if (rem != 0) {
293 half2* p_scratch = reinterpret_cast<half2*>(scratch);
294 pstoreu(scratch, reducer.template initializePacket<PacketType>());
295 for (int i = 0; i < rem / 2; i++) {
296 *p_scratch = __halves2half2(input.coeff(num_coeffs - packet_width + 2 * i),
297 input.coeff(num_coeffs - packet_width + 2 * i + 1));
298 p_scratch++;
299 }
300 if ((num_coeffs & 1) != 0) {
301 half last = input.coeff(num_coeffs - 1);
302 *p_scratch = __halves2half2(last, reducer.initialize());
303 }
304 } else {
305 PacketType reduce = reducer.template initializePacket<PacketType>();
306 pstoreu(scratch, reduce);
307 }
308 }
309 __syncthreads();
310 }
311
312 PacketType accum = reducer.template initializePacket<PacketType>();
313 const Index max_iter =
314 numext::mini<Index>((num_coeffs - first_index) / packet_width, NumPerThread * BlockSize / packet_width);
315 for (Index i = 0; i < max_iter; i += BlockSize) {
316 const Index index = first_index + packet_width * i;
317 eigen_assert(index + packet_width < num_coeffs);
318 PacketType val = input.template packet<Unaligned>(index);
319 reducer.reducePacket(val, &accum);
320 }
321
322#pragma unroll
323 for (int offset = warpSize / 2; offset > 0; offset /= 2) {
324#if defined(EIGEN_HIPCC)
325 PacketType r1;
326 half2* hr = reinterpret_cast<half2*>(&r1);
327 half2* hacc = reinterpret_cast<half2*>(&accum);
328 for (int i = 0; i < packet_width / 2; i++) {
329 // FIXME : remove this workaround once we have native half/half2 support for __shfl_down
330 union {
331 int i;
332 half2 h;
333 } wka_in, wka_out;
334 wka_in.h = hacc[i];
335 wka_out.i = __shfl_down(wka_in.i, offset, warpSize);
336 hr[i] = wka_out.h;
337 }
338 reducer.reducePacket(r1, &accum);
339#else
340 PacketType r1;
341 half2* hr = reinterpret_cast<half2*>(&r1);
342 half2* hacc = reinterpret_cast<half2*>(&accum);
343 for (int i = 0; i < packet_width / 2; i++) {
344 hr[i] = __shfl_down_sync(0xFFFFFFFF, hacc[i], (unsigned)offset, warpSize);
345 }
346 reducer.reducePacket(r1, &accum);
347
348#endif
349 }
350
351 if ((threadIdx.x & (warpSize - 1)) == 0) {
352 atomicReduce(reinterpret_cast<PacketType*>(scratch), accum, reducer);
353 }
354
355 __syncthreads();
356 half2* rv1 = reinterpret_cast<half2*>(scratch);
357 if (packet_width > 2) {
358 reducer.reducePacket(rv1[2], rv1);
359 reducer.reducePacket(rv1[3], rv1 + 1);
360 reducer.reducePacket(rv1[1], rv1);
361 }
362 if (gridDim.x == 1) {
363 if (first_index == 0) {
364 half tmp = __low2half(*rv1);
365 reducer.reduce(__high2half(*rv1), &tmp);
366 *output = tmp;
367 }
368 }
369}
370
371template <typename Op>
372__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void ReductionCleanupKernelHalfFloat(Op reducer, half* output, half* scratch) {
373 eigen_assert(threadIdx.x == 1);
374 using packet_type = packet_traits<Eigen::half>::type;
375 EIGEN_IF_CONSTEXPR (unpacket_traits<packet_type>::size == 1) {
376 *output = *scratch;
377 } else {
378 half2* pscratch = reinterpret_cast<half2*>(scratch);
379 half tmp = __float2half(0.f);
380 for (int i = 0; i < unpacket_traits<packet_type>::size; i += 2) {
381 reducer.reduce(__low2half(*pscratch), &tmp);
382 reducer.reduce(__high2half(*pscratch), &tmp);
383 pscratch++;
384 }
385 *output = tmp;
386 }
387}
388
389template <typename Self, typename Op, typename OutputType, bool PacketAccess, typename Enabled = void>
390struct FullReductionLauncher {
391 static void run(const Self&, Op&, const GpuDevice&, OutputType*, typename Self::Index) {
392 gpu_assert(false && "Should only be called on doubles, floats and half floats");
393 }
394};
395
396// Specialization for float and double
397template <typename Self, typename Op, typename OutputType, bool PacketAccess>
398struct FullReductionLauncher<
399 Self, Op, OutputType, PacketAccess,
400 std::enable_if_t<std::is_same<float, OutputType>::value || std::is_same<double, OutputType>::value, void>> {
401 static void run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output,
402 typename Self::Index num_coeffs) {
403 using Index = typename Self::Index;
404 const int block_size = 256;
405 const int num_per_thread = 128;
406 const int num_blocks = static_cast<int>(numext::div_ceil<Index>(num_coeffs, block_size * num_per_thread));
407
408 unsigned int* semaphore = nullptr;
409 if (num_blocks > 1) {
410 semaphore = device.semaphore();
411 }
412
413 LAUNCH_GPU_KERNEL((FullReductionKernel<block_size, num_per_thread, Self, Op, Index>), num_blocks, block_size, 0,
414 device, reducer, self, num_coeffs, output, semaphore);
415 }
416};
417
418// Half-float reduction specializations.
419template <typename Self, typename Op>
420struct FullReductionLauncher<Self, Op, Eigen::half, false> {
421 static void run(const Self&, Op&, const GpuDevice&, half*, typename Self::Index) {
422 gpu_assert(false && "Should not be called since there is no packet accessor");
423 }
424};
425
426template <typename Self, typename Op>
427struct FullReductionLauncher<Self, Op, Eigen::half, true> {
428 static void run(const Self& self, Op& reducer, const GpuDevice& device, half* output,
429 typename Self::Index num_coeffs) {
430 using Index = typename Self::Index;
431
432 const int block_size = 256;
433 const int num_per_thread = 128;
434 const int num_blocks = static_cast<int>(numext::div_ceil<Index>(num_coeffs, block_size * num_per_thread));
435 half* scratch = static_cast<half*>(device.scratchpad());
436
437 if (num_blocks > 1) {
438 // We initialize the output and the scratchpad outside the reduction kernel when we can't be sure that there
439 // won't be race conditions between multiple thread blocks.
440 LAUNCH_GPU_KERNEL((ReductionInitFullReduxKernelHalfFloat<Self, Op, Index>), 1, 1, 0, device, reducer, self,
441 num_coeffs, scratch);
442 }
443
444 LAUNCH_GPU_KERNEL((FullReductionKernelHalfFloat<block_size, num_per_thread, Self, Op, Index>), num_blocks,
445 block_size, 0, device, reducer, self, num_coeffs, output, scratch);
446
447 if (num_blocks > 1) {
448 LAUNCH_GPU_KERNEL((ReductionCleanupKernelHalfFloat<Op>), 1, 1, 0, device, reducer, output, scratch);
449 }
450 }
451};
452
453template <typename Self, typename Op, bool Vectorizable>
454struct FullReducer<Self, Op, GpuDevice, Vectorizable> {
455 // Unfortunately nvidia doesn't support well exotic types such as complex,
456 // so reduce the scope of the optimized version of the code to the simple cases
457 // of doubles, floats and half floats
458 // Half-float reduction specializations.
459 static constexpr bool HasOptimizedImplementation =
460 !Self::ReducerTraits::IsStateful && (std::is_same<typename Self::CoeffReturnType, float>::value ||
461 std::is_same<typename Self::CoeffReturnType, double>::value ||
462 (std::is_same<typename Self::CoeffReturnType, Eigen::half>::value &&
463 reducer_traits<Op, GpuDevice>::PacketAccess));
464
465 template <typename OutputType>
466 static void run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output) {
467 gpu_assert(HasOptimizedImplementation && "Should only be called on doubles, floats or half floats");
468 const Index num_coeffs = array_prod(self.m_impl.dimensions());
469 // An empty input reduces to the reducer's identity, as on the other devices.
470 if (num_coeffs == 0) {
471 const OutputType identity = reducer.finalize(reducer.initialize());
472 device.fill(output, output + 1, identity);
473 return;
474 }
475
476 FullReductionLauncher<Self, Op, OutputType, reducer_traits<Op, GpuDevice>::PacketAccess>::run(self, reducer, device,
477 output, num_coeffs);
478 }
479};
480
481template <int NumPerThread, typename Self, typename Reducer, typename Index>
482__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void InnerReductionKernel(Reducer reducer, const Self input,
483 Index num_coeffs_to_reduce,
484 Index num_preserved_coeffs,
485 typename Self::CoeffReturnType* output) {
486 using Type = typename Self::CoeffReturnType;
487 eigen_assert(blockDim.y == 1);
488 eigen_assert(blockDim.z == 1);
489 eigen_assert(gridDim.y == 1);
490 eigen_assert(gridDim.z == 1);
491
492 const int unroll_times = 16;
493 eigen_assert(NumPerThread % unroll_times == 0);
494
495 const Index input_col_blocks = numext::div_ceil<Index>(num_coeffs_to_reduce, blockDim.x * NumPerThread);
496 const Index num_input_blocks = input_col_blocks * num_preserved_coeffs;
497
498 const Index num_threads = blockDim.x * gridDim.x;
499 const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x;
500
501 // Initialize the output values if they weren't initialized by the ReductionInitKernel
502 if (gridDim.x == 1) {
503 for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) {
504 output[i] = reducer.initialize();
505 }
506 __syncthreads();
507 }
508
509 for (Index i = blockIdx.x; i < num_input_blocks; i += gridDim.x) {
510 const Index row = i / input_col_blocks;
511
512 // `i`, and so `row`, is uniform across the block: a warp enters this branch as a whole, which the full-mask
513 // shuffle tree below relies on.
514 if (row < num_preserved_coeffs) {
515 const Index col_block = i % input_col_blocks;
516 const Index col_begin = col_block * blockDim.x * NumPerThread + threadIdx.x;
517
518 Type reduced_val = reducer.initialize();
519
520 for (Index j = 0; j < NumPerThread; j += unroll_times) {
521 const Index last_col = col_begin + blockDim.x * (j + unroll_times - 1);
522 if (last_col >= num_coeffs_to_reduce) {
523 for (Index col = col_begin + blockDim.x * j; col < num_coeffs_to_reduce; col += blockDim.x) {
524 const Type val = input.m_impl.coeff(row * num_coeffs_to_reduce + col);
525 reducer.reduce(val, &reduced_val);
526 }
527 break;
528 } else {
529 // Faster version of the loop with no branches after unrolling.
530#pragma unroll
531 for (int k = 0; k < unroll_times; ++k) {
532 const Index col = col_begin + blockDim.x * (j + k);
533 reducer.reduce(input.m_impl.coeff(row * num_coeffs_to_reduce + col), &reduced_val);
534 }
535 }
536 }
537
538#pragma unroll
539 for (int offset = warpSize / 2; offset > 0; offset /= 2) {
540 reducer.reduce(reduction_shuffle_down(reduced_val, offset), &reduced_val);
541 }
542
543 if ((threadIdx.x & (warpSize - 1)) == 0) {
544 atomicReduce(&(output[row]), reduced_val, reducer);
545 }
546 }
547 }
548}
549
550// Half-float reduction specializations.
551
552template <int NumPerThread, typename Self, typename Reducer, typename Index>
553__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void InnerReductionKernelHalfFloat(Reducer reducer, const Self input,
554 Index num_coeffs_to_reduce,
555 Index num_preserved_coeffs, half* output) {
556 eigen_assert(blockDim.y == 1);
557 eigen_assert(blockDim.z == 1);
558 eigen_assert(gridDim.y == 1);
559 eigen_assert(gridDim.z == 1);
560
561 using PacketType = typename packet_traits<Eigen::half>::type;
562 const int packet_width = unpacket_traits<PacketType>::size;
563 const int unroll_times = 16 / packet_width;
564 eigen_assert(NumPerThread % unroll_times == 0);
565 eigen_assert(unroll_times % 2 == 0);
566
567 const Index input_col_blocks = numext::div_ceil<Index>(num_coeffs_to_reduce, blockDim.x * NumPerThread * 2);
568 const Index num_input_blocks = numext::div_ceil<Index>(input_col_blocks * num_preserved_coeffs, 2);
569
570 const Index num_threads = blockDim.x * gridDim.x;
571 const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x;
572
573 // Initialize the output values if they weren't initialized by the ReductionInitKernel
574 if (gridDim.x == 1) {
575 Index i = packet_width * thread_id;
576 for (; i + packet_width <= num_preserved_coeffs; i += packet_width * num_threads) {
577 PacketType* poutput = reinterpret_cast<PacketType*>(output + i);
578 *poutput = reducer.template initializePacket<PacketType>();
579 }
580 if (i < num_preserved_coeffs) {
581 output[i] = reducer.initialize();
582 }
583 __syncthreads();
584 }
585
586 for (Index i = blockIdx.x; i < num_input_blocks; i += gridDim.x) {
587 const Index row = 2 * (i / input_col_blocks); // everybody takes 2 rows
588
589 // Block-uniform, as in InnerReductionKernel: the full-mask shuffles below are safe.
590 if (row + 1 < num_preserved_coeffs) {
591 const Index col_block = i % input_col_blocks;
592 const Index col_begin = packet_width * (col_block * blockDim.x * NumPerThread + threadIdx.x);
593
594 PacketType reduced_val1 = reducer.template initializePacket<PacketType>();
595 PacketType reduced_val2 = reducer.template initializePacket<PacketType>();
596
597 for (Index j = 0; j < NumPerThread; j += unroll_times) {
598 const Index last_col = col_begin + blockDim.x * (j + unroll_times - 1) * packet_width;
599 if (last_col >= num_coeffs_to_reduce) {
600 Index col = col_begin + blockDim.x * j;
601 for (; col + packet_width <= num_coeffs_to_reduce; col += blockDim.x) {
602 const PacketType val1 = input.m_impl.template packet<Unaligned>(row * num_coeffs_to_reduce + col);
603 reducer.reducePacket(val1, &reduced_val1);
604 const PacketType val2 = input.m_impl.template packet<Unaligned>((row + 1) * num_coeffs_to_reduce + col);
605 reducer.reducePacket(val2, &reduced_val2);
606 }
607 if (col < num_coeffs_to_reduce) {
608 PacketType r1 = reducer.template initializePacket<PacketType>();
609 PacketType r2 = reducer.template initializePacket<PacketType>();
610 half2* hr1 = reinterpret_cast<half2*>(&r1);
611 half2* hr2 = reinterpret_cast<half2*>(&r2);
612 while (col + 1 < num_coeffs_to_reduce) {
613 *hr1 = __halves2half2(input.m_impl.coeff(row * num_coeffs_to_reduce + col),
614 input.m_impl.coeff(row * num_coeffs_to_reduce + col + 1));
615 *hr2 = __halves2half2(input.m_impl.coeff((row + 1) * num_coeffs_to_reduce + col),
616 input.m_impl.coeff((row + 1) * num_coeffs_to_reduce + col + 1));
617 hr1++;
618 hr2++;
619 col += 2;
620 }
621 if (col < num_coeffs_to_reduce) {
622 // Peel;
623 const half last1 = input.m_impl.coeff(row * num_coeffs_to_reduce + col);
624 *hr1 = __halves2half2(last1, reducer.initialize());
625 const half last2 = input.m_impl.coeff((row + 1) * num_coeffs_to_reduce + col);
626 *hr2 = __halves2half2(last2, reducer.initialize());
627 }
628 reducer.reducePacket(r1, &reduced_val1);
629 reducer.reducePacket(r2, &reduced_val2);
630 }
631 break;
632 } else {
633 // Faster version of the loop with no branches after unrolling.
634#pragma unroll
635 for (int k = 0; k < unroll_times; ++k) {
636 const Index col = col_begin + blockDim.x * (j + k) * packet_width;
637 reducer.reducePacket(input.m_impl.template packet<Unaligned>(row * num_coeffs_to_reduce + col),
638 &reduced_val1);
639 reducer.reducePacket(input.m_impl.template packet<Unaligned>((row + 1) * num_coeffs_to_reduce + col),
640 &reduced_val2);
641 }
642 }
643 }
644
645#pragma unroll
646 for (int offset = warpSize / 2; offset > 0; offset /= 2) {
647#if defined(EIGEN_HIPCC)
648 PacketType r1;
649 PacketType r2;
650 half2* hr1 = reinterpret_cast<half2*>(&r1);
651 half2* hr2 = reinterpret_cast<half2*>(&r2);
652 half2* rv1 = reinterpret_cast<half2*>(&reduced_val1);
653 half2* rv2 = reinterpret_cast<half2*>(&reduced_val2);
654 for (int i = 0; i < packet_width / 2; i++) {
655 // FIXME : remove this workaround once we have native half/half2 support for __shfl_down
656 union {
657 int i;
658 half2 h;
659 } wka_in1, wka_out1;
660 wka_in1.h = rv1[i];
661 wka_out1.i = __shfl_down(wka_in1.i, offset, warpSize);
662 hr1[i] = wka_out1.h;
663
664 union {
665 int i;
666 half2 h;
667 } wka_in2, wka_out2;
668 wka_in2.h = rv2[i];
669 wka_out2.i = __shfl_down(wka_in2.i, offset, warpSize);
670 hr2[i] = wka_out2.h;
671 }
672 reducer.reducePacket(r1, &reduced_val1);
673 reducer.reducePacket(r2, &reduced_val2);
674#else
675 PacketType r1;
676 PacketType r2;
677 half2* hr1 = reinterpret_cast<half2*>(&r1);
678 half2* hr2 = reinterpret_cast<half2*>(&r2);
679 half2* rr1 = reinterpret_cast<half2*>(&reduced_val1);
680 half2* rr2 = reinterpret_cast<half2*>(&reduced_val2);
681 for (int j = 0; j < packet_width / 2; j++) {
682 hr1[j] = __shfl_down_sync(0xFFFFFFFF, rr1[j], (unsigned)offset, warpSize);
683 hr2[j] = __shfl_down_sync(0xFFFFFFFF, rr2[j], (unsigned)offset, warpSize);
684 }
685 reducer.reducePacket(r1, &reduced_val1);
686 reducer.reducePacket(r2, &reduced_val2);
687
688#endif
689 }
690 half2* rv1 = reinterpret_cast<half2*>(&reduced_val1);
691 half2* rv2 = reinterpret_cast<half2*>(&reduced_val2);
692 half2 val;
693 if (packet_width > 2) {
694 reducer.reducePacket(rv1[2], rv1);
695 reducer.reducePacket(rv1[3], rv1 + 1);
696 reducer.reducePacket(rv1[1], rv1);
697 reducer.reducePacket(rv2[2], rv2);
698 reducer.reducePacket(rv2[3], rv2 + 1);
699 reducer.reducePacket(rv2[1], rv2);
700 }
701 half val1 = __low2half(*rv1);
702 reducer.reduce(__high2half(*rv1), &val1);
703 half val2 = __low2half(*rv2);
704 reducer.reduce(__high2half(*rv2), &val2);
705 val = __halves2half2(val1, val2);
706 if ((threadIdx.x & (warpSize - 1)) == 0) {
707 half* loc = output + row;
708 atomicReduce(reinterpret_cast<half2*>(loc), val, reducer);
709 }
710 }
711 }
712}
713
714template <typename Self, typename Op, typename OutputType, bool PacketAccess, typename Enabled = void>
715struct InnerReductionLauncher {
716 static EIGEN_DEVICE_FUNC bool run(const Self&, Op&, const GpuDevice&, OutputType*, typename Self::Index,
717 typename Self::Index) {
718 gpu_assert(false && "Should only be called to reduce doubles, floats and half floats on a gpu device");
719 return true;
720 }
721};
722
723// Specialization for float and double
724template <typename Self, typename Op, typename OutputType, bool PacketAccess>
725struct InnerReductionLauncher<
726 Self, Op, OutputType, PacketAccess,
727 std::enable_if_t<std::is_same<float, OutputType>::value || std::is_same<double, OutputType>::value, void>> {
728 static bool run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output,
729 typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) {
730 using Index = typename Self::Index;
731
732 const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals;
733 const int block_size = 256;
734 const int num_per_thread = 128;
735 const Index dyn_blocks = numext::div_ceil<Index>(num_coeffs, block_size * num_per_thread);
736 const int max_blocks = device.getNumGpuMultiProcessors() * device.maxGpuThreadsPerMultiProcessor() / block_size;
737 const int num_blocks = static_cast<int>(numext::mini<Index>(max_blocks, dyn_blocks));
738
739 if (num_blocks > 1) {
740 // We initialize the outputs outside the reduction kernel when we can't be sure that there
741 // won't be race conditions between multiple thread blocks.
742 const Index dyn_blocks2 = numext::div_ceil<Index>(num_preserved_vals, 1024);
743 const int max_blocks2 = device.getNumGpuMultiProcessors() * device.maxGpuThreadsPerMultiProcessor() / 1024;
744 const int num_blocks2 = static_cast<int>(numext::mini<Index>(max_blocks2, dyn_blocks2));
745 LAUNCH_GPU_KERNEL((ReductionInitKernel<OutputType, Index>), num_blocks2, 1024, 0, device, reducer.initialize(),
746 num_preserved_vals, output);
747 }
748
749 LAUNCH_GPU_KERNEL((InnerReductionKernel<num_per_thread, Self, Op, Index>), num_blocks, block_size, 0, device,
750 reducer, self, num_coeffs_to_reduce, num_preserved_vals, output);
751
752 return false;
753 }
754};
755
756// Half-float reduction specializations.
757template <typename Self, typename Op>
758struct InnerReductionLauncher<Self, Op, Eigen::half, false> {
759 static bool run(const Self&, Op&, const GpuDevice&, half*, typename Self::Index, typename Self::Index) {
760 gpu_assert(false && "Should not be called since there is no packet accessor");
761 return true;
762 }
763};
764
765template <typename Self, typename Op>
766struct InnerReductionLauncher<Self, Op, Eigen::half, true> {
767 static bool run(const Self& self, Op& reducer, const GpuDevice& device, half* output,
768 typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) {
769 using Index = typename Self::Index;
770
771 if (num_preserved_vals % 2 != 0) {
772 // Not supported yet, revert to the slower code path
773 return true;
774 }
775
776 const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals;
777 const int block_size = 128;
778 const int num_per_thread = 64;
779 const Index dyn_blocks = numext::div_ceil<Index>(num_coeffs, block_size * num_per_thread);
780 const int max_blocks = device.getNumGpuMultiProcessors() * device.maxGpuThreadsPerMultiProcessor() / block_size;
781 const int num_blocks = static_cast<int>(numext::mini<Index>(max_blocks, dyn_blocks));
782
783 if (num_blocks > 1) {
784 // We initialize the outputs outside the reduction kernel when we can't be sure that there
785 // won't be race conditions between multiple thread blocks.
786 LAUNCH_GPU_KERNEL((ReductionInitKernelHalfFloat<Self, Op, Index>), 1, 1, 0, device, reducer, self,
787 num_preserved_vals, output);
788 }
789
790 LAUNCH_GPU_KERNEL((InnerReductionKernelHalfFloat<num_per_thread, Self, Op, Index>), num_blocks, block_size, 0,
791 device, reducer, self, num_coeffs_to_reduce, num_preserved_vals, output);
792
793 return false;
794 }
795};
796
797template <typename Self, typename Op>
798struct InnerReducer<Self, Op, GpuDevice> {
799 // Unfortunately nvidia doesn't support well exotic types such as complex,
800 // so reduce the scope of the optimized version of the code to the simple case
801 // of floats and half floats.
802 // Half-float reduction specializations.
803 static constexpr bool HasOptimizedImplementation =
804 !Self::ReducerTraits::IsStateful && (std::is_same<typename Self::CoeffReturnType, float>::value ||
805 std::is_same<typename Self::CoeffReturnType, double>::value ||
806 (std::is_same<typename Self::CoeffReturnType, Eigen::half>::value &&
807 reducer_traits<Op, GpuDevice>::PacketAccess));
808
809 template <typename OutputType>
810 static bool run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output,
811 typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) {
812 gpu_assert(HasOptimizedImplementation && "Should only be called on doubles, floats or half floats");
813 const Index num_coeffs = array_prod(self.m_impl.dimensions());
814 // Don't crash when we're called with an input tensor of size 0.
815 if (num_coeffs == 0) {
816 return true;
817 }
818 // It's faster to use the usual code.
819 if (num_coeffs_to_reduce <= 128) {
820 return true;
821 }
822
823 return InnerReductionLauncher<Self, Op, OutputType, reducer_traits<Op, GpuDevice>::PacketAccess>::run(
824 self, reducer, device, output, num_coeffs_to_reduce, num_preserved_vals);
825 }
826};
827
828template <int NumPerThread, typename Self, typename Reducer, typename Index>
829__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void OuterReductionKernel(Reducer reducer, const Self input,
830 Index num_coeffs_to_reduce,
831 Index num_preserved_coeffs,
832 typename Self::CoeffReturnType* output) {
833 const Index num_threads = blockDim.x * gridDim.x;
834 const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x;
835 // Initialize the output values if they weren't initialized by the ReductionInitKernel
836 if (gridDim.x == 1) {
837 for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) {
838 output[i] = reducer.initialize();
839 }
840 __syncthreads();
841 }
842
843 // Do the reduction.
844 const Index max_iter = num_preserved_coeffs * numext::div_ceil<Index>(num_coeffs_to_reduce, NumPerThread);
845 for (Index i = thread_id; i < max_iter; i += num_threads) {
846 const Index input_col = i % num_preserved_coeffs;
847 const Index input_row = (i / num_preserved_coeffs) * NumPerThread;
848 typename Self::CoeffReturnType reduced_val = reducer.initialize();
849 const Index max_row = numext::mini(input_row + NumPerThread, num_coeffs_to_reduce);
850 for (Index j = input_row; j < max_row; j++) {
851 typename Self::CoeffReturnType val = input.m_impl.coeff(j * num_preserved_coeffs + input_col);
852 reducer.reduce(val, &reduced_val);
853 }
854 atomicReduce(&(output[input_col]), reduced_val, reducer);
855 }
856}
857
858template <typename Self, typename Op>
859struct OuterReducer<Self, Op, GpuDevice> {
860 // Unfortunately nvidia doesn't support well exotic types such as complex,
861 // so reduce the scope of the optimized version of the code to the simple case
862 // of floats and doubles.
863 template <typename T>
864 using IsFloatOrDouble = bool_constant<std::is_same<T, float>::value || std::is_same<T, double>::value>;
865
866 static constexpr bool HasOptimizedImplementation =
867 !Self::ReducerTraits::IsStateful && IsFloatOrDouble<typename Self::CoeffReturnType>::value;
868 template <typename Device, typename OutputType>
869 static
870#if !defined(EIGEN_HIPCC)
871 // FIXME : leaving this EIGEN_DEVICE_FUNC in, results in the following runtime error
872 // (in the tensor_reduction_gpu test)
873 //
874 // terminate called after throwing an instance of 'std::runtime_error'
875 // what(): No device code available for function: _ZN5Eigen8internal20OuterReductionKernelIL...
876 //
877 // don't know why this happens (and why is it a runtime error instead of a compile time error)
878 //
879 // this will be fixed by HIP PR#457
880 EIGEN_DEVICE_FUNC
881#endif
882 bool
883 run(const Self&, Op&, const Device&, OutputType*, typename Self::Index, typename Self::Index) {
884 gpu_assert(false && "Should only be called to reduce doubles or floats on a gpu device");
885 return true;
886 }
887
888 // OuterReductionKernel is generic in CoeffReturnType, so one launch serves both types that
889 // HasOptimizedImplementation admits. The enable_if keeps every other OutputType on the overload above: the
890 // caller's EIGEN_IF_CONSTEXPR is a plain `if` in C++14, so the call is instantiated for those types as well.
891 template <typename OutputType, EIGEN_SFINAE_ENABLE_IF(IsFloatOrDouble<OutputType>::value)>
892 static bool run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output,
893 typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) {
894 using Index = typename Self::Index;
895
896 // It's faster to use the usual code.
897 if (num_coeffs_to_reduce <= 32) {
898 return true;
899 }
900
901 // CUDA has native FP64 atomicAdd from sm_60 (the atomicReduce<double> specialization above is guarded by
902 // EIGEN_CUDA_ARCH >= 600; sm_50-sm_53 device code takes the correct but slower compare-and-swap loop, and host
903 // code cannot see the target architecture). Keep the measured shape band for the other double reducers and for
904 // HIP, where atomicAdd may still use compare-and-swap.
905#if defined(EIGEN_CUDACC)
906 constexpr bool has_native_double_sum = std::is_same<Op, SumReducer<double>>::value;
907#else
908 constexpr bool has_native_double_sum = false;
909#endif
910 if (std::is_same<OutputType, double>::value && !has_native_double_sum) {
911 const Index multi_processors = device.getNumGpuMultiProcessors();
912 if (num_coeffs_to_reduce < 64 || num_preserved_vals < 8 * multi_processors ||
913 num_preserved_vals > 64 * multi_processors) {
914 return true;
915 }
916 }
917
918 const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals;
919 const int block_size = 256;
920 const int num_per_thread = 16;
921 const Index dyn_blocks = numext::div_ceil<Index>(num_coeffs, block_size * num_per_thread);
922 const int max_blocks = device.getNumGpuMultiProcessors() * device.maxGpuThreadsPerMultiProcessor() / block_size;
923 const int num_blocks = static_cast<int>(numext::mini<Index>(max_blocks, dyn_blocks));
924
925 if (num_blocks > 1) {
926 // We initialize the outputs outside the reduction kernel when we can't be sure that there
927 // won't be race conditions between multiple thread blocks.
928 const Index dyn_blocks2 = numext::div_ceil<Index>(num_preserved_vals, 1024);
929 const int max_blocks2 = device.getNumGpuMultiProcessors() * device.maxGpuThreadsPerMultiProcessor() / 1024;
930 const int num_blocks2 = static_cast<int>(numext::mini<Index>(max_blocks2, dyn_blocks2));
931 LAUNCH_GPU_KERNEL((ReductionInitKernel<OutputType, Index>), num_blocks2, 1024, 0, device, reducer.initialize(),
932 num_preserved_vals, output);
933 }
934
935 LAUNCH_GPU_KERNEL((OuterReductionKernel<num_per_thread, Self, Op, Index>), num_blocks, block_size, 0, device,
936 reducer, self, num_coeffs_to_reduce, num_preserved_vals, output);
937
938 return false;
939 }
940};
941
942#endif // defined(EIGEN_USE_GPU) && defined(EIGEN_GPUCC)
943
944} // end namespace internal
945} // end namespace Eigen
946
947#endif // EIGEN_TENSOR_TENSOR_REDUCTION_GPU_H
Namespace containing all symbols from the Eigen library.