12#ifndef EIGEN_TENSOR_TENSOR_REDUCTION_H
13#define EIGEN_TENSOR_TENSOR_REDUCTION_H
18#if defined(__clang__) && (defined(__CUDA__) || defined(__HIP__))
19#define KERNEL_FRIEND friend __global__ EIGEN_HIP_LAUNCH_BOUNDS_1024
21#define KERNEL_FRIEND friend
26#include "./InternalHeaderCheck.h"
31template <
typename Op,
typename Dims,
typename XprType,
template <
class>
class MakePointer_>
32struct traits<TensorReductionOp<Op, Dims, XprType, MakePointer_> > : traits<XprType> {
33 using XprTraits = traits<XprType>;
34 using Scalar =
typename XprTraits::Scalar;
35 using StorageKind =
typename XprTraits::StorageKind;
36 using Index =
typename XprTraits::Index;
37 static constexpr int NumDimensions = XprTraits::NumDimensions - array_size<Dims>::value;
38 static constexpr int Layout = XprTraits::Layout;
39 using PointerType =
typename XprTraits::PointerType;
43 using Type =
typename MakePointer_<T>::Type;
47template <
typename Op,
typename Dims,
typename XprType,
template <
class>
class MakePointer_>
48struct eval<TensorReductionOp<Op, Dims, XprType, MakePointer_>, Eigen::Dense> {
49 using type =
const TensorReductionOp<Op, Dims, XprType, MakePointer_>&;
52template <
typename OutputDims>
53struct DimInitializer {
54 template <
typename InputDims,
typename ReducedDims>
55 EIGEN_DEVICE_FUNC
static void run(
const InputDims& input_dims,
56 const array<
bool, internal::array_size<InputDims>::value>& reduced,
57 OutputDims* output_dims, ReducedDims* reduced_dims) {
58 const int NumInputDims = internal::array_size<InputDims>::value;
61 for (
int i = 0; i < NumInputDims; ++i) {
63 (*reduced_dims)[reduceIndex] = input_dims[i];
66 (*output_dims)[outputIndex] = input_dims[i];
74struct DimInitializer<Sizes<> > {
75 template <
typename InputDims,
typename Index,
size_t Rank>
76 EIGEN_DEVICE_FUNC
static void run(
const InputDims& input_dims,
const array<bool, Rank>&, Sizes<>*,
77 array<Index, Rank>* reduced_dims) {
78 const int NumInputDims = internal::array_size<InputDims>::value;
79 for (
int i = 0; i < NumInputDims; ++i) {
80 (*reduced_dims)[i] = input_dims[i];
85template <
typename ReducedDims,
int NumTensorDims,
int Layout>
86struct are_inner_most_dims {
87 static constexpr bool value =
false;
89template <
typename ReducedDims,
int NumTensorDims,
int Layout>
90struct preserve_inner_most_dims {
91 static constexpr bool value =
false;
94template <
typename ReducedDims,
int NumTensorDims>
95struct are_inner_most_dims<ReducedDims, NumTensorDims,
ColMajor> {
96 static constexpr bool tmp1 = indices_statically_known_to_increase<ReducedDims>();
97 static constexpr bool tmp2 = index_statically_eq<ReducedDims>(0, 0);
98 static constexpr bool tmp3 =
99 index_statically_eq<ReducedDims>(array_size<ReducedDims>::value - 1, array_size<ReducedDims>::value - 1);
100 static constexpr bool value = tmp1 & tmp2 & tmp3;
102template <
typename ReducedDims,
int NumTensorDims>
103struct are_inner_most_dims<ReducedDims, NumTensorDims,
RowMajor> {
104 static constexpr bool tmp1 = indices_statically_known_to_increase<ReducedDims>();
105 static constexpr bool tmp2 = index_statically_eq<ReducedDims>(0, NumTensorDims - array_size<ReducedDims>::value);
106 static constexpr bool tmp3 = index_statically_eq<ReducedDims>(array_size<ReducedDims>::value - 1, NumTensorDims - 1);
107 static constexpr bool value = tmp1 & tmp2 & tmp3;
109template <
typename ReducedDims,
int NumTensorDims>
110struct preserve_inner_most_dims<ReducedDims, NumTensorDims,
ColMajor> {
111 static constexpr bool tmp1 = indices_statically_known_to_increase<ReducedDims>();
112 static constexpr bool tmp2 = index_statically_gt<ReducedDims>(0, 0);
113 static constexpr bool value = tmp1 & tmp2;
115template <
typename ReducedDims,
int NumTensorDims>
116struct preserve_inner_most_dims<ReducedDims, NumTensorDims,
RowMajor> {
117 static constexpr bool tmp1 = indices_statically_known_to_increase<ReducedDims>();
118 static constexpr bool tmp2 = index_statically_lt<ReducedDims>(array_size<ReducedDims>::value - 1, NumTensorDims - 1);
119 static constexpr bool value = tmp1 & tmp2;
122template <
int DimIndex,
typename Self,
typename Op>
123struct GenericDimReducer {
124 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void reduce(
const Self& self,
typename Self::Index firstIndex,
125 Op& reducer,
typename Self::CoeffReturnType* accum) {
126 EIGEN_STATIC_ASSERT((DimIndex > 0), YOU_MADE_A_PROGRAMMING_MISTAKE);
127 for (
int j = 0; j < self.m_reducedDims[DimIndex]; ++j) {
128 const typename Self::Index input = firstIndex + j * self.m_reducedStrides[DimIndex];
129 GenericDimReducer<DimIndex - 1, Self, Op>::reduce(self, input, reducer, accum);
133template <
typename Self,
typename Op>
134struct GenericDimReducer<0, Self, Op> {
135 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void reduce(
const Self& self,
typename Self::Index firstIndex,
136 Op& reducer,
typename Self::CoeffReturnType* accum) {
137 for (
int j = 0; j < self.m_reducedDims[0]; ++j) {
138 const typename Self::Index input = firstIndex + j * self.m_reducedStrides[0];
139 reducer.reduce(self.m_impl.coeff(input), accum);
143template <
typename Self,
typename Op>
144struct GenericDimReducer<-1, Self, Op> {
145 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void reduce(
const Self& self,
typename Self::Index index, Op& reducer,
146 typename Self::CoeffReturnType* accum) {
147 reducer.reduce(self.m_impl.coeff(index), accum);
151template <
typename Self,
typename Op,
152 bool Vectorizable = (Self::InputPacketAccess && Self::ReducerTraits::PacketAccess),
153 bool UseTreeReduction = (!Self::ReducerTraits::IsStateful && !Self::ReducerTraits::IsExactlyAssociative &&
156 !Self::RunningOnGPU)>
157struct InnerMostDimReducer {
158 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename Self::CoeffReturnType reduce(
159 const Self& self,
typename Self::Index firstIndex,
typename Self::Index numValuesToReduce, Op& reducer) {
160 using Index =
typename Self::Index;
161 typename Self::CoeffReturnType accum0 = reducer.initialize();
166 EIGEN_IF_CONSTEXPR (reducer_can_reorder_accumulators<Op>::value) {
167 if (numValuesToReduce >= 8) {
168 typename Self::CoeffReturnType accum1 = reducer.initialize(), accum2 = reducer.initialize();
169 typename Self::CoeffReturnType accum3 = reducer.initialize(), accum4 = reducer.initialize();
170 typename Self::CoeffReturnType accum5 = reducer.initialize(), accum6 = reducer.initialize();
171 typename Self::CoeffReturnType accum7 = reducer.initialize();
172 const Index unrolledEnd = numValuesToReduce - numValuesToReduce % 8;
173 for (; j < unrolledEnd; j += 8) {
174 reducer.reduce(self.m_impl.coeff(firstIndex + j + 0), &accum0);
175 reducer.reduce(self.m_impl.coeff(firstIndex + j + 1), &accum1);
176 reducer.reduce(self.m_impl.coeff(firstIndex + j + 2), &accum2);
177 reducer.reduce(self.m_impl.coeff(firstIndex + j + 3), &accum3);
178 reducer.reduce(self.m_impl.coeff(firstIndex + j + 4), &accum4);
179 reducer.reduce(self.m_impl.coeff(firstIndex + j + 5), &accum5);
180 reducer.reduce(self.m_impl.coeff(firstIndex + j + 6), &accum6);
181 reducer.reduce(self.m_impl.coeff(firstIndex + j + 7), &accum7);
183 reducer.reduce(accum1, &accum0);
184 reducer.reduce(accum2, &accum0);
185 reducer.reduce(accum3, &accum0);
186 reducer.reduce(accum4, &accum0);
187 reducer.reduce(accum5, &accum0);
188 reducer.reduce(accum6, &accum0);
189 reducer.reduce(accum7, &accum0);
192 for (; j < numValuesToReduce; ++j) {
193 reducer.reduce(self.m_impl.coeff(firstIndex + j), &accum0);
195 return reducer.finalize(accum0);
199template <
typename Self,
typename Op>
200struct InnerMostDimReducer<Self, Op, true, false> {
201 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename Self::CoeffReturnType reduce(
202 const Self& self,
typename Self::Index firstIndex,
typename Self::Index numValuesToReduce, Op& reducer0) {
203 using Index =
typename Self::Index;
204 constexpr Index packetSize = internal::unpacket_traits<typename Self::PacketReturnType>::size;
206 typename Self::PacketReturnType paccum0 = reducer0.template initializePacket<typename Self::PacketReturnType>();
207 EIGEN_IF_CONSTEXPR (reducer_can_reorder_accumulators<Op>::value) {
208 if (numValuesToReduce >= 4 * packetSize) {
209 const Index VectorizedSize4 = (numValuesToReduce / (4 * packetSize)) * (4 * packetSize);
210 typename Self::PacketReturnType paccum1 = reducer0.template initializePacket<typename Self::PacketReturnType>();
211 typename Self::PacketReturnType paccum2 = reducer0.template initializePacket<typename Self::PacketReturnType>();
212 typename Self::PacketReturnType paccum3 = reducer0.template initializePacket<typename Self::PacketReturnType>();
213 const Index offset0 = firstIndex;
214 const Index offset1 = firstIndex + packetSize;
215 const Index offset2 = firstIndex + 2 * packetSize;
216 const Index offset3 = firstIndex + 3 * packetSize;
217 for (Index j = 0; j < VectorizedSize4; j += 4 * packetSize) {
218 reducer0.reducePacket(self.m_impl.template packet<Unaligned>(offset0 + j), &paccum0);
219 reducer0.reducePacket(self.m_impl.template packet<Unaligned>(offset1 + j), &paccum1);
220 reducer0.reducePacket(self.m_impl.template packet<Unaligned>(offset2 + j), &paccum2);
221 reducer0.reducePacket(self.m_impl.template packet<Unaligned>(offset3 + j), &paccum3);
223 reducer0.reducePacket(paccum1, &paccum0);
224 reducer0.reducePacket(paccum2, &paccum0);
225 reducer0.reducePacket(paccum3, &paccum0);
226 start = VectorizedSize4;
229 if (start <= (numValuesToReduce - packetSize)) {
230 const Index VectorizedSize = (numValuesToReduce / packetSize) * packetSize;
231 for (Index j = start; j < VectorizedSize; j += packetSize) {
232 reducer0.reducePacket(self.m_impl.template packet<Unaligned>(firstIndex + j), &paccum0);
234 start = VectorizedSize;
236 typename Self::CoeffReturnType accum = reducer0.initialize();
237 for (Index j = start; j < numValuesToReduce; ++j) {
238 reducer0.reduce(self.m_impl.coeff(firstIndex + j), &accum);
240 return reducer0.finalizeBoth(accum, paccum0);
244#if !defined(EIGEN_HIPCC)
250EIGEN_DEVICE_FUNC
inline Index LeafSize() {
254EIGEN_DEVICE_FUNC
inline Index LeafSize<half>() {
258EIGEN_DEVICE_FUNC
inline Index LeafSize<bfloat16>() {
262template <
typename Self,
typename Op>
263struct InnerMostDimReducer<Self, Op, false, true> {
264 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename Self::CoeffReturnType reduce(
265 const Self& self,
typename Self::Index firstIndex,
typename Self::Index numValuesToReduce, Op& reducer) {
266 const Index kLeafSize = LeafSize<typename Self::CoeffReturnType>();
267 typename Self::CoeffReturnType accum = reducer.initialize();
268 if (numValuesToReduce > kLeafSize) {
269 const typename Self::Index half = numValuesToReduce / 2;
271 reducer.reduce(reduce(self, firstIndex, half, reducer), &accum);
272 reducer.reduce(reduce(self, firstIndex + half, numValuesToReduce - half, reducer), &accum);
273 return reducer.finalize(accum);
275 return InnerMostDimReducer<Self, Op, false, false>::reduce(self, firstIndex, numValuesToReduce, reducer);
280template <
typename Self,
typename Op>
281struct InnerMostDimReducer<Self, Op, true, true> {
282 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename Self::CoeffReturnType reduce(
283 const Self& self,
typename Self::Index firstIndex,
typename Self::Index numValuesToReduce, Op& reducer) {
284 const Index kLeafSize = LeafSize<typename Self::CoeffReturnType>();
285 const typename Self::Index packetSize = internal::unpacket_traits<typename Self::PacketReturnType>::size;
286 typename Self::CoeffReturnType accum = reducer.initialize();
287 if (numValuesToReduce > packetSize * kLeafSize) {
289 const typename Self::Index split =
291 numext::div_ceil(firstIndex + numext::div_ceil(numValuesToReduce,
typename Self::Index(2)), packetSize);
292 const typename Self::Index num_left = numext::mini(split - firstIndex, numValuesToReduce);
293 reducer.reduce(reduce(self, firstIndex, num_left, reducer), &accum);
294 if (num_left < numValuesToReduce) {
295 reducer.reduce(reduce(self, split, numValuesToReduce - num_left, reducer), &accum);
297 return reducer.finalize(accum);
299 return InnerMostDimReducer<Self, Op, true, false>::reduce(self, firstIndex, numValuesToReduce, reducer);
305template <
int DimIndex,
typename Self,
typename Op,
306 bool vectorizable = (Self::InputPacketAccess && Self::ReducerTraits::PacketAccess)>
307struct InnerMostDimPreserver {
308 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void reduce(
const Self&,
typename Self::Index, Op&,
309 typename Self::PacketReturnType*) {
310 eigen_assert(
false &&
"should never be called");
314template <
int DimIndex,
typename Self,
typename Op>
315struct InnerMostDimPreserver<DimIndex, Self, Op, true> {
316 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void reduce(
const Self& self,
typename Self::Index firstIndex,
317 Op& reducer,
typename Self::PacketReturnType* accum) {
318 EIGEN_STATIC_ASSERT((DimIndex > 0), YOU_MADE_A_PROGRAMMING_MISTAKE);
319 for (
typename Self::Index j = 0; j < self.m_reducedDims[DimIndex]; ++j) {
320 const typename Self::Index input = firstIndex + j * self.m_reducedStrides[DimIndex];
321 InnerMostDimPreserver<DimIndex - 1, Self, Op>::reduce(self, input, reducer, accum);
326template <
typename Self,
typename Op>
327struct InnerMostDimPreserver<0, Self, Op, true> {
328 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void reduce(
const Self& self,
typename Self::Index firstIndex,
329 Op& reducer0,
typename Self::PacketReturnType* accum0) {
330 using Index =
typename Self::Index;
331 const Index stride = self.m_reducedStrides[0];
332 const Index size = self.m_reducedDims[0];
333 EIGEN_IF_CONSTEXPR (reducer_can_reorder_accumulators<Op>::value) {
335 const Index unrolled_size4 = (size / 4) * 4;
336 typename Self::PacketReturnType accum1 = reducer0.template initializePacket<typename Self::PacketReturnType>();
337 typename Self::PacketReturnType accum2 = reducer0.template initializePacket<typename Self::PacketReturnType>();
338 typename Self::PacketReturnType accum3 = reducer0.template initializePacket<typename Self::PacketReturnType>();
339 for (Index j = 0; j < unrolled_size4; j += 4) {
340 const Index input0 = firstIndex + j * stride;
341 reducer0.reducePacket(self.m_impl.template packet<Unaligned>(input0), accum0);
342 const Index input1 = firstIndex + (j + 1) * stride;
343 reducer0.reducePacket(self.m_impl.template packet<Unaligned>(input1), &accum1);
344 const Index input2 = firstIndex + (j + 2) * stride;
345 reducer0.reducePacket(self.m_impl.template packet<Unaligned>(input2), &accum2);
346 const Index input3 = firstIndex + (j + 3) * stride;
347 reducer0.reducePacket(self.m_impl.template packet<Unaligned>(input3), &accum3);
349 reducer0.reducePacket(accum1, accum0);
350 reducer0.reducePacket(accum2, accum0);
351 reducer0.reducePacket(accum3, accum0);
352 for (Index j = unrolled_size4; j < size; ++j) {
353 Index input = firstIndex + j * stride;
354 reducer0.reducePacket(self.m_impl.template packet<Unaligned>(input), accum0);
359 for (Index j = 0; j < size; ++j) {
360 Index input = firstIndex + j * stride;
361 reducer0.reducePacket(self.m_impl.template packet<Unaligned>(input), accum0);
365template <
typename Self,
typename Op>
366struct InnerMostDimPreserver<-1, Self, Op,
true> {
367 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void reduce(
const Self&,
typename Self::Index, Op&,
368 typename Self::PacketReturnType*) {
369 eigen_assert(
false &&
"should never be called");
374template <
typename Self,
typename Op,
typename Device,
375 bool Vectorizable = (Self::InputPacketAccess && Self::ReducerTraits::PacketAccess)>
377 static constexpr bool HasOptimizedImplementation =
false;
379 static EIGEN_DEVICE_FUNC
void run(
const Self& self, Op& reducer,
const Device&,
380 typename Self::EvaluatorPointerType output) {
381 const typename Self::Index num_coeffs = array_prod(self.m_impl.dimensions());
382 *output = InnerMostDimReducer<Self, Op, Vectorizable>::reduce(self, 0, num_coeffs, reducer);
386#ifdef EIGEN_USE_THREADS
388template <
typename Self,
typename Op,
bool Vectorizable>
389struct FullReducer<Self, Op, ThreadPoolDevice, Vectorizable> {
391 static constexpr bool HasOptimizedImplementation =
392 !Self::ReducerTraits::IsStateful && reducer_can_reorder_accumulators<Op>::value;
393 static constexpr Index PacketSize = unpacket_traits<typename Self::PacketReturnType>::size;
396 static void run(
const Self& self, Op& reducer,
const ThreadPoolDevice& device,
397 typename Self::CoeffReturnType* output) {
398 using Index =
typename Self::Index;
399 const Index num_coeffs = array_prod(self.m_impl.dimensions());
400 if (num_coeffs == 0) {
401 *output = reducer.finalize(reducer.initialize());
404 const TensorOpCost cost = self.m_impl.costPerCoeff(Vectorizable) +
405 TensorOpCost(0, 0, internal::functor_traits<Op>::Cost, Vectorizable, PacketSize);
406 const Index num_threads = TensorCostModel<ThreadPoolDevice>::numThreads(num_coeffs, cost, device.numThreads());
407 if (num_threads == 1) {
408 *output = InnerMostDimReducer<Self, Op, Vectorizable>::reduce(self, 0, num_coeffs, reducer);
411 const Index blocksize = num_coeffs / num_threads;
412 const Index numblocks = blocksize > 0 ? num_coeffs / blocksize : 0;
413 eigen_assert(num_coeffs >= numblocks * blocksize);
415 Barrier barrier(internal::convert_index<unsigned int>(numblocks));
416 MaxSizeVector<typename Self::CoeffReturnType> shards(numblocks, reducer.initialize());
417 for (Index i = 0; i < numblocks; ++i) {
418 auto run_shard = [i, blocksize, &self, &barrier, &shards, &reducer]() {
419 shards[i] = InnerMostDimReducer<Self, Op, Vectorizable>::reduce(self, i * blocksize, blocksize, reducer);
422 device.enqueue(std::move(run_shard));
424 typename Self::CoeffReturnType finalShard;
425 if (numblocks * blocksize < num_coeffs) {
426 finalShard = InnerMostDimReducer<Self, Op, Vectorizable>::reduce(self, numblocks * blocksize,
427 num_coeffs - numblocks * blocksize, reducer);
429 finalShard = reducer.initialize();
433 for (Index i = 0; i < numblocks; ++i) {
434 reducer.reduce(shards[i], &finalShard);
436 *output = reducer.finalize(finalShard);
443template <
typename Self,
typename Op,
typename Device>
445 static constexpr bool HasOptimizedImplementation =
false;
447 EIGEN_DEVICE_FUNC
static bool run(
const Self&, Op&,
const Device&,
typename Self::CoeffReturnType*,
448 typename Self::Index,
typename Self::Index) {
449 eigen_assert(
false &&
"Not implemented");
455template <
typename Self,
typename Op,
typename Device>
457 static constexpr bool HasOptimizedImplementation =
false;
459 EIGEN_DEVICE_FUNC
static bool run(
const Self&, Op&,
const Device&,
typename Self::CoeffReturnType*,
460 typename Self::Index,
typename Self::Index) {
461 eigen_assert(
false &&
"Not implemented");
468template <
typename Self,
typename Op,
typename Device>
469struct GenericReducer {
470 static constexpr bool HasOptimizedImplementation =
false;
472 EIGEN_DEVICE_FUNC
static bool run(
const Self&, Op&,
const Device&,
typename Self::CoeffReturnType*,
473 typename Self::Index,
typename Self::Index) {
474 eigen_assert(
false &&
"Not implemented");
480#if defined(EIGEN_USE_GPU) && (defined(EIGEN_GPUCC))
481template <
int B,
int N,
typename S,
typename R,
typename I_>
482__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024
void FullReductionKernel(R,
const S, I_,
typename S::CoeffReturnType*,
485#if defined(EIGEN_GPUCC)
488template <
typename S,
typename R,
typename I_>
489__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024
void ReductionInitFullReduxKernelHalfFloat(R,
const S, I_, half*);
490template <
int B,
int N,
typename S,
typename R,
typename I_>
491__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024
void FullReductionKernelHalfFloat(R,
const S, I_, half*, half*);
492template <
int NPT,
typename S,
typename R,
typename I_>
493__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024
void InnerReductionKernelHalfFloat(R,
const S, I_, I_, half*);
497template <
int NPT,
typename S,
typename R,
typename I_>
498__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024
void InnerReductionKernel(R,
const S, I_, I_,
typename S::CoeffReturnType*);
500template <
int NPT,
typename S,
typename R,
typename I_>
501__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024
void OuterReductionKernel(R,
const S, I_, I_,
typename S::CoeffReturnType*);
512template <
typename Op,
typename CoeffReturnType>
514#if defined(EIGEN_USE_SYCL)
515 using type = std::remove_const_t<decltype(std::declval<Op>().initialize())>;
517 using type = std::remove_const_t<CoeffReturnType>;
529template <
typename Op,
typename Dims,
typename XprType,
template <
class>
class MakePointer_>
530class TensorReductionOp :
public TensorBase<TensorReductionOp<Op, Dims, XprType, MakePointer_>, ReadOnlyAccessors> {
532 using Scalar =
typename Eigen::internal::traits<TensorReductionOp>::Scalar;
534 using CoeffReturnType = std::remove_const_t<typename XprType::CoeffReturnType>;
535 using Nested =
typename Eigen::internal::ref_selector<TensorReductionOp>::type;
536 using StorageKind =
typename Eigen::internal::traits<TensorReductionOp>::StorageKind;
537 using Index =
typename Eigen::internal::traits<TensorReductionOp>::Index;
539 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorReductionOp(
const XprType& expr,
const Dims& dims)
540 : m_expr(expr), m_dims(dims) {}
541 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorReductionOp(
const XprType& expr,
const Dims& dims,
const Op& reducer)
542 : m_expr(expr), m_dims(dims), m_reducer(reducer) {}
544 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const XprType& expression()
const {
return m_expr; }
545 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Dims& dims()
const {
return m_dims; }
546 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Op& reducer()
const {
return m_reducer; }
549 template <int NumDims = internal::traits<TensorReductionOp>::NumDimensions, EIGEN_SFINAE_ENABLE_IF(NumDims == 0)>
550 EIGEN_STRONG_INLINE
operator CoeffReturnType()
const {
552 evaluator.evalSubExprsIfNeeded(
nullptr);
553 const CoeffReturnType result = evaluator.coeff(0);
558#if !defined(EIGEN_PARSED_BY_DOXYGEN)
563#define EIGEN_TENSOR_REDUCTION_SCALAR_BINOP(op, name) \
564 template <typename T, EIGEN_SFINAE_ENABLE_IF((internal::is_scalar_operand<T, Scalar>::value))> \
565 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend const TensorCwiseUnaryOp< \
566 internal::bind1st_op<internal::scalar_##name##_op<Scalar> >, const TensorReductionOp> \
567 op(const T& lhs, const TensorReductionOp& rhs) { \
568 return rhs.unaryExpr(internal::bind1st_op<internal::scalar_##name##_op<Scalar> >(static_cast<Scalar>(lhs))); \
570 template <typename T, EIGEN_SFINAE_ENABLE_IF((internal::is_scalar_operand<T, Scalar>::value))> \
571 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE friend const TensorCwiseUnaryOp< \
572 internal::bind2nd_op<internal::scalar_##name##_op<Scalar> >, const TensorReductionOp> \
573 op(const TensorReductionOp& lhs, const T& rhs) { \
574 return lhs.unaryExpr(internal::bind2nd_op<internal::scalar_##name##_op<Scalar> >(static_cast<Scalar>(rhs))); \
577 EIGEN_TENSOR_REDUCTION_SCALAR_BINOP(
operator+, sum)
578 EIGEN_TENSOR_REDUCTION_SCALAR_BINOP(
operator-, difference)
579 EIGEN_TENSOR_REDUCTION_SCALAR_BINOP(
operator*, product)
580 EIGEN_TENSOR_REDUCTION_SCALAR_BINOP(
operator/, quotient)
581#undef EIGEN_TENSOR_REDUCTION_SCALAR_BINOP
585 typename XprType::Nested m_expr;
590template <
typename ArgType,
typename Device>
591struct TensorReductionEvaluatorBase;
594template <
typename Op,
typename Dims,
typename ArgType,
template <
class>
class MakePointer_,
typename Device>
595struct TensorReductionEvaluatorBase<const
TensorReductionOp<Op, Dims, ArgType, MakePointer_>, Device> {
596 using ReducerTraits = internal::reducer_traits<Op, Device>;
597 using ReducedDims = Dims;
599 using Index =
typename XprType::Index;
600 using ChildType = ArgType;
601 using InputDimensions =
typename TensorEvaluator<ArgType, Device>::Dimensions;
602 static constexpr int NumInputDims = internal::array_size<InputDimensions>::value;
603 static constexpr int NumReducedDims = internal::array_size<Dims>::value;
604 static constexpr int NumOutputDims = NumInputDims - NumReducedDims;
605 using Dimensions = std::conditional_t<NumOutputDims == 0, Sizes<>, DSizes<Index, NumOutputDims> >;
606 using Scalar =
typename XprType::Scalar;
607 using Self = TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, MakePointer_>, Device>;
608 static constexpr bool InputPacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess;
609 using CoeffReturnType =
typename internal::ReductionReturnType<Op, typename XprType::CoeffReturnType>::type;
610 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
611 static constexpr Index PacketSize = PacketType<CoeffReturnType, Device>::size;
613 using TensorPointerType =
typename Eigen::internal::traits<XprType>::PointerType;
614 using Storage = StorageMemory<CoeffReturnType, Device>;
615 using EvaluatorPointerType =
typename Storage::Type;
619 static constexpr int NumPreservedStrides = max_n_1<NumOutputDims>::size;
622#if defined(EIGEN_USE_GPU) && (defined(EIGEN_GPUCC))
623 static constexpr bool RunningOnGPU = std::is_same<Device, Eigen::GpuDevice>::value;
624 static constexpr bool RunningOnSycl =
false;
625#elif defined(EIGEN_USE_SYCL)
626 static constexpr bool RunningOnSycl = std::is_same<internal::remove_all_t<Device>, Eigen::SyclDevice>::value;
627 static constexpr bool RunningOnGPU =
false;
629 static constexpr bool RunningOnGPU =
false;
630 static constexpr bool RunningOnSycl =
false;
633 static constexpr int Layout = TensorEvaluator<ArgType, Device>::Layout;
636 PacketAccess = Self::InputPacketAccess && ReducerTraits::PacketAccess,
638 PreferBlockAccess =
true,
643 using ScalarNoConst = std::remove_const_t<Scalar>;
646 using TensorBlock = internal::TensorBlockNotImplemented;
649 static constexpr bool ReducingInnerMostDims = internal::are_inner_most_dims<Dims, NumInputDims, Layout>::value;
650 static constexpr bool PreservingInnerMostDims = internal::preserve_inner_most_dims<Dims, NumInputDims, Layout>::value;
651 static constexpr bool RunningFullReduction = (NumOutputDims == 0);
653 EIGEN_STRONG_INLINE TensorReductionEvaluatorBase(
const XprType& op,
const Device& device)
654 : m_impl(op.expression(), device), m_reducer(op.reducer()), m_result(nullptr), m_device(device) {
655 EIGEN_STATIC_ASSERT((NumInputDims >= NumReducedDims), YOU_MADE_A_PROGRAMMING_MISTAKE);
656 EIGEN_STATIC_ASSERT((!ReducingInnerMostDims | !PreservingInnerMostDims | (NumReducedDims == NumInputDims)),
657 YOU_MADE_A_PROGRAMMING_MISTAKE);
660 for (
int i = 0; i < NumInputDims; ++i) {
661 m_reduced[i] =
false;
663 for (
int i = 0; i < NumReducedDims; ++i) {
664 eigen_assert(op.dims()[i] >= 0);
665 eigen_assert(op.dims()[i] < NumInputDims);
666 m_reduced[op.dims()[i]] =
true;
669 const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
670 internal::DimInitializer<Dimensions>::run(input_dims, m_reduced, &m_dimensions, &m_reducedDims);
673 EIGEN_IF_CONSTEXPR (NumOutputDims > 0) {
674 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
675 m_outputStrides[0] = 1;
676 for (
int i = 1; i < NumOutputDims; ++i) {
677 m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
678 m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(m_outputStrides[i] > 0 ? m_outputStrides[i] : 1);
681 m_outputStrides[
static_cast<size_t>(NumOutputDims - 1)] = 1;
682 for (
int i = NumOutputDims - 2; i >= 0; --i) {
683 m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1];
684 m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(m_outputStrides[i] > 0 ? m_outputStrides[i] : 1);
690 EIGEN_IF_CONSTEXPR (NumInputDims > 0) {
691 array<Index, NumInputDims> input_strides;
692 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
693 input_strides[0] = 1;
694 for (
int i = 1; i < NumInputDims; ++i) {
695 input_strides[i] = input_strides[i - 1] * input_dims[i - 1];
698 input_strides.back() = 1;
699 for (
int i = NumInputDims - 2; i >= 0; --i) {
700 input_strides[i] = input_strides[i + 1] * input_dims[i + 1];
706 for (
int i = 0; i < NumInputDims; ++i) {
708 m_reducedStrides[reduceIndex] = input_strides[i];
711 m_preservedStrides[outputIndex] = input_strides[i];
712 m_output_to_input_dim_map[outputIndex] = i;
719 EIGEN_IF_CONSTEXPR (NumOutputDims == 0) {
720 m_preservedStrides[0] = internal::array_prod(input_dims);
723 m_numValuesToReduce = NumOutputDims == 0 ? internal::array_prod(input_dims)
724 : (static_cast<int>(Layout) == static_cast<int>(
ColMajor))
725 ? m_preservedStrides[0]
726 : m_preservedStrides[static_cast<size_t>(NumOutputDims - 1)];
730 m_reducingInnerMostDims = (NumReducedDims > 0);
731 for (
int i = 0; i < NumReducedDims && m_reducingInnerMostDims; ++i) {
732 const int axis = (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) ? i : NumInputDims - 1 - i;
733 if (!m_reduced[axis]) m_reducingInnerMostDims =
false;
737 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Dimensions& dimensions()
const {
return m_dimensions; }
739 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeededCommon(EvaluatorPointerType data) {
741 EIGEN_IF_CONSTEXPR (RunningFullReduction) {
742 if (RunningOnSycl || (internal::FullReducer<Self, Op, Device>::HasOptimizedImplementation &&
743 ((RunningOnGPU && (m_device.majorDeviceVersion() >= 3)) || !RunningOnGPU))) {
744 bool need_assign =
false;
746 m_result =
static_cast<EvaluatorPointerType
>(
747 m_device.get((CoeffReturnType*)m_device.allocate_temp(
sizeof(CoeffReturnType))));
751 Op reducer(m_reducer);
752 internal::FullReducer<Self, Op, Device>::run(*
this, reducer, m_device, data);
758 EIGEN_IF_CONSTEXPR (RunningOnGPU || RunningOnSycl) {
759 if ((RunningOnGPU && (m_device.majorDeviceVersion() >= 3)) || (RunningOnSycl)) {
760 bool reducing_inner_dims =
true;
761 for (
int i = 0; i < NumReducedDims; ++i) {
762 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
763 reducing_inner_dims &= m_reduced[i];
765 reducing_inner_dims &= m_reduced[NumInputDims - 1 - i];
768 EIGEN_IF_CONSTEXPR ((internal::InnerReducer<Self, Op, Device>::HasOptimizedImplementation)) {
769 if (reducing_inner_dims || ReducingInnerMostDims) {
770 const Index num_values_to_reduce = internal::array_prod(m_reducedDims);
771 const Index num_coeffs_to_preserve =
static_cast<Index
>(internal::array_prod(m_dimensions));
773 if ((num_coeffs_to_preserve < 1024 && num_values_to_reduce > num_coeffs_to_preserve &&
774 num_values_to_reduce > 128) ||
776 data =
static_cast<EvaluatorPointerType
>(m_device.get(
777 (CoeffReturnType*)m_device.allocate_temp(
sizeof(CoeffReturnType) * num_coeffs_to_preserve)));
783 Op reducer(m_reducer);
785 if (internal::InnerReducer<Self, Op, Device>::run(*
this, reducer, m_device, data, num_values_to_reduce,
786 num_coeffs_to_preserve)) {
788 m_device.deallocate_temp(m_result);
793 return (m_result !=
nullptr);
798 bool preserving_inner_dims =
true;
799 for (
int i = 0; i < NumReducedDims; ++i) {
800 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
801 preserving_inner_dims &= m_reduced[NumInputDims - 1 - i];
803 preserving_inner_dims &= m_reduced[i];
806 EIGEN_IF_CONSTEXPR ((internal::OuterReducer<Self, Op, Device>::HasOptimizedImplementation)) {
807 if (preserving_inner_dims) {
808 const Index num_values_to_reduce = internal::array_prod(m_reducedDims);
809 const Index num_coeffs_to_preserve =
static_cast<Index
>(internal::array_prod(m_dimensions));
811 if ((num_coeffs_to_preserve < 1024 && num_values_to_reduce > num_coeffs_to_preserve &&
812 num_values_to_reduce > 32) ||
814 data =
static_cast<EvaluatorPointerType
>(m_device.get(
815 (CoeffReturnType*)m_device.allocate_temp(
sizeof(CoeffReturnType) * num_coeffs_to_preserve)));
821 Op reducer(m_reducer);
823 if (internal::OuterReducer<Self, Op, Device>::run(*
this, reducer, m_device, data, num_values_to_reduce,
824 num_coeffs_to_preserve)) {
826 m_device.deallocate_temp(m_result);
831 return (m_result !=
nullptr);
835#if defined(EIGEN_USE_SYCL)
838 EIGEN_IF_CONSTEXPR (RunningOnSycl) {
839 const Index num_values_to_reduce = internal::array_prod(m_reducedDims);
840 const Index num_coeffs_to_preserve =
static_cast<Index
>(internal::array_prod(m_dimensions));
842 data =
static_cast<EvaluatorPointerType
>(m_device.get(
843 (CoeffReturnType*)m_device.allocate_temp(
sizeof(CoeffReturnType) * num_coeffs_to_preserve)));
846 Op reducer(m_reducer);
847 internal::GenericReducer<Self, Op, Device>::run(*
this, reducer, m_device, data, num_values_to_reduce,
848 num_coeffs_to_preserve);
849 return (m_result !=
nullptr);
857#ifdef EIGEN_USE_THREADS
858 template <
typename EvalSubExprsCallback>
859 EIGEN_STRONG_INLINE
void evalSubExprsIfNeededAsync(EvaluatorPointerType data, EvalSubExprsCallback done) {
860 m_impl.evalSubExprsIfNeededAsync(
nullptr, [
this, data, done](
bool) { done(evalSubExprsIfNeededCommon(data)); });
864 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(EvaluatorPointerType data) {
865 m_impl.evalSubExprsIfNeeded(
nullptr);
866 return evalSubExprsIfNeededCommon(data);
869 EIGEN_STRONG_INLINE
void cleanup() {
872 m_device.deallocate_temp(m_result);
877 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index)
const {
878 EIGEN_IF_CONSTEXPR (RunningFullReduction || RunningOnGPU) {
880 return *(m_result + index);
883 Op reducer(m_reducer);
884 EIGEN_IF_CONSTEXPR (ReducingInnerMostDims || RunningFullReduction) {
885 const Index num_values_to_reduce = (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor))
886 ? m_preservedStrides[0]
887 : m_preservedStrides[NumPreservedStrides - 1];
888 return internal::InnerMostDimReducer<Self, Op>::reduce(*
this, firstInput(index), num_values_to_reduce, reducer);
889 }
else if (m_reducingInnerMostDims) {
890 return internal::InnerMostDimReducer<Self, Op>::reduce(*
this, index * m_numValuesToReduce, m_numValuesToReduce,
893 typename Self::CoeffReturnType accum = reducer.initialize();
894 internal::GenericDimReducer<NumReducedDims - 1, Self, Op>::reduce(*
this, firstInput(index), reducer, &accum);
895 return reducer.finalize(accum);
900 template <
int LoadMode>
901 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index)
const {
902 eigen_assert(index + PacketSize - 1 < Index(internal::array_prod(dimensions())));
904 EIGEN_IF_CONSTEXPR (RunningOnGPU) {
906 return internal::pload<PacketReturnType>(m_result + index);
910 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment)
911 std::remove_const_t<CoeffReturnType> values[PacketSize];
914 EIGEN_IF_CONSTEXPR (ReducingInnerMostDims) {
915 const Index num_values_to_reduce = (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor))
916 ? m_preservedStrides[0]
917 : m_preservedStrides[NumPreservedStrides - 1];
918 const Index firstIndex = firstInput(index);
919 for (Index i = 0; i < PacketSize; ++i) {
920 Op reducer(m_reducer);
921 const CoeffReturnType value = internal::InnerMostDimReducer<Self, Op>::reduce(
922 *
this, firstIndex + i * num_values_to_reduce, num_values_to_reduce, reducer);
925 }
else EIGEN_IF_CONSTEXPR (PreservingInnerMostDims) {
926 const Index firstIndex = firstInput(index);
927 constexpr int innermost_dim = (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) ? 0 : NumOutputDims - 1;
929 if (((firstIndex % m_dimensions[innermost_dim]) + PacketSize - 1) < m_dimensions[innermost_dim]) {
930 Op reducer(m_reducer);
931 typename Self::PacketReturnType accum = reducer.template initializePacket<typename Self::PacketReturnType>();
932 internal::InnerMostDimPreserver<NumReducedDims - 1, Self, Op>::reduce(*
this, firstIndex, reducer, &accum);
933 return reducer.finalizePacket(accum);
935 for (
int i = 0; i < PacketSize; ++i) {
936 values[i] = coeff(index + i);
939 }
else if (m_reducingInnerMostDims) {
942 const Index firstIndex = index * m_numValuesToReduce;
943 for (Index i = 0; i < PacketSize; ++i) {
944 Op reducer(m_reducer);
945 const CoeffReturnType value = internal::InnerMostDimReducer<Self, Op>::reduce(
946 *
this, firstIndex + i * m_numValuesToReduce, m_numValuesToReduce, reducer);
950 for (
int i = 0; i < PacketSize; ++i) {
951 values[i] = coeff(index + i);
954 PacketReturnType rslt = internal::pload<PacketReturnType>(values);
959 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool vectorized)
const {
960 EIGEN_IF_CONSTEXPR (RunningFullReduction) {
962 return TensorOpCost(
sizeof(CoeffReturnType), 0, 0, vectorized, PacketSize);
965 const Index num_values_to_reduce = internal::array_prod(m_reducedDims);
966 const double compute_cost = num_values_to_reduce * internal::functor_traits<Op>::Cost;
967 return m_impl.costPerCoeff(vectorized) * num_values_to_reduce +
968 TensorOpCost(0, 0, compute_cost, vectorized, PacketSize);
971 EIGEN_DEVICE_FUNC EvaluatorPointerType data()
const {
return m_result; }
972 EIGEN_DEVICE_FUNC
const TensorEvaluator<ArgType, Device>& impl()
const {
return m_impl; }
973 EIGEN_DEVICE_FUNC
const Device& device()
const {
return m_device; }
976 template <
int,
typename,
typename>
977 friend struct internal::GenericDimReducer;
978 template <
typename,
typename,
bool,
bool>
979 friend struct internal::InnerMostDimReducer;
980 template <
int,
typename,
typename,
bool>
981 friend struct internal::InnerMostDimPreserver;
982 template <
typename S,
typename O,
typename D,
bool V>
983 friend struct internal::FullReducer;
984#if defined(EIGEN_USE_GPU) && (defined(EIGEN_GPUCC))
985 template <
int B,
int N,
typename S,
typename R,
typename I_>
986 KERNEL_FRIEND
void internal::FullReductionKernel(R,
const S, I_,
typename S::CoeffReturnType*,
unsigned int*);
987#if defined(EIGEN_GPUCC)
988 template <
typename S,
typename R,
typename I_>
989 KERNEL_FRIEND
void internal::ReductionInitFullReduxKernelHalfFloat(R,
const S, I_, half*);
990 template <
int B,
int N,
typename S,
typename R,
typename I_>
991 KERNEL_FRIEND
void internal::FullReductionKernelHalfFloat(R,
const S, I_, half*, half*);
992 template <
int NPT,
typename S,
typename R,
typename I_>
993 KERNEL_FRIEND
void internal::InnerReductionKernelHalfFloat(R,
const S, I_, I_, half*);
995 template <
int NPT,
typename S,
typename R,
typename I_>
996 KERNEL_FRIEND
void internal::InnerReductionKernel(R,
const S, I_, I_,
typename S::CoeffReturnType*);
998 template <
int NPT,
typename S,
typename R,
typename I_>
999 KERNEL_FRIEND
void internal::OuterReductionKernel(R,
const S, I_, I_,
typename S::CoeffReturnType*);
1002#if defined(EIGEN_USE_SYCL)
1003 template <
typename Evaluator_,
typename Op__>
1004 friend class TensorSycl::internal::GenericNondeterministicReducer;
1006 template <
typename,
typename,
typename>
1007 friend struct internal::GenericReducer;
1010 template <
typename S,
typename O,
typename D>
1011 friend struct internal::InnerReducer;
1015 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index)
const {
1016 EIGEN_IF_CONSTEXPR (ReducingInnerMostDims) {
1017 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
1018 return index * m_preservedStrides[0];
1020 return index * m_preservedStrides[NumPreservedStrides - 1];
1024 Index startInput = 0;
1025 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
1026 for (
int i = NumOutputDims - 1; i > 0; --i) {
1028 const Index idx = index / m_outputStrides[i];
1029 startInput += idx * m_preservedStrides[i];
1030 index -= idx * m_outputStrides[i];
1032 EIGEN_IF_CONSTEXPR (PreservingInnerMostDims) {
1033 eigen_assert(m_preservedStrides[0] == 1);
1034 startInput += index;
1036 startInput += index * m_preservedStrides[0];
1039 for (
int i = 0; i < NumOutputDims - 1; ++i) {
1041 const Index idx = index / m_outputStrides[i];
1042 startInput += idx * m_preservedStrides[i];
1043 index -= idx * m_outputStrides[i];
1045 EIGEN_IF_CONSTEXPR (PreservingInnerMostDims) {
1046 eigen_assert(m_preservedStrides[NumPreservedStrides - 1] == 1);
1047 startInput += index;
1049 startInput += index * m_preservedStrides[NumPreservedStrides - 1];
1056 array<bool, NumInputDims> m_reduced;
1058 Dimensions m_dimensions;
1061 array<Index, (std::max)(NumOutputDims, 1)> m_outputStrides;
1062 array<internal::TensorIntDivisor<Index>, (std::max)(NumOutputDims, 1)> m_fastOutputStrides;
1063 array<Index, (std::max)(NumPreservedStrides, 1)> m_preservedStrides;
1065 array<Index, (std::max)(NumOutputDims, 1)> m_output_to_input_dim_map;
1067 Index m_numValuesToReduce;
1070 bool m_reducingInnerMostDims;
1074 array<Index, NumReducedDims> m_reducedStrides;
1077 array<Index, NumReducedDims> m_reducedDims;
1080 TensorEvaluator<ArgType, Device> m_impl;
1085 EvaluatorPointerType m_result;
1087 const Device EIGEN_DEVICE_REF m_device;
1090template <
typename Op,
typename Dims,
typename ArgType,
template <
class>
class MakePointer_,
typename Device>
1092 :
public TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, MakePointer_>, Device> {
1093 using Base = TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, MakePointer_>, Device>;
1094 EIGEN_STRONG_INLINE TensorEvaluator(
const typename Base::XprType& op,
const Device& device) : Base(op, device) {}
1097template <
typename Op,
typename Dims,
typename ArgType,
template <
class>
class MakePointer_>
1099 :
public TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, MakePointer_>, Eigen::SyclDevice> {
1101 TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, MakePointer_>, Eigen::SyclDevice>;
1102 EIGEN_STRONG_INLINE TensorEvaluator(
const typename Base::XprType& op,
const Eigen::SyclDevice& device)
1103 : Base(op, device) {}
1106 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename Base::CoeffReturnType coeff(
typename Base::Index index)
const {
1107 return *(this->data() + index);
1111 template <
int LoadMode>
1112 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename Base::PacketReturnType packet(
typename Base::Index index)
const {
1113 return internal::pload<typename Base::PacketReturnType>(this->data() + index);
The tensor base class.
Definition TensorForwardDeclarations.h:69
Tensor reduction class.
Definition TensorReduction.h:530
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47
Definition TensorReduction.h:513