11#ifndef EIGEN_TENSOR_TENSOR_EVALUATOR_H
12#define EIGEN_TENSOR_TENSOR_EVALUATOR_H
15#include "./InternalHeaderCheck.h"
26template <
typename Functor,
typename =
void>
27struct tensor_functor_is_repeatable : std::false_type {};
29template <
typename Functor>
30struct tensor_functor_is_repeatable<Functor, void_t<decltype(functor_traits<Functor>::IsRepeatable)>>
31 : bool_constant<functor_traits<Functor>::IsRepeatable> {};
46template <
typename Derived,
typename Device>
47struct TensorEvaluator {
48 using Index =
typename Derived::Index;
49 using Scalar =
typename Derived::Scalar;
50 using CoeffReturnType =
typename Derived::Scalar;
51 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
52 using Dimensions =
typename Derived::Dimensions;
53 using XprType = Derived;
54 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
55 using TensorPointerType =
typename internal::traits<Derived>::template MakePointer<Scalar>::Type;
56 using Storage = StorageMemory<Scalar, Device>;
57 using EvaluatorPointerType =
typename Storage::Type;
60 static constexpr int NumCoords =
61 internal::traits<Derived>::NumDimensions > 0 ? internal::traits<Derived>::NumDimensions : 0;
62 static constexpr int Layout = Derived::Layout;
65 IsAligned = Derived::IsAligned,
66 PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1),
67 BlockAccess = internal::is_arithmetic<std::remove_const_t<Scalar>>::value,
68 PreferBlockAccess =
false,
69 CoordAccess = NumCoords > 0,
73 using ScalarNoConst = std::remove_const_t<Scalar>;
76 using TensorBlockDesc = internal::TensorBlockDescriptor<NumCoords, Index>;
77 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
79 using TensorBlock =
typename internal::TensorMaterializedBlock<ScalarNoConst, NumCoords, Layout, Index>;
82 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(
const Derived& m,
const Device& device)
83 : m_data(device.get(
const_cast<TensorPointerType
>(m.data()))), m_dims(m.dimensions()), m_device(device) {}
85 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Dimensions& dimensions()
const {
return m_dims; }
87 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(EvaluatorPointerType dest) {
88 EIGEN_IF_CONSTEXPR (!
NumTraits<std::remove_const_t<Scalar>>::RequireInitialization) {
90 m_device.memcpy((
void*)(m_device.get(dest)), m_device.get(m_data), m_dims.TotalSize() *
sizeof(Scalar));
97#ifdef EIGEN_USE_THREADS
98 template <
typename EvalSubExprsCallback>
99 EIGEN_STRONG_INLINE
void evalSubExprsIfNeededAsync(EvaluatorPointerType dest, EvalSubExprsCallback done) {
101 done(evalSubExprsIfNeeded(dest));
105 EIGEN_STRONG_INLINE
void cleanup() {}
107 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index)
const {
108 eigen_assert(m_data !=
nullptr);
109 return m_data[index];
112 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index)
const {
113 eigen_assert(m_data !=
nullptr);
114 return m_data[index];
117 template <
int LoadMode>
118 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index)
const {
119 return internal::ploadt<PacketReturnType, LoadMode>(m_data + index);
127 template <
typename PacketReturnTypeT>
128 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
129 std::enable_if_t<internal::unpacket_traits<PacketReturnTypeT>::masked_load_available, PacketReturnTypeT>
130 partialPacket(Index index,
typename internal::unpacket_traits<PacketReturnTypeT>::mask_t umask)
const {
131 return internal::ploadu<PacketReturnTypeT>(m_data + index, umask);
134 template <
int StoreMode>
135 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void writePacket(Index index,
const PacketReturnType& x)
const {
136 return internal::pstoret<Scalar, PacketReturnType, StoreMode>(m_data + index, x);
139 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(
const array<DenseIndex, NumCoords>& coords)
const {
140 eigen_assert(m_data !=
nullptr);
141 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
142 return m_data[m_dims.IndexOfColMajor(coords)];
144 return m_data[m_dims.IndexOfRowMajor(coords)];
148 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(
const array<DenseIndex, NumCoords>& coords)
const {
149 eigen_assert(m_data !=
nullptr);
150 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
151 return m_data[m_dims.IndexOfColMajor(coords)];
153 return m_data[m_dims.IndexOfRowMajor(coords)];
157 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool vectorized)
const {
158 return TensorOpCost(
sizeof(CoeffReturnType), 0, 0, vectorized, PacketType<CoeffReturnType, Device>::size);
161 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements()
const {
162 return internal::TensorBlockResourceRequirements::any();
165 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
166 bool =
false)
const {
167 eigen_assert(m_data !=
nullptr);
168 return TensorBlock::materialize(m_data, m_dims, desc, scratch);
171 template <
typename TensorBlock>
172 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void writeBlock(
const TensorBlockDesc& desc,
const TensorBlock& block) {
173 eigen_assert(m_data !=
nullptr);
175 using TensorBlockExpr =
typename TensorBlock::XprType;
176 using TensorBlockAssign = internal::TensorBlockAssignment<Scalar, NumCoords, TensorBlockExpr, Index>;
178 TensorBlockAssign::Run(
179 TensorBlockAssign::target(desc.dimensions(), internal::strides<Layout>(m_dims), m_data, desc.offset()),
183 EIGEN_DEVICE_FUNC EvaluatorPointerType data()
const {
return m_data; }
186 EvaluatorPointerType m_data;
188 const Device EIGEN_DEVICE_REF m_device;
193EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T loadConstant(
const T* address) {
197#if defined(EIGEN_CUDA_ARCH)
199EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
float loadConstant(
const float* address) {
200 return __ldg(address);
203EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
double loadConstant(
const double* address) {
204 return __ldg(address);
207EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Eigen::half loadConstant(
const Eigen::half* address) {
208 return Eigen::half(half_impl::raw_uint16_to_half(__ldg(&address->x)));
215template <
typename Derived,
typename Device>
217 using Index =
typename Derived::Index;
218 using Scalar =
typename Derived::Scalar;
219 using CoeffReturnType =
typename Derived::Scalar;
220 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
221 using Dimensions =
typename Derived::Dimensions;
222 using XprType =
const Derived;
223 using TensorPointerType =
typename internal::traits<Derived>::template MakePointer<const Scalar>::Type;
224 using Storage = StorageMemory<const Scalar, Device>;
225 using EvaluatorPointerType =
typename Storage::Type;
227 using ScalarNoConst = std::remove_const_t<Scalar>;
230 static constexpr int NumCoords =
231 internal::traits<Derived>::NumDimensions > 0 ? internal::traits<Derived>::NumDimensions : 0;
232 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
233 static constexpr int Layout = Derived::Layout;
236 IsAligned = Derived::IsAligned,
237 PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1),
238 BlockAccess = internal::is_arithmetic<ScalarNoConst>::value,
239 PreferBlockAccess =
false,
240 CoordAccess = NumCoords > 0,
245 using TensorBlockDesc = internal::TensorBlockDescriptor<NumCoords, Index>;
246 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
248 using TensorBlock =
typename internal::TensorMaterializedBlock<ScalarNoConst, NumCoords, Layout, Index>;
251 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC TensorEvaluator(
const Derived& m,
const Device& device)
252 : m_data(device.get(m.data())), m_dims(m.dimensions()), m_device(device) {}
254 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Dimensions& dimensions()
const {
return m_dims; }
256 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(EvaluatorPointerType data) {
257 EIGEN_IF_CONSTEXPR (!NumTraits<std::remove_const_t<Scalar>>::RequireInitialization) {
259 m_device.memcpy((
void*)(m_device.get(data)), m_device.get(m_data), m_dims.TotalSize() *
sizeof(Scalar));
266#ifdef EIGEN_USE_THREADS
267 template <
typename EvalSubExprsCallback>
268 EIGEN_STRONG_INLINE
void evalSubExprsIfNeededAsync(EvaluatorPointerType dest, EvalSubExprsCallback done) {
270 done(evalSubExprsIfNeeded(dest));
274 EIGEN_STRONG_INLINE
void cleanup() {}
276 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index)
const {
277 eigen_assert(m_data !=
nullptr);
278 return internal::loadConstant(m_data + index);
281 template <
int LoadMode>
282 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index)
const {
283 return internal::ploadt_ro<PacketReturnType, LoadMode>(m_data + index);
291 template <
typename PacketReturnTypeT>
292 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
293 std::enable_if_t<internal::unpacket_traits<PacketReturnTypeT>::masked_load_available, PacketReturnTypeT>
294 partialPacket(Index index,
typename internal::unpacket_traits<PacketReturnTypeT>::mask_t umask)
const {
295 return internal::ploadu<PacketReturnTypeT>(m_data + index, umask);
298 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(
const array<DenseIndex, NumCoords>& coords)
const {
299 eigen_assert(m_data !=
nullptr);
300 const Index index = (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) ? m_dims.IndexOfColMajor(coords)
301 : m_dims.IndexOfRowMajor(coords);
302 return internal::loadConstant(m_data + index);
305 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool vectorized)
const {
306 return TensorOpCost(
sizeof(CoeffReturnType), 0, 0, vectorized, PacketType<CoeffReturnType, Device>::size);
309 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements()
const {
310 return internal::TensorBlockResourceRequirements::any();
313 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
314 bool =
false)
const {
315 eigen_assert(m_data !=
nullptr);
316 return TensorBlock::materialize(m_data, m_dims, desc, scratch);
319 EIGEN_DEVICE_FUNC EvaluatorPointerType data()
const {
return m_data; }
322 EvaluatorPointerType m_data;
324 const Device EIGEN_DEVICE_REF m_device;
329template <
typename NullaryOp,
typename ArgType,
typename Device>
331 using XprType = TensorCwiseNullaryOp<NullaryOp, ArgType>;
333 EIGEN_DEVICE_FUNC TensorEvaluator(
const XprType& op,
const Device& device)
334 : m_functor(op.functor()), m_argImpl(op.nestedExpression(), device), m_wrapper() {}
336 using Index =
typename XprType::Index;
337 using Scalar =
typename XprType::Scalar;
338 using CoeffReturnType =
typename internal::traits<XprType>::Scalar;
339 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
340 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
341 using Dimensions =
typename TensorEvaluator<ArgType, Device>::Dimensions;
342 using Storage = StorageMemory<CoeffReturnType, Device>;
343 using EvaluatorPointerType =
typename Storage::Type;
344 static constexpr int NumDims = internal::array_size<Dimensions>::value;
345 using ScalarNoConst = std::remove_const_t<CoeffReturnType>;
352 static constexpr bool RepeatableFunctor = internal::tensor_functor_is_repeatable<NullaryOp>::value;
356 static constexpr bool IndexDependentFunctor =
357 internal::has_unary_operator<NullaryOp, Index>::value || internal::has_binary_operator<NullaryOp, Index>::value;
365 static constexpr bool IndexIndependentFunctor =
366 RepeatableFunctor && !IndexDependentFunctor && internal::has_nullary_operator<NullaryOp, Index>::value;
368 static constexpr int Layout = TensorEvaluator<ArgType, Device>::Layout;
371 PacketAccess = internal::functor_traits<NullaryOp>::PacketAccess
373 && (PacketType<CoeffReturnType, Device>::size > 1)
383 BlockAccess = NumDims > 0 && internal::is_arithmetic<ScalarNoConst>::value &&
384 (IndexIndependentFunctor || (IndexDependentFunctor && RepeatableFunctor)),
385 PreferBlockAccess =
false,
391 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
392 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
398 class TensorNullaryBlock {
400 using ArgXprType = TensorMap<const Tensor<ScalarNoConst, NumDims, Layout, Index>>;
401 using XprType = TensorCwiseNullaryOp<NullaryOp, const ArgXprType>;
402 using Scalar =
typename internal::XprScalar<XprType>::type;
404 TensorNullaryBlock(
const NullaryOp& functor,
const DSizes<Index, NumDims>& dimensions)
405 : m_functor(functor), m_dimensions(dimensions) {}
407 constexpr internal::TensorBlockKind kind()
const {
return internal::TensorBlockKind::kExpr; }
408 XprType expr()
const {
409 return XprType(ArgXprType(
static_cast<const ScalarNoConst*
>(
nullptr), m_dimensions), m_functor);
411 const Scalar* data()
const {
return nullptr; }
416 DSizes<Index, NumDims> m_dimensions;
419 using MaterializedTensorBlock = internal::TensorMaterializedBlock<ScalarNoConst, NumDims, Layout, Index>;
420 using TensorBlock = std::conditional_t<IndexIndependentFunctor, TensorNullaryBlock, MaterializedTensorBlock>;
423 EIGEN_DEVICE_FUNC
const Dimensions& dimensions()
const {
return m_argImpl.dimensions(); }
425 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(EvaluatorPointerType) {
return true; }
427#ifdef EIGEN_USE_THREADS
428 template <
typename EvalSubExprsCallback>
429 EIGEN_STRONG_INLINE
void evalSubExprsIfNeededAsync(EvaluatorPointerType, EvalSubExprsCallback done) {
434 EIGEN_STRONG_INLINE
void cleanup() {}
436 EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index)
const {
return m_wrapper(m_functor, index); }
438 template <
int LoadMode>
439 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index)
const {
440 return m_wrapper.template packetOp<PacketReturnType, Index>(m_functor, index);
443 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool vectorized)
const {
449 return TensorOpCost(0, 0, 0, vectorized, PacketType<CoeffReturnType, Device>::size);
452 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements()
const {
453 return internal::TensorBlockResourceRequirements::any();
459 template <
typename Self,
bool Vectorizable>
460 struct NullaryBlockFill {
461 static EIGEN_STRONG_INLINE
void Run(
const Self& self, ScalarNoConst* buffer, Index tensor_index, Index count) {
462 for (Index i = 0; i < count; ++i) {
463 buffer[i] = self.m_wrapper(self.m_functor, tensor_index + i);
468 template <
typename Self>
469 struct NullaryBlockFill<Self, true> {
470 static EIGEN_STRONG_INLINE
void Run(
const Self& self, ScalarNoConst* buffer, Index tensor_index, Index count) {
472 for (; i + PacketSize <= count; i += PacketSize) {
473 internal::pstoreu(buffer + i,
474 self.m_wrapper.template packetOp<PacketReturnType, Index>(self.m_functor, tensor_index + i));
476 for (; i < count; ++i) {
477 buffer[i] = self.m_wrapper(self.m_functor, tensor_index + i);
482 template <
typename Self,
bool IndexIndependent>
483 struct NullaryBlockImpl {
486 static EIGEN_STRONG_INLINE TensorBlock Run(
const Self& self, TensorBlockDesc& desc, TensorBlockScratch&) {
487 return TensorBlock(self.m_functor, desc.dimensions());
491 template <
typename Self>
492 struct NullaryBlockImpl<Self, false> {
498 static TensorBlock Run(
const Self& self, TensorBlockDesc& desc, TensorBlockScratch& scratch) {
499 constexpr bool is_col_major =
static_cast<int>(Layout) ==
static_cast<int>(ColMajor);
500 using Fill = NullaryBlockFill<Self, bool(PacketAccess)>;
502 if (desc.size() == 0) {
503 return TensorBlock(internal::TensorBlockKind::kView,
nullptr, desc.dimensions());
507 const Dimensions& dims = self.m_argImpl.dimensions();
508 array<Index, NumDims> tensor_strides;
509 EIGEN_IF_CONSTEXPR (is_col_major) {
510 tensor_strides[0] = 1;
511 for (
int i = 1; i < NumDims; ++i) tensor_strides[i] = tensor_strides[i - 1] * dims[i - 1];
513 tensor_strides[NumDims - 1] = 1;
514 for (
int i = NumDims - 2; i >= 0; --i) tensor_strides[i] = tensor_strides[i + 1] * dims[i + 1];
518 struct BlockIteratorState {
524 array<BlockIteratorState, NumDims> it;
525 for (
int i = 0; i < NumDims; ++i) {
526 const int dim = is_col_major ? i : NumDims - 1 - i;
527 const Index size = desc.dimension(dim);
528 const Index stride = tensor_strides[dim];
529 it[i] = {size, 0, stride, stride * (size - 1)};
531 eigen_assert(it[0].tensor_stride == 1);
533 const typename TensorBlock::Storage block_storage = TensorBlock::prepareStorage(desc, scratch);
534 ScalarNoConst* block_buffer = block_storage.data();
536 const Index inner_size = it[0].size;
537 Index tensor_index = desc.offset();
540 Fill::Run(self, block_buffer + offset, tensor_index, inner_size);
541 offset += inner_size;
545 for (; i < NumDims; ++i) {
546 if (++it[i].count < it[i].size) {
547 tensor_index += it[i].tensor_stride;
551 tensor_index -= it[i].tensor_span;
553 if (i == NumDims)
break;
556 return block_storage.AsTensorMaterializedBlock();
560 EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
561 bool =
false)
const {
562 using Self = TensorEvaluator<const TensorCwiseNullaryOp<NullaryOp, ArgType>, Device>;
563 return NullaryBlockImpl<Self, IndexIndependentFunctor>::Run(*
this, desc, scratch);
566 EIGEN_DEVICE_FUNC EvaluatorPointerType data()
const {
return nullptr; }
569 const NullaryOp m_functor;
570 TensorEvaluator<ArgType, Device> m_argImpl;
571 const internal::nullary_wrapper<CoeffReturnType, NullaryOp> m_wrapper;
576template <
typename UnaryOp,
typename ArgType,
typename Device>
578 using XprType = TensorCwiseUnaryOp<UnaryOp, ArgType>;
580 static constexpr int Layout = TensorEvaluator<ArgType, Device>::Layout;
582 IsAligned = TensorEvaluator<ArgType, Device>::IsAligned,
584 int(TensorEvaluator<ArgType, Device>::PacketAccess) & int(internal::functor_traits<UnaryOp>::PacketAccess),
585 BlockAccess = TensorEvaluator<ArgType, Device>::BlockAccess,
586 PreferBlockAccess = TensorEvaluator<ArgType, Device>::PreferBlockAccess,
591 EIGEN_DEVICE_FUNC TensorEvaluator(
const XprType& op,
const Device& device)
592 : m_device(device), m_functor(op.functor()), m_argImpl(op.nestedExpression(), device) {}
594 using Index =
typename XprType::Index;
595 using Scalar =
typename XprType::Scalar;
596 using ScalarNoConst = std::remove_const_t<Scalar>;
597 using CoeffReturnType =
typename internal::traits<XprType>::Scalar;
598 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
599 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
600 using Dimensions =
typename TensorEvaluator<ArgType, Device>::Dimensions;
601 using Storage = StorageMemory<CoeffReturnType, Device>;
602 using EvaluatorPointerType =
typename Storage::Type;
603 static constexpr int NumDims = internal::array_size<Dimensions>::value;
606 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
607 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
609 using ArgTensorBlock =
typename TensorEvaluator<const ArgType, Device>::TensorBlock;
611 using TensorBlock = internal::TensorCwiseUnaryBlock<UnaryOp, ArgTensorBlock>;
614 EIGEN_DEVICE_FUNC
const Dimensions& dimensions()
const {
return m_argImpl.dimensions(); }
616 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(EvaluatorPointerType) {
617 m_argImpl.evalSubExprsIfNeeded(
nullptr);
621#ifdef EIGEN_USE_THREADS
622 template <
typename EvalSubExprsCallback>
623 EIGEN_STRONG_INLINE
void evalSubExprsIfNeededAsync(EvaluatorPointerType, EvalSubExprsCallback done) {
624 m_argImpl.evalSubExprsIfNeededAsync(
nullptr, [done](
bool) { done(
true); });
628 EIGEN_STRONG_INLINE
void cleanup() { m_argImpl.cleanup(); }
630 EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index)
const {
return m_functor(m_argImpl.coeff(index)); }
632 template <
int LoadMode>
633 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index)
const {
634 return m_functor.packetOp(m_argImpl.template packet<LoadMode>(index));
637 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool vectorized)
const {
638 const double functor_cost = internal::functor_traits<UnaryOp>::Cost;
639 return m_argImpl.costPerCoeff(vectorized) + TensorOpCost(0, 0, functor_cost, vectorized, PacketSize);
642 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements()
const {
643 static constexpr double functor_cost = internal::functor_traits<UnaryOp>::Cost;
644 return m_argImpl.getResourceRequirements().addCostPerCoeff(
645 TensorOpCost(0, 0, functor_cost, PacketAccess, PacketSize));
648 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
649 bool =
false)
const {
657 constexpr bool ScalarTypesMatch =
658 std::is_same<std::remove_const_t<Scalar>,
659 std::remove_const_t<typename TensorEvaluator<ArgType, Device>::Scalar>>::value;
660 EIGEN_IF_CONSTEXPR (!ScalarTypesMatch) {
661 desc.DropDestinationBuffer();
663 return TensorBlock(m_argImpl.block(desc, scratch), m_functor);
666 EIGEN_DEVICE_FUNC EvaluatorPointerType data()
const {
return nullptr; }
669 const Device EIGEN_DEVICE_REF m_device;
670 const UnaryOp m_functor;
671 TensorEvaluator<ArgType, Device> m_argImpl;
676template <
typename BinaryOp,
typename LeftArgType,
typename RightArgType,
typename Device>
678 using XprType = TensorCwiseBinaryOp<BinaryOp, LeftArgType, RightArgType>;
680 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
683 int(TensorEvaluator<LeftArgType, Device>::IsAligned) & int(TensorEvaluator<RightArgType, Device>::IsAligned),
684 PacketAccess = int(TensorEvaluator<LeftArgType, Device>::PacketAccess) &
685 int(TensorEvaluator<RightArgType, Device>::PacketAccess) &
686 int(internal::functor_traits<BinaryOp>::PacketAccess),
687 BlockAccess = int(TensorEvaluator<LeftArgType, Device>::BlockAccess) &
688 int(TensorEvaluator<RightArgType, Device>::BlockAccess),
689 PreferBlockAccess = int(TensorEvaluator<LeftArgType, Device>::PreferBlockAccess) |
690 int(TensorEvaluator<RightArgType, Device>::PreferBlockAccess),
695 EIGEN_DEVICE_FUNC TensorEvaluator(
const XprType& op,
const Device& device)
697 m_functor(op.functor()),
698 m_leftImpl(op.lhsExpression(), device),
699 m_rightImpl(op.rhsExpression(), device) {
700 EIGEN_STATIC_ASSERT((
static_cast<int>(TensorEvaluator<LeftArgType, Device>::Layout) ==
701 static_cast<int>(TensorEvaluator<RightArgType, Device>::Layout) ||
702 internal::traits<XprType>::NumDimensions <= 1),
703 YOU_MADE_A_PROGRAMMING_MISTAKE);
704 EIGEN_INTERNAL_ASSERT_TENSOR_SHAPE(m_leftImpl, m_rightImpl);
707 using Index =
typename XprType::Index;
708 using Scalar =
typename XprType::Scalar;
709 using CoeffReturnType =
typename internal::traits<XprType>::Scalar;
710 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
711 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
712 using Dimensions =
typename TensorEvaluator<LeftArgType, Device>::Dimensions;
713 using Storage = StorageMemory<CoeffReturnType, Device>;
714 using EvaluatorPointerType =
typename Storage::Type;
716 static constexpr int NumDims = internal::array_size<typename TensorEvaluator<LeftArgType, Device>::Dimensions>::value;
719 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
720 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
722 using LeftTensorBlock =
typename TensorEvaluator<const LeftArgType, Device>::TensorBlock;
723 using RightTensorBlock =
typename TensorEvaluator<const RightArgType, Device>::TensorBlock;
725 using TensorBlock = internal::TensorCwiseBinaryBlock<BinaryOp, LeftTensorBlock, RightTensorBlock>;
728 EIGEN_DEVICE_FUNC
const Dimensions& dimensions()
const {
730 return m_leftImpl.dimensions();
733 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(EvaluatorPointerType) {
734 m_leftImpl.evalSubExprsIfNeeded(
nullptr);
736 EIGEN_TRY { m_rightImpl.evalSubExprsIfNeeded(
nullptr); }
738 m_leftImpl.cleanup();
744#ifdef EIGEN_USE_THREADS
745 template <
typename EvalSubExprsCallback>
746 EIGEN_STRONG_INLINE
void evalSubExprsIfNeededAsync(EvaluatorPointerType, EvalSubExprsCallback done) {
748 m_leftImpl.evalSubExprsIfNeededAsync(
749 nullptr, [
this, done](
bool) { m_rightImpl.evalSubExprsIfNeededAsync(
nullptr, [done](
bool) { done(
true); }); });
753 EIGEN_STRONG_INLINE
void cleanup() {
754 m_leftImpl.cleanup();
755 m_rightImpl.cleanup();
758 EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index)
const {
759 return m_functor(m_leftImpl.coeff(index), m_rightImpl.coeff(index));
761 template <
int LoadMode>
762 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index)
const {
763 return m_functor.packetOp(m_leftImpl.template packet<LoadMode>(index),
764 m_rightImpl.template packet<LoadMode>(index));
767 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool vectorized)
const {
768 const double functor_cost = internal::functor_traits<BinaryOp>::Cost;
769 return m_leftImpl.costPerCoeff(vectorized) + m_rightImpl.costPerCoeff(vectorized) +
770 TensorOpCost(0, 0, functor_cost, vectorized, PacketSize);
773 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements()
const {
774 static constexpr double functor_cost = internal::functor_traits<BinaryOp>::Cost;
775 return internal::TensorBlockResourceRequirements::merge(m_leftImpl.getResourceRequirements(),
776 m_rightImpl.getResourceRequirements())
777 .addCostPerCoeff(TensorOpCost(0, 0, functor_cost, PacketAccess, PacketSize));
780 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
781 bool =
false)
const {
782 desc.DropDestinationBuffer();
783 return TensorBlock(m_leftImpl.block(desc, scratch), m_rightImpl.block(desc, scratch), m_functor);
786 EIGEN_DEVICE_FUNC EvaluatorPointerType data()
const {
return nullptr; }
789 const Device EIGEN_DEVICE_REF m_device;
790 const BinaryOp m_functor;
791 TensorEvaluator<LeftArgType, Device> m_leftImpl;
792 TensorEvaluator<RightArgType, Device> m_rightImpl;
797template <
typename TernaryOp,
typename Arg1Type,
typename Arg2Type,
typename Arg3Type,
typename Device>
798struct TensorEvaluator<const TensorCwiseTernaryOp<TernaryOp, Arg1Type, Arg2Type, Arg3Type>, Device> {
799 using XprType = TensorCwiseTernaryOp<TernaryOp, Arg1Type, Arg2Type, Arg3Type>;
801 static constexpr int Layout = TensorEvaluator<Arg1Type, Device>::Layout;
803 IsAligned = TensorEvaluator<Arg1Type, Device>::IsAligned && TensorEvaluator<Arg2Type, Device>::IsAligned &&
804 TensorEvaluator<Arg3Type, Device>::IsAligned,
805 PacketAccess = TensorEvaluator<Arg1Type, Device>::PacketAccess && TensorEvaluator<Arg2Type, Device>::PacketAccess &&
806 TensorEvaluator<Arg3Type, Device>::PacketAccess && internal::functor_traits<TernaryOp>::PacketAccess,
807 BlockAccess = TensorEvaluator<Arg1Type, Device>::BlockAccess && TensorEvaluator<Arg2Type, Device>::BlockAccess &&
808 TensorEvaluator<Arg3Type, Device>::BlockAccess,
809 PreferBlockAccess = TensorEvaluator<Arg1Type, Device>::PreferBlockAccess ||
810 TensorEvaluator<Arg2Type, Device>::PreferBlockAccess ||
811 TensorEvaluator<Arg3Type, Device>::PreferBlockAccess,
816 EIGEN_DEVICE_FUNC TensorEvaluator(
const XprType& op,
const Device& device)
817 : m_functor(op.functor()),
818 m_arg1Impl(op.arg1Expression(), device),
819 m_arg2Impl(op.arg2Expression(), device),
820 m_arg3Impl(op.arg3Expression(), device) {
821 EIGEN_STATIC_ASSERT((
static_cast<int>(TensorEvaluator<Arg1Type, Device>::Layout) ==
822 static_cast<int>(TensorEvaluator<Arg3Type, Device>::Layout) ||
823 internal::traits<XprType>::NumDimensions <= 1),
824 YOU_MADE_A_PROGRAMMING_MISTAKE);
826 EIGEN_STATIC_ASSERT((std::is_same<
typename internal::traits<Arg1Type>::StorageKind,
827 typename internal::traits<Arg2Type>::StorageKind>::value),
828 STORAGE_KIND_MUST_MATCH)
829 EIGEN_STATIC_ASSERT((std::is_same<
typename internal::traits<Arg1Type>::StorageKind,
830 typename internal::traits<Arg3Type>::StorageKind>::value),
831 STORAGE_KIND_MUST_MATCH)
833 (std::is_same<
typename internal::traits<Arg1Type>::Index,
typename internal::traits<Arg2Type>::Index>::value),
834 STORAGE_INDEX_MUST_MATCH)
836 (std::is_same<
typename internal::traits<Arg1Type>::Index,
typename internal::traits<Arg3Type>::Index>::value),
837 STORAGE_INDEX_MUST_MATCH)
839 EIGEN_INTERNAL_ASSERT_TENSOR_SHAPE(m_arg1Impl, m_arg2Impl);
840 EIGEN_INTERNAL_ASSERT_TENSOR_SHAPE(m_arg1Impl, m_arg3Impl);
843 using Index =
typename XprType::Index;
844 using Scalar =
typename XprType::Scalar;
845 using CoeffReturnType =
typename internal::traits<XprType>::Scalar;
846 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
847 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
848 using Dimensions =
typename TensorEvaluator<Arg1Type, Device>::Dimensions;
849 using Storage = StorageMemory<CoeffReturnType, Device>;
850 using EvaluatorPointerType =
typename Storage::Type;
851 static constexpr int NumDims = internal::array_size<Dimensions>::value;
854 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
855 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
857 using Arg1TensorBlock =
typename TensorEvaluator<const Arg1Type, Device>::TensorBlock;
858 using Arg2TensorBlock =
typename TensorEvaluator<const Arg2Type, Device>::TensorBlock;
859 using Arg3TensorBlock =
typename TensorEvaluator<const Arg3Type, Device>::TensorBlock;
863 struct TensorCwiseTernaryOpBlockFactory {
866 template <
typename Arg1XprType,
typename Arg2XprType,
typename Arg3XprType>
868 using type = TensorCwiseTernaryOp<TernaryOp, const Arg1XprType, const Arg2XprType, const Arg3XprType>;
871 template <
typename Arg1XprType,
typename Arg2XprType,
typename Arg3XprType>
872 typename XprType<Arg1XprType, Arg2XprType, Arg3XprType>::type expr(
const Arg1XprType& arg1,
const Arg2XprType& arg2,
873 const Arg3XprType& arg3)
const {
874 return typename XprType<Arg1XprType, Arg2XprType, Arg3XprType>::type(arg1, arg2, arg3, func);
878 using TensorBlock = internal::TensorTernaryExprBlock<TensorCwiseTernaryOpBlockFactory, Arg1TensorBlock,
879 Arg2TensorBlock, Arg3TensorBlock>;
882 EIGEN_DEVICE_FUNC
const Dimensions& dimensions()
const {
884 return m_arg1Impl.dimensions();
887 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(EvaluatorPointerType) {
888 m_arg1Impl.evalSubExprsIfNeeded(
nullptr);
890 EIGEN_TRY { m_arg2Impl.evalSubExprsIfNeeded(
nullptr); }
892 m_arg1Impl.cleanup();
895 EIGEN_TRY { m_arg3Impl.evalSubExprsIfNeeded(
nullptr); }
897 m_arg1Impl.cleanup();
898 m_arg2Impl.cleanup();
903 EIGEN_STRONG_INLINE
void cleanup() {
904 m_arg1Impl.cleanup();
905 m_arg2Impl.cleanup();
906 m_arg3Impl.cleanup();
909 EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index)
const {
910 return m_functor(m_arg1Impl.coeff(index), m_arg2Impl.coeff(index), m_arg3Impl.coeff(index));
912 template <
int LoadMode>
913 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index)
const {
914 return m_functor.packetOp(m_arg1Impl.template packet<LoadMode>(index), m_arg2Impl.template packet<LoadMode>(index),
915 m_arg3Impl.template packet<LoadMode>(index));
918 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool vectorized)
const {
919 const double functor_cost = internal::functor_traits<TernaryOp>::Cost;
920 return m_arg1Impl.costPerCoeff(vectorized) + m_arg2Impl.costPerCoeff(vectorized) +
921 m_arg3Impl.costPerCoeff(vectorized) + TensorOpCost(0, 0, functor_cost, vectorized, PacketSize);
924 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements()
const {
925 static constexpr double functor_cost = internal::functor_traits<TernaryOp>::Cost;
926 return internal::TensorBlockResourceRequirements::merge(
927 m_arg1Impl.getResourceRequirements(),
928 internal::TensorBlockResourceRequirements::merge(m_arg2Impl.getResourceRequirements(),
929 m_arg3Impl.getResourceRequirements()))
930 .addCostPerCoeff(TensorOpCost(0, 0, functor_cost, PacketAccess, PacketSize));
933 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
934 bool =
false)
const {
937 desc.DropDestinationBuffer();
938 return TensorBlock(m_arg1Impl.block(desc, scratch), m_arg2Impl.block(desc, scratch),
939 m_arg3Impl.block(desc, scratch), TensorCwiseTernaryOpBlockFactory{m_functor});
942 EIGEN_DEVICE_FUNC EvaluatorPointerType data()
const {
return nullptr; }
945 const TernaryOp m_functor;
946 TensorEvaluator<Arg1Type, Device> m_arg1Impl;
947 TensorEvaluator<Arg2Type, Device> m_arg2Impl;
948 TensorEvaluator<Arg3Type, Device> m_arg3Impl;
953template <
typename IfArgType,
typename ThenArgType,
typename ElseArgType,
typename Device>
954struct TensorEvaluator<const TensorSelectOp<IfArgType, ThenArgType, ElseArgType>, Device> {
955 using XprType = TensorSelectOp<IfArgType, ThenArgType, ElseArgType>;
956 using Scalar =
typename XprType::Scalar;
958 using TernarySelectOp = internal::scalar_boolean_select_op<typename internal::traits<ThenArgType>::Scalar,
959 typename internal::traits<ElseArgType>::Scalar,
960 typename internal::traits<IfArgType>::Scalar>;
961 static constexpr bool TernaryPacketAccess =
962 TensorEvaluator<ThenArgType, Device>::PacketAccess && TensorEvaluator<ElseArgType, Device>::PacketAccess &&
963 TensorEvaluator<IfArgType, Device>::PacketAccess && internal::functor_traits<TernarySelectOp>::PacketAccess;
965 static constexpr int Layout = TensorEvaluator<IfArgType, Device>::Layout;
967 IsAligned = TensorEvaluator<ThenArgType, Device>::IsAligned && TensorEvaluator<ElseArgType, Device>::IsAligned,
969 (TensorEvaluator<ThenArgType, Device>::PacketAccess && TensorEvaluator<ElseArgType, Device>::PacketAccess) ||
971 BlockAccess = TensorEvaluator<IfArgType, Device>::BlockAccess &&
972 TensorEvaluator<ThenArgType, Device>::BlockAccess &&
973 TensorEvaluator<ElseArgType, Device>::BlockAccess,
974 PreferBlockAccess = TensorEvaluator<IfArgType, Device>::PreferBlockAccess ||
975 TensorEvaluator<ThenArgType, Device>::PreferBlockAccess ||
976 TensorEvaluator<ElseArgType, Device>::PreferBlockAccess,
981 EIGEN_DEVICE_FUNC TensorEvaluator(
const XprType& op,
const Device& device)
982 : m_condImpl(op.ifExpression(), device),
983 m_thenImpl(op.thenExpression(), device),
984 m_elseImpl(op.elseExpression(), device),
985 m_condSize(internal::array_prod(m_condImpl.dimensions())) {
986 EIGEN_STATIC_ASSERT((
static_cast<int>(TensorEvaluator<IfArgType, Device>::Layout) ==
987 static_cast<int>(TensorEvaluator<ThenArgType, Device>::Layout)),
988 YOU_MADE_A_PROGRAMMING_MISTAKE);
989 EIGEN_STATIC_ASSERT((
static_cast<int>(TensorEvaluator<IfArgType, Device>::Layout) ==
990 static_cast<int>(TensorEvaluator<ElseArgType, Device>::Layout)),
991 YOU_MADE_A_PROGRAMMING_MISTAKE);
992 EIGEN_INTERNAL_ASSERT_TENSOR_SHAPE(m_condImpl, m_thenImpl);
993 EIGEN_INTERNAL_ASSERT_TENSOR_SHAPE(m_thenImpl, m_elseImpl);
996 using Index =
typename XprType::Index;
997 using CoeffReturnType =
typename internal::traits<XprType>::Scalar;
998 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
999 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
1000 using CondScalar = std::remove_const_t<typename internal::traits<IfArgType>::Scalar>;
1001 using CondPacket =
typename PacketType<CondScalar, Device>::type;
1002 static constexpr int CondPacketSize = PacketType<CondScalar, Device>::size;
1003 static constexpr bool VectorizedCondCast =
1004 std::is_same<CondScalar, bool>::value && TensorEvaluator<IfArgType, Device>::PacketAccess &&
1005 internal::type_casting_traits<CondScalar, std::remove_const_t<Scalar>>::VectorizedCast;
1008 static constexpr int CondPacketMode = !VectorizedCondCast ? 0
1009 : CondPacketSize == PacketSize ? 1
1010 : PacketSize == 2 * CondPacketSize ? 2
1011 : CondPacketSize > PacketSize ? 3
1013 using Dimensions =
typename TensorEvaluator<IfArgType, Device>::Dimensions;
1014 using Storage = StorageMemory<CoeffReturnType, Device>;
1015 using EvaluatorPointerType =
typename Storage::Type;
1017 static constexpr int NumDims = internal::array_size<Dimensions>::value;
1020 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
1021 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
1023 using IfArgTensorBlock =
typename TensorEvaluator<const IfArgType, Device>::TensorBlock;
1024 using ThenArgTensorBlock =
typename TensorEvaluator<const ThenArgType, Device>::TensorBlock;
1025 using ElseArgTensorBlock =
typename TensorEvaluator<const ElseArgType, Device>::TensorBlock;
1027 struct TensorSelectOpBlockFactory {
1028 template <
typename IfArgXprType,
typename ThenArgXprType,
typename ElseArgXprType>
1030 using type = TensorSelectOp<const IfArgXprType, const ThenArgXprType, const ElseArgXprType>;
1033 template <
typename IfArgXprType,
typename ThenArgXprType,
typename ElseArgXprType>
1034 typename XprType<IfArgXprType, ThenArgXprType, ElseArgXprType>::type expr(
const IfArgXprType& if_expr,
1035 const ThenArgXprType& then_expr,
1036 const ElseArgXprType& else_expr)
const {
1037 return typename XprType<IfArgXprType, ThenArgXprType, ElseArgXprType>::type(if_expr, then_expr, else_expr);
1041 using TensorBlock = internal::TensorTernaryExprBlock<TensorSelectOpBlockFactory, IfArgTensorBlock, ThenArgTensorBlock,
1042 ElseArgTensorBlock>;
1045 EIGEN_DEVICE_FUNC
const Dimensions& dimensions()
const {
1047 return m_condImpl.dimensions();
1050 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(EvaluatorPointerType) {
1051 m_condImpl.evalSubExprsIfNeeded(
nullptr);
1053 EIGEN_TRY { m_thenImpl.evalSubExprsIfNeeded(
nullptr); }
1055 m_condImpl.cleanup();
1058 EIGEN_TRY { m_elseImpl.evalSubExprsIfNeeded(
nullptr); }
1060 m_condImpl.cleanup();
1061 m_thenImpl.cleanup();
1067#ifdef EIGEN_USE_THREADS
1068 template <
typename EvalSubExprsCallback>
1069 EIGEN_STRONG_INLINE
void evalSubExprsIfNeededAsync(EvaluatorPointerType, EvalSubExprsCallback done) {
1070 m_condImpl.evalSubExprsIfNeededAsync(
nullptr, [
this, done](
bool) {
1071 m_thenImpl.evalSubExprsIfNeededAsync(
1072 nullptr, [
this, done](
bool) { m_elseImpl.evalSubExprsIfNeededAsync(
nullptr, [done](
bool) { done(
true); }); });
1077 EIGEN_STRONG_INLINE
void cleanup() {
1078 m_condImpl.cleanup();
1079 m_thenImpl.cleanup();
1080 m_elseImpl.cleanup();
1083 EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index)
const {
1084 return m_condImpl.coeff(index) ? m_thenImpl.coeff(index) : m_elseImpl.coeff(index);
1090 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType condPacket(Index index)
const {
1091 return condPacket(index, std::integral_constant<int, CondPacketMode>());
1094 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType condPacket(Index index, std::integral_constant<int, 0>)
const {
1095 return scalarCondPacket(index);
1098 template <
typename Unused =
void>
1099 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType condPacket(Index index, std::integral_constant<int, 1>)
const {
1100 return internal::pcast<CondPacket, PacketReturnType>(m_condImpl.template packet<Unaligned>(index));
1103 template <
typename Unused =
void>
1104 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType condPacket(Index index, std::integral_constant<int, 2>)
const {
1105 return internal::pcast<CondPacket, PacketReturnType>(m_condImpl.template packet<Unaligned>(index),
1106 m_condImpl.template packet<Unaligned>(index + CondPacketSize));
1109 template <
typename Unused =
void>
1110 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType condPacket(Index index, std::integral_constant<int, 3>)
const {
1111 if (index + CondPacketSize <= m_condSize) {
1112 return internal::pcast<CondPacket, PacketReturnType>(m_condImpl.template packet<Unaligned>(index));
1114 return scalarCondPacket(index);
1117 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType scalarCondPacket(Index index)
const {
1118 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment)
1119 std::remove_const_t<Scalar> arr[PacketSize];
1121 for (Index i = 0; i < PacketSize; ++i) {
1122 arr[i] = m_condImpl.coeff(index + i) ? Scalar(
true) : Scalar(false);
1124 return internal::pload<PacketReturnType>(arr);
1127 template <
int LoadMode,
bool UseTernary = TernaryPacketAccess, std::enable_if_t<!UseTernary,
bool> = true>
1128 EIGEN_DEVICE_FUNC PacketReturnType packet(Index index)
const {
1129 return TernarySelectOp().template packetOp<PacketReturnType>(
1130 m_thenImpl.template packet<LoadMode>(index), m_elseImpl.template packet<LoadMode>(index), condPacket(index));
1133 template <
int LoadMode,
bool UseTernary = TernaryPacketAccess, std::enable_if_t<UseTernary,
bool> = true>
1134 EIGEN_DEVICE_FUNC PacketReturnType packet(Index index)
const {
1135 return TernarySelectOp().template packetOp<PacketReturnType>(m_thenImpl.template packet<LoadMode>(index),
1136 m_elseImpl.template packet<LoadMode>(index),
1137 m_condImpl.template packet<LoadMode>(index));
1140 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool vectorized)
const {
1141 const bool cond_vectorized = vectorized && (TernaryPacketAccess || CondPacketMode != 0);
1142 return m_condImpl.costPerCoeff(cond_vectorized) +
1143 (vectorized ? (m_thenImpl.costPerCoeff(vectorized) + m_elseImpl.costPerCoeff(vectorized) +
1144 TensorOpCost(0, 0, internal::functor_traits<TernarySelectOp>::Cost, vectorized, PacketSize))
1145 : m_thenImpl.costPerCoeff(vectorized).cwiseMax(m_elseImpl.costPerCoeff(vectorized)));
1148 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements()
const {
1149 auto then_req = m_thenImpl.getResourceRequirements();
1150 auto else_req = m_elseImpl.getResourceRequirements();
1152 auto merged_req = internal::TensorBlockResourceRequirements::merge(then_req, else_req);
1153 merged_req.cost_per_coeff =
1154 PacketAccess ? (then_req.cost_per_coeff + else_req.cost_per_coeff +
1155 TensorOpCost(0, 0, internal::functor_traits<TernarySelectOp>::Cost, PacketAccess, PacketSize))
1156 : then_req.cost_per_coeff.cwiseMax(else_req.cost_per_coeff);
1158 return internal::TensorBlockResourceRequirements::merge(m_condImpl.getResourceRequirements(), merged_req);
1161 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
1162 bool =
false)
const {
1165 desc.DropDestinationBuffer();
1167 return TensorBlock(m_condImpl.block(desc, scratch), m_thenImpl.block(desc, scratch),
1168 m_elseImpl.block(desc, scratch), TensorSelectOpBlockFactory());
1171 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EvaluatorPointerType data()
const {
return nullptr; }
1173#ifdef EIGEN_USE_SYCL
1175 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void bind(cl::sycl::handler& cgh)
const {
1176 m_condImpl.bind(cgh);
1177 m_thenImpl.bind(cgh);
1178 m_elseImpl.bind(cgh);
1182 TensorEvaluator<IfArgType, Device> m_condImpl;
1183 TensorEvaluator<ThenArgType, Device> m_thenImpl;
1184 TensorEvaluator<ElseArgType, Device> m_elseImpl;
1185 const Index m_condSize;
1190#if defined(EIGEN_USE_SYCL) && defined(SYCL_COMPILER_IS_DPCPP)
1191template <
typename Derived,
typename Device>
1192struct cl::sycl::is_device_copyable<
1193 Eigen::TensorEvaluator<Derived, Device>,
1194 std::enable_if_t<!std::is_trivially_copyable<Eigen::TensorEvaluator<Derived, Device>>::value>> : std::true_type {};
Tensor binary expression.
Definition TensorExpr.h:153
Tensor nullary expression.
Definition TensorExpr.h:41
Tensor unary expression.
Definition TensorExpr.h:90
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47