Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TensorEvaluator.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_TENSOR_TENSOR_EVALUATOR_H
12#define EIGEN_TENSOR_TENSOR_EVALUATOR_H
13
14// IWYU pragma: private
15#include "./InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21// Whether functor_traits declare the functor repeatable: pure calls that may
22// be re-evaluated freely and invoked from concurrent threads. The primary
23// functor_traits template defaults IsRepeatable to false, but specializations
24// for custom functors routinely omit the member, so absence must also map to
25// the conservative default.
26template <typename Functor, typename = void>
27struct tensor_functor_is_repeatable : std::false_type {};
28
29template <typename Functor>
30struct tensor_functor_is_repeatable<Functor, void_t<decltype(functor_traits<Functor>::IsRepeatable)>>
31 : bool_constant<functor_traits<Functor>::IsRepeatable> {};
32
33} // namespace internal
34
35// Generic evaluator
46template <typename Derived, typename Device>
47struct TensorEvaluator {
48 using Index = typename Derived::Index;
49 using Scalar = typename Derived::Scalar;
50 using CoeffReturnType = typename Derived::Scalar;
51 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
52 using Dimensions = typename Derived::Dimensions;
53 using XprType = Derived;
54 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
55 using TensorPointerType = typename internal::traits<Derived>::template MakePointer<Scalar>::Type;
56 using Storage = StorageMemory<Scalar, Device>;
57 using EvaluatorPointerType = typename Storage::Type;
58
59 // NumDimensions is -1 for variable dim tensors
60 static constexpr int NumCoords =
61 internal::traits<Derived>::NumDimensions > 0 ? internal::traits<Derived>::NumDimensions : 0;
62 static constexpr int Layout = Derived::Layout;
63
64 enum {
65 IsAligned = Derived::IsAligned,
66 PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1),
67 BlockAccess = internal::is_arithmetic<std::remove_const_t<Scalar>>::value,
68 PreferBlockAccess = false,
69 CoordAccess = NumCoords > 0,
70 RawAccess = true
71 };
72
73 using ScalarNoConst = std::remove_const_t<Scalar>;
74
75 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
76 using TensorBlockDesc = internal::TensorBlockDescriptor<NumCoords, Index>;
77 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
78
79 using TensorBlock = typename internal::TensorMaterializedBlock<ScalarNoConst, NumCoords, Layout, Index>;
80 //===--------------------------------------------------------------------===//
81
82 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const Derived& m, const Device& device)
83 : m_data(device.get(const_cast<TensorPointerType>(m.data()))), m_dims(m.dimensions()), m_device(device) {}
84
85 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dims; }
86
87 EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(EvaluatorPointerType dest) {
88 EIGEN_IF_CONSTEXPR (!NumTraits<std::remove_const_t<Scalar>>::RequireInitialization) {
89 if (dest) {
90 m_device.memcpy((void*)(m_device.get(dest)), m_device.get(m_data), m_dims.TotalSize() * sizeof(Scalar));
91 return false;
92 }
93 }
94 return true;
95 }
96
97#ifdef EIGEN_USE_THREADS
98 template <typename EvalSubExprsCallback>
99 EIGEN_STRONG_INLINE void evalSubExprsIfNeededAsync(EvaluatorPointerType dest, EvalSubExprsCallback done) {
100 // TODO(ezhulenev): ThreadPoolDevice memcpy is a blocking operation.
101 done(evalSubExprsIfNeeded(dest));
102 }
103#endif // EIGEN_USE_THREADS
104
105 EIGEN_STRONG_INLINE void cleanup() {}
106
107 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
108 eigen_assert(m_data != nullptr);
109 return m_data[index];
110 }
111
112 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index) const {
113 eigen_assert(m_data != nullptr);
114 return m_data[index];
115 }
116
117 template <int LoadMode>
118 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
119 return internal::ploadt<PacketReturnType, LoadMode>(m_data + index);
120 }
121
122 // Return a packet starting at `index` where `umask` specifies which elements
123 // have to be loaded. Type/size of mask depends on PacketReturnType, e.g. for
124 // Packet16f, `umask` is of type uint16_t and if a bit is 1, corresponding
125 // float element will be loaded, otherwise 0 will be loaded.
126 // Function has been templatized to enable Sfinae.
127 template <typename PacketReturnTypeT>
128 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
129 std::enable_if_t<internal::unpacket_traits<PacketReturnTypeT>::masked_load_available, PacketReturnTypeT>
130 partialPacket(Index index, typename internal::unpacket_traits<PacketReturnTypeT>::mask_t umask) const {
131 return internal::ploadu<PacketReturnTypeT>(m_data + index, umask);
132 }
133
134 template <int StoreMode>
135 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketReturnType& x) const {
136 return internal::pstoret<Scalar, PacketReturnType, StoreMode>(m_data + index, x);
137 }
138
139 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array<DenseIndex, NumCoords>& coords) const {
140 eigen_assert(m_data != nullptr);
141 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
142 return m_data[m_dims.IndexOfColMajor(coords)];
143 } else {
144 return m_data[m_dims.IndexOfRowMajor(coords)];
145 }
146 }
147
148 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(const array<DenseIndex, NumCoords>& coords) const {
149 eigen_assert(m_data != nullptr);
150 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
151 return m_data[m_dims.IndexOfColMajor(coords)];
152 } else {
153 return m_data[m_dims.IndexOfRowMajor(coords)];
154 }
155 }
156
157 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
158 return TensorOpCost(sizeof(CoeffReturnType), 0, 0, vectorized, PacketType<CoeffReturnType, Device>::size);
159 }
160
161 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
162 return internal::TensorBlockResourceRequirements::any();
163 }
164
165 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
166 bool /*root_of_expr_ast*/ = false) const {
167 eigen_assert(m_data != nullptr);
168 return TensorBlock::materialize(m_data, m_dims, desc, scratch);
169 }
170
171 template <typename TensorBlock>
172 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writeBlock(const TensorBlockDesc& desc, const TensorBlock& block) {
173 eigen_assert(m_data != nullptr);
174
175 using TensorBlockExpr = typename TensorBlock::XprType;
176 using TensorBlockAssign = internal::TensorBlockAssignment<Scalar, NumCoords, TensorBlockExpr, Index>;
177
178 TensorBlockAssign::Run(
179 TensorBlockAssign::target(desc.dimensions(), internal::strides<Layout>(m_dims), m_data, desc.offset()),
180 block.expr());
181 }
182
183 EIGEN_DEVICE_FUNC EvaluatorPointerType data() const { return m_data; }
184
185 protected:
186 EvaluatorPointerType m_data;
187 Dimensions m_dims;
188 const Device EIGEN_DEVICE_REF m_device;
189};
190
191namespace internal {
192template <typename T>
193EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T loadConstant(const T* address) {
194 return *address;
195}
196// Use the texture cache on CUDA devices whenever possible
197#if defined(EIGEN_CUDA_ARCH)
198template <>
199EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float loadConstant(const float* address) {
200 return __ldg(address);
201}
202template <>
203EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double loadConstant(const double* address) {
204 return __ldg(address);
205}
206template <>
207EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Eigen::half loadConstant(const Eigen::half* address) {
208 return Eigen::half(half_impl::raw_uint16_to_half(__ldg(&address->x)));
209}
210#endif
211
212} // namespace internal
213
214// Default evaluator for rvalues
215template <typename Derived, typename Device>
216struct TensorEvaluator<const Derived, Device> {
217 using Index = typename Derived::Index;
218 using Scalar = typename Derived::Scalar;
219 using CoeffReturnType = typename Derived::Scalar;
220 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
221 using Dimensions = typename Derived::Dimensions;
222 using XprType = const Derived;
223 using TensorPointerType = typename internal::traits<Derived>::template MakePointer<const Scalar>::Type;
224 using Storage = StorageMemory<const Scalar, Device>;
225 using EvaluatorPointerType = typename Storage::Type;
226
227 using ScalarNoConst = std::remove_const_t<Scalar>;
228
229 // NumDimensions is -1 for variable dim tensors
230 static constexpr int NumCoords =
231 internal::traits<Derived>::NumDimensions > 0 ? internal::traits<Derived>::NumDimensions : 0;
232 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
233 static constexpr int Layout = Derived::Layout;
234
235 enum {
236 IsAligned = Derived::IsAligned,
237 PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1),
238 BlockAccess = internal::is_arithmetic<ScalarNoConst>::value,
239 PreferBlockAccess = false,
240 CoordAccess = NumCoords > 0,
241 RawAccess = true
242 };
243
244 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
245 using TensorBlockDesc = internal::TensorBlockDescriptor<NumCoords, Index>;
246 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
247
248 using TensorBlock = typename internal::TensorMaterializedBlock<ScalarNoConst, NumCoords, Layout, Index>;
249 //===--------------------------------------------------------------------===//
250
251 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC TensorEvaluator(const Derived& m, const Device& device)
252 : m_data(device.get(m.data())), m_dims(m.dimensions()), m_device(device) {}
253
254 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dims; }
255
256 EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(EvaluatorPointerType data) {
257 EIGEN_IF_CONSTEXPR (!NumTraits<std::remove_const_t<Scalar>>::RequireInitialization) {
258 if (data) {
259 m_device.memcpy((void*)(m_device.get(data)), m_device.get(m_data), m_dims.TotalSize() * sizeof(Scalar));
260 return false;
261 }
262 }
263 return true;
264 }
265
266#ifdef EIGEN_USE_THREADS
267 template <typename EvalSubExprsCallback>
268 EIGEN_STRONG_INLINE void evalSubExprsIfNeededAsync(EvaluatorPointerType dest, EvalSubExprsCallback done) {
269 // TODO(ezhulenev): ThreadPoolDevice memcpy is a blocking operation.
270 done(evalSubExprsIfNeeded(dest));
271 }
272#endif // EIGEN_USE_THREADS
273
274 EIGEN_STRONG_INLINE void cleanup() {}
275
276 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
277 eigen_assert(m_data != nullptr);
278 return internal::loadConstant(m_data + index);
279 }
280
281 template <int LoadMode>
282 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
283 return internal::ploadt_ro<PacketReturnType, LoadMode>(m_data + index);
284 }
285
286 // Return a packet starting at `index` where `umask` specifies which elements
287 // have to be loaded. Type/size of mask depends on PacketReturnType, e.g. for
288 // Packet16f, `umask` is of type uint16_t and if a bit is 1, corresponding
289 // float element will be loaded, otherwise 0 will be loaded.
290 // Function has been templatized to enable Sfinae.
291 template <typename PacketReturnTypeT>
292 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
293 std::enable_if_t<internal::unpacket_traits<PacketReturnTypeT>::masked_load_available, PacketReturnTypeT>
294 partialPacket(Index index, typename internal::unpacket_traits<PacketReturnTypeT>::mask_t umask) const {
295 return internal::ploadu<PacketReturnTypeT>(m_data + index, umask);
296 }
297
298 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array<DenseIndex, NumCoords>& coords) const {
299 eigen_assert(m_data != nullptr);
300 const Index index = (static_cast<int>(Layout) == static_cast<int>(ColMajor)) ? m_dims.IndexOfColMajor(coords)
301 : m_dims.IndexOfRowMajor(coords);
302 return internal::loadConstant(m_data + index);
303 }
304
305 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
306 return TensorOpCost(sizeof(CoeffReturnType), 0, 0, vectorized, PacketType<CoeffReturnType, Device>::size);
307 }
308
309 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
310 return internal::TensorBlockResourceRequirements::any();
311 }
312
313 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
314 bool /*root_of_expr_ast*/ = false) const {
315 eigen_assert(m_data != nullptr);
316 return TensorBlock::materialize(m_data, m_dims, desc, scratch);
317 }
318
319 EIGEN_DEVICE_FUNC EvaluatorPointerType data() const { return m_data; }
320
321 protected:
322 EvaluatorPointerType m_data;
323 Dimensions m_dims;
324 const Device EIGEN_DEVICE_REF m_device;
325};
326
327// -------------------- CwiseNullaryOp --------------------
328
329template <typename NullaryOp, typename ArgType, typename Device>
330struct TensorEvaluator<const TensorCwiseNullaryOp<NullaryOp, ArgType>, Device> {
331 using XprType = TensorCwiseNullaryOp<NullaryOp, ArgType>;
332
333 EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device)
334 : m_functor(op.functor()), m_argImpl(op.nestedExpression(), device), m_wrapper() {}
335
336 using Index = typename XprType::Index;
337 using Scalar = typename XprType::Scalar;
338 using CoeffReturnType = typename internal::traits<XprType>::Scalar;
339 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
340 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
341 using Dimensions = typename TensorEvaluator<ArgType, Device>::Dimensions;
342 using Storage = StorageMemory<CoeffReturnType, Device>;
343 using EvaluatorPointerType = typename Storage::Type;
344 static constexpr int NumDims = internal::array_size<Dimensions>::value;
345 using ScalarNoConst = std::remove_const_t<CoeffReturnType>;
346
347 // Only functors whose functor_traits declare IsRepeatable (e.g. the
348 // scalar_constant_op behind constant(), cwiseMax(Scalar) and clip(), and the
349 // index-seeded random generators) are known to be pure; unannotated custom
350 // functors conservatively default to non-repeatable, since a functor that
351 // carries state between calls cannot be re-evaluated or visited out of order.
352 static constexpr bool RepeatableFunctor = internal::tensor_functor_is_repeatable<NullaryOp>::value;
353
354 // nullary_wrapper dispatches to an indexed operator() whenever one exists,
355 // even if a zero-argument overload is also present.
356 static constexpr bool IndexDependentFunctor =
357 internal::has_unary_operator<NullaryOp, Index>::value || internal::has_binary_operator<NullaryOp, Index>::value;
358
359 // A lazy block rebuilds the nullary expression over the block's local
360 // extent with a copy of the functor, so it is only correct for repeatable
361 // functors evaluated through the zero-argument overload: an index-dependent
362 // functor would see indices restart at the block origin, and a
363 // non-repeatable one would restart its state per block. Repeatable indexed
364 // functors are instead materialized with their true tensor-linear indices.
365 static constexpr bool IndexIndependentFunctor =
366 RepeatableFunctor && !IndexDependentFunctor && internal::has_nullary_operator<NullaryOp, Index>::value;
367
368 static constexpr int Layout = TensorEvaluator<ArgType, Device>::Layout;
369 enum {
370 IsAligned = true,
371 PacketAccess = internal::functor_traits<NullaryOp>::PacketAccess
372#ifdef EIGEN_USE_SYCL
373 && (PacketType<CoeffReturnType, Device>::size > 1)
374#endif
375 ,
376 // A nullary leaf can serve any block; without this, a single constant()
377 // in an expression disables tiled evaluation for the whole tree. Never
378 // *prefer* block access on its own account, though. Blocks are declined
379 // when they could change behavior relative to coefficient evaluation. In
380 // particular, non-repeatable functors never serve blocks: even on a
381 // single-threaded device, block traversal permutes their call sequence
382 // relative to linear coefficient order.
383 BlockAccess = NumDims > 0 && internal::is_arithmetic<ScalarNoConst>::value &&
384 (IndexIndependentFunctor || (IndexDependentFunctor && RepeatableFunctor)),
385 PreferBlockAccess = false,
386 CoordAccess = false, // to be implemented
387 RawAccess = false
388 };
389
390 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
391 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
392 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
393
394 // Lazy block for index-independent functors: rebuilds the nullary
395 // expression over the block extent. The nested map exists only to give the
396 // expression its dimensions; the nullary evaluator never touches its
397 // argument's data.
398 class TensorNullaryBlock {
399 public:
400 using ArgXprType = TensorMap<const Tensor<ScalarNoConst, NumDims, Layout, Index>>;
401 using XprType = TensorCwiseNullaryOp<NullaryOp, const ArgXprType>;
402 using Scalar = typename internal::XprScalar<XprType>::type;
403
404 TensorNullaryBlock(const NullaryOp& functor, const DSizes<Index, NumDims>& dimensions)
405 : m_functor(functor), m_dimensions(dimensions) {}
406
407 constexpr internal::TensorBlockKind kind() const { return internal::TensorBlockKind::kExpr; }
408 XprType expr() const {
409 return XprType(ArgXprType(static_cast<const ScalarNoConst*>(nullptr), m_dimensions), m_functor);
410 }
411 const Scalar* data() const { return nullptr; }
412 void cleanup() {}
413
414 private:
415 NullaryOp m_functor;
416 DSizes<Index, NumDims> m_dimensions;
417 };
418
419 using MaterializedTensorBlock = internal::TensorMaterializedBlock<ScalarNoConst, NumDims, Layout, Index>;
420 using TensorBlock = std::conditional_t<IndexIndependentFunctor, TensorNullaryBlock, MaterializedTensorBlock>;
421 //===--------------------------------------------------------------------===//
422
423 EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_argImpl.dimensions(); }
424
425 EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(EvaluatorPointerType) { return true; }
426
427#ifdef EIGEN_USE_THREADS
428 template <typename EvalSubExprsCallback>
429 EIGEN_STRONG_INLINE void evalSubExprsIfNeededAsync(EvaluatorPointerType, EvalSubExprsCallback done) {
430 done(true);
431 }
432#endif // EIGEN_USE_THREADS
433
434 EIGEN_STRONG_INLINE void cleanup() {}
435
436 EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const { return m_wrapper(m_functor, index); }
437
438 template <int LoadMode>
439 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
440 return m_wrapper.template packetOp<PacketReturnType, Index>(m_functor, index);
441 }
442
443 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
444 // NullaryOps (constants, zero, identity, random) generate values from
445 // registers or minimal state — they do not load from memory. Report
446 // zero bytes_loaded so the cost model correctly classifies expressions
447 // containing many constants (e.g. Horner polynomials) as compute-bound
448 // rather than memory-bound.
449 return TensorOpCost(0, 0, 0, vectorized, PacketType<CoeffReturnType, Device>::size);
450 }
451
452 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
453 return internal::TensorBlockResourceRequirements::any();
454 }
455
456 // Fills `count` elements of an inner run, whose tensor-linear indices are
457 // tensor_index .. tensor_index + count - 1. Specialized on the functor's
458 // packet support so that packetOp is only instantiated when it exists.
459 template <typename Self, bool Vectorizable>
460 struct NullaryBlockFill {
461 static EIGEN_STRONG_INLINE void Run(const Self& self, ScalarNoConst* buffer, Index tensor_index, Index count) {
462 for (Index i = 0; i < count; ++i) {
463 buffer[i] = self.m_wrapper(self.m_functor, tensor_index + i);
464 }
465 }
466 };
467
468 template <typename Self>
469 struct NullaryBlockFill<Self, true> {
470 static EIGEN_STRONG_INLINE void Run(const Self& self, ScalarNoConst* buffer, Index tensor_index, Index count) {
471 Index i = 0;
472 for (; i + PacketSize <= count; i += PacketSize) {
473 internal::pstoreu(buffer + i,
474 self.m_wrapper.template packetOp<PacketReturnType, Index>(self.m_functor, tensor_index + i));
475 }
476 for (; i < count; ++i) {
477 buffer[i] = self.m_wrapper(self.m_functor, tensor_index + i);
478 }
479 }
480 };
481
482 template <typename Self, bool IndexIndependent>
483 struct NullaryBlockImpl {
484 // Index-independent functors: return a lazy block; the consumer folds the
485 // functor into its own (vectorized) evaluation loop.
486 static EIGEN_STRONG_INLINE TensorBlock Run(const Self& self, TensorBlockDesc& desc, TensorBlockScratch&) {
487 return TensorBlock(self.m_functor, desc.dimensions());
488 }
489 };
490
491 template <typename Self>
492 struct NullaryBlockImpl<Self, false> {
493 // Index-dependent functors: materialize the block by calling the functor
494 // with the block elements' true tensor-linear indices, so the values
495 // match linear evaluation semantics. Along the inner dimension a block
496 // row is contiguous in linear-index space, so each run is one
497 // packet-sized sweep.
498 static TensorBlock Run(const Self& self, TensorBlockDesc& desc, TensorBlockScratch& scratch) {
499 constexpr bool is_col_major = static_cast<int>(Layout) == static_cast<int>(ColMajor);
500 using Fill = NullaryBlockFill<Self, bool(PacketAccess)>;
501
502 if (desc.size() == 0) {
503 return TensorBlock(internal::TensorBlockKind::kView, nullptr, desc.dimensions());
504 }
505
506 // Strides of the full tensor in linear-index space.
507 const Dimensions& dims = self.m_argImpl.dimensions();
508 array<Index, NumDims> tensor_strides;
509 EIGEN_IF_CONSTEXPR (is_col_major) {
510 tensor_strides[0] = 1;
511 for (int i = 1; i < NumDims; ++i) tensor_strides[i] = tensor_strides[i - 1] * dims[i - 1];
512 } else {
513 tensor_strides[NumDims - 1] = 1;
514 for (int i = NumDims - 2; i >= 0; --i) tensor_strides[i] = tensor_strides[i + 1] * dims[i + 1];
515 }
516
517 // Block iteration state, inner-most dimension first.
518 struct BlockIteratorState {
519 Index size;
520 Index count;
521 Index tensor_stride;
522 Index tensor_span;
523 };
524 array<BlockIteratorState, NumDims> it;
525 for (int i = 0; i < NumDims; ++i) {
526 const int dim = is_col_major ? i : NumDims - 1 - i;
527 const Index size = desc.dimension(dim);
528 const Index stride = tensor_strides[dim];
529 it[i] = {/*size=*/size, /*count=*/0, /*tensor_stride=*/stride, /*tensor_span=*/stride * (size - 1)};
530 }
531 eigen_assert(it[0].tensor_stride == 1);
532
533 const typename TensorBlock::Storage block_storage = TensorBlock::prepareStorage(desc, scratch);
534 ScalarNoConst* block_buffer = block_storage.data();
535
536 const Index inner_size = it[0].size;
537 Index tensor_index = desc.offset();
538 Index offset = 0;
539 for (;;) {
540 Fill::Run(self, block_buffer + offset, tensor_index, inner_size);
541 offset += inner_size;
542
543 // Advance the odometer over the outer dimensions.
544 int i = 1;
545 for (; i < NumDims; ++i) {
546 if (++it[i].count < it[i].size) {
547 tensor_index += it[i].tensor_stride;
548 break;
549 }
550 it[i].count = 0;
551 tensor_index -= it[i].tensor_span;
552 }
553 if (i == NumDims) break;
554 }
555
556 return block_storage.AsTensorMaterializedBlock();
557 }
558 };
559
560 EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
561 bool /*root_of_expr_ast*/ = false) const {
562 using Self = TensorEvaluator<const TensorCwiseNullaryOp<NullaryOp, ArgType>, Device>;
563 return NullaryBlockImpl<Self, IndexIndependentFunctor>::Run(*this, desc, scratch);
564 }
565
566 EIGEN_DEVICE_FUNC EvaluatorPointerType data() const { return nullptr; }
567
568 private:
569 const NullaryOp m_functor;
570 TensorEvaluator<ArgType, Device> m_argImpl;
571 const internal::nullary_wrapper<CoeffReturnType, NullaryOp> m_wrapper;
572};
573
574// -------------------- CwiseUnaryOp --------------------
575
576template <typename UnaryOp, typename ArgType, typename Device>
577struct TensorEvaluator<const TensorCwiseUnaryOp<UnaryOp, ArgType>, Device> {
578 using XprType = TensorCwiseUnaryOp<UnaryOp, ArgType>;
579
580 static constexpr int Layout = TensorEvaluator<ArgType, Device>::Layout;
581 enum {
582 IsAligned = TensorEvaluator<ArgType, Device>::IsAligned,
583 PacketAccess =
584 int(TensorEvaluator<ArgType, Device>::PacketAccess) & int(internal::functor_traits<UnaryOp>::PacketAccess),
585 BlockAccess = TensorEvaluator<ArgType, Device>::BlockAccess,
586 PreferBlockAccess = TensorEvaluator<ArgType, Device>::PreferBlockAccess,
587 CoordAccess = false, // to be implemented
588 RawAccess = false
589 };
590
591 EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device)
592 : m_device(device), m_functor(op.functor()), m_argImpl(op.nestedExpression(), device) {}
593
594 using Index = typename XprType::Index;
595 using Scalar = typename XprType::Scalar;
596 using ScalarNoConst = std::remove_const_t<Scalar>;
597 using CoeffReturnType = typename internal::traits<XprType>::Scalar;
598 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
599 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
600 using Dimensions = typename TensorEvaluator<ArgType, Device>::Dimensions;
601 using Storage = StorageMemory<CoeffReturnType, Device>;
602 using EvaluatorPointerType = typename Storage::Type;
603 static constexpr int NumDims = internal::array_size<Dimensions>::value;
604
605 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
606 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
607 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
608
609 using ArgTensorBlock = typename TensorEvaluator<const ArgType, Device>::TensorBlock;
610
611 using TensorBlock = internal::TensorCwiseUnaryBlock<UnaryOp, ArgTensorBlock>;
612 //===--------------------------------------------------------------------===//
613
614 EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_argImpl.dimensions(); }
615
616 EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(EvaluatorPointerType) {
617 m_argImpl.evalSubExprsIfNeeded(nullptr);
618 return true;
619 }
620
621#ifdef EIGEN_USE_THREADS
622 template <typename EvalSubExprsCallback>
623 EIGEN_STRONG_INLINE void evalSubExprsIfNeededAsync(EvaluatorPointerType, EvalSubExprsCallback done) {
624 m_argImpl.evalSubExprsIfNeededAsync(nullptr, [done](bool) { done(true); });
625 }
626#endif // EIGEN_USE_THREADS
627
628 EIGEN_STRONG_INLINE void cleanup() { m_argImpl.cleanup(); }
629
630 EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const { return m_functor(m_argImpl.coeff(index)); }
631
632 template <int LoadMode>
633 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
634 return m_functor.packetOp(m_argImpl.template packet<LoadMode>(index));
635 }
636
637 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
638 const double functor_cost = internal::functor_traits<UnaryOp>::Cost;
639 return m_argImpl.costPerCoeff(vectorized) + TensorOpCost(0, 0, functor_cost, vectorized, PacketSize);
640 }
641
642 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
643 static constexpr double functor_cost = internal::functor_traits<UnaryOp>::Cost;
644 return m_argImpl.getResourceRequirements().addCostPerCoeff(
645 TensorOpCost(0, 0, functor_cost, PacketAccess, PacketSize));
646 }
647
648 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
649 bool /*root_of_expr_ast*/ = false) const {
650 // The forwarded destination buffer is sized for *our* output Scalar (the
651 // assign LHS), but the child block evaluator below us writes ArgScalar.
652 // For functors that change the scalar type (abs(complex)->real,
653 // isnan/isfinite->bool, ...) the buffer would be misinterpreted by any
654 // block-materializing child's prepareStorage (assert in debug, corruption
655 // in release). Drop the buffer in that case; the child falls back to
656 // scratch and writeBlock still lands the converted values in the LHS.
657 constexpr bool ScalarTypesMatch =
658 std::is_same<std::remove_const_t<Scalar>,
659 std::remove_const_t<typename TensorEvaluator<ArgType, Device>::Scalar>>::value;
660 EIGEN_IF_CONSTEXPR (!ScalarTypesMatch) {
661 desc.DropDestinationBuffer();
662 }
663 return TensorBlock(m_argImpl.block(desc, scratch), m_functor);
664 }
665
666 EIGEN_DEVICE_FUNC EvaluatorPointerType data() const { return nullptr; }
667
668 private:
669 const Device EIGEN_DEVICE_REF m_device;
670 const UnaryOp m_functor;
671 TensorEvaluator<ArgType, Device> m_argImpl;
672};
673
674// -------------------- CwiseBinaryOp --------------------
675
676template <typename BinaryOp, typename LeftArgType, typename RightArgType, typename Device>
677struct TensorEvaluator<const TensorCwiseBinaryOp<BinaryOp, LeftArgType, RightArgType>, Device> {
678 using XprType = TensorCwiseBinaryOp<BinaryOp, LeftArgType, RightArgType>;
679
680 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
681 enum {
682 IsAligned =
683 int(TensorEvaluator<LeftArgType, Device>::IsAligned) & int(TensorEvaluator<RightArgType, Device>::IsAligned),
684 PacketAccess = int(TensorEvaluator<LeftArgType, Device>::PacketAccess) &
685 int(TensorEvaluator<RightArgType, Device>::PacketAccess) &
686 int(internal::functor_traits<BinaryOp>::PacketAccess),
687 BlockAccess = int(TensorEvaluator<LeftArgType, Device>::BlockAccess) &
688 int(TensorEvaluator<RightArgType, Device>::BlockAccess),
689 PreferBlockAccess = int(TensorEvaluator<LeftArgType, Device>::PreferBlockAccess) |
690 int(TensorEvaluator<RightArgType, Device>::PreferBlockAccess),
691 CoordAccess = false, // to be implemented
692 RawAccess = false
693 };
694
695 EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device)
696 : m_device(device),
697 m_functor(op.functor()),
698 m_leftImpl(op.lhsExpression(), device),
699 m_rightImpl(op.rhsExpression(), device) {
700 EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<LeftArgType, Device>::Layout) ==
701 static_cast<int>(TensorEvaluator<RightArgType, Device>::Layout) ||
702 internal::traits<XprType>::NumDimensions <= 1),
703 YOU_MADE_A_PROGRAMMING_MISTAKE);
704 EIGEN_INTERNAL_ASSERT_TENSOR_SHAPE(m_leftImpl, m_rightImpl);
705 }
706
707 using Index = typename XprType::Index;
708 using Scalar = typename XprType::Scalar;
709 using CoeffReturnType = typename internal::traits<XprType>::Scalar;
710 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
711 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
712 using Dimensions = typename TensorEvaluator<LeftArgType, Device>::Dimensions;
713 using Storage = StorageMemory<CoeffReturnType, Device>;
714 using EvaluatorPointerType = typename Storage::Type;
715
716 static constexpr int NumDims = internal::array_size<typename TensorEvaluator<LeftArgType, Device>::Dimensions>::value;
717
718 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
719 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
720 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
721
722 using LeftTensorBlock = typename TensorEvaluator<const LeftArgType, Device>::TensorBlock;
723 using RightTensorBlock = typename TensorEvaluator<const RightArgType, Device>::TensorBlock;
724
725 using TensorBlock = internal::TensorCwiseBinaryBlock<BinaryOp, LeftTensorBlock, RightTensorBlock>;
726 //===--------------------------------------------------------------------===//
727
728 EIGEN_DEVICE_FUNC const Dimensions& dimensions() const {
729 // TODO: use right impl instead if right impl dimensions are known at compile time.
730 return m_leftImpl.dimensions();
731 }
732
733 EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(EvaluatorPointerType) {
734 m_leftImpl.evalSubExprsIfNeeded(nullptr);
735 // The executor does not call cleanup() when evaluation throws, so release the operand evaluated first.
736 EIGEN_TRY { m_rightImpl.evalSubExprsIfNeeded(nullptr); }
737 EIGEN_CATCH(...) {
738 m_leftImpl.cleanup();
739 EIGEN_THROW;
740 }
741 return true;
742 }
743
744#ifdef EIGEN_USE_THREADS
745 template <typename EvalSubExprsCallback>
746 EIGEN_STRONG_INLINE void evalSubExprsIfNeededAsync(EvaluatorPointerType, EvalSubExprsCallback done) {
747 // TODO(ezhulenev): Evaluate two expression in parallel?
748 m_leftImpl.evalSubExprsIfNeededAsync(
749 nullptr, [this, done](bool) { m_rightImpl.evalSubExprsIfNeededAsync(nullptr, [done](bool) { done(true); }); });
750 }
751#endif // EIGEN_USE_THREADS
752
753 EIGEN_STRONG_INLINE void cleanup() {
754 m_leftImpl.cleanup();
755 m_rightImpl.cleanup();
756 }
757
758 EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const {
759 return m_functor(m_leftImpl.coeff(index), m_rightImpl.coeff(index));
760 }
761 template <int LoadMode>
762 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
763 return m_functor.packetOp(m_leftImpl.template packet<LoadMode>(index),
764 m_rightImpl.template packet<LoadMode>(index));
765 }
766
767 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
768 const double functor_cost = internal::functor_traits<BinaryOp>::Cost;
769 return m_leftImpl.costPerCoeff(vectorized) + m_rightImpl.costPerCoeff(vectorized) +
770 TensorOpCost(0, 0, functor_cost, vectorized, PacketSize);
771 }
772
773 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
774 static constexpr double functor_cost = internal::functor_traits<BinaryOp>::Cost;
775 return internal::TensorBlockResourceRequirements::merge(m_leftImpl.getResourceRequirements(),
776 m_rightImpl.getResourceRequirements())
777 .addCostPerCoeff(TensorOpCost(0, 0, functor_cost, PacketAccess, PacketSize));
778 }
779
780 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
781 bool /*root_of_expr_ast*/ = false) const {
782 desc.DropDestinationBuffer();
783 return TensorBlock(m_leftImpl.block(desc, scratch), m_rightImpl.block(desc, scratch), m_functor);
784 }
785
786 EIGEN_DEVICE_FUNC EvaluatorPointerType data() const { return nullptr; }
787
788 private:
789 const Device EIGEN_DEVICE_REF m_device;
790 const BinaryOp m_functor;
791 TensorEvaluator<LeftArgType, Device> m_leftImpl;
792 TensorEvaluator<RightArgType, Device> m_rightImpl;
793};
794
795// -------------------- CwiseTernaryOp --------------------
796
797template <typename TernaryOp, typename Arg1Type, typename Arg2Type, typename Arg3Type, typename Device>
798struct TensorEvaluator<const TensorCwiseTernaryOp<TernaryOp, Arg1Type, Arg2Type, Arg3Type>, Device> {
799 using XprType = TensorCwiseTernaryOp<TernaryOp, Arg1Type, Arg2Type, Arg3Type>;
800
801 static constexpr int Layout = TensorEvaluator<Arg1Type, Device>::Layout;
802 enum {
803 IsAligned = TensorEvaluator<Arg1Type, Device>::IsAligned && TensorEvaluator<Arg2Type, Device>::IsAligned &&
804 TensorEvaluator<Arg3Type, Device>::IsAligned,
805 PacketAccess = TensorEvaluator<Arg1Type, Device>::PacketAccess && TensorEvaluator<Arg2Type, Device>::PacketAccess &&
806 TensorEvaluator<Arg3Type, Device>::PacketAccess && internal::functor_traits<TernaryOp>::PacketAccess,
807 BlockAccess = TensorEvaluator<Arg1Type, Device>::BlockAccess && TensorEvaluator<Arg2Type, Device>::BlockAccess &&
808 TensorEvaluator<Arg3Type, Device>::BlockAccess,
809 PreferBlockAccess = TensorEvaluator<Arg1Type, Device>::PreferBlockAccess ||
810 TensorEvaluator<Arg2Type, Device>::PreferBlockAccess ||
811 TensorEvaluator<Arg3Type, Device>::PreferBlockAccess,
812 CoordAccess = false, // to be implemented
813 RawAccess = false
814 };
815
816 EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device)
817 : m_functor(op.functor()),
818 m_arg1Impl(op.arg1Expression(), device),
819 m_arg2Impl(op.arg2Expression(), device),
820 m_arg3Impl(op.arg3Expression(), device) {
821 EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<Arg1Type, Device>::Layout) ==
822 static_cast<int>(TensorEvaluator<Arg3Type, Device>::Layout) ||
823 internal::traits<XprType>::NumDimensions <= 1),
824 YOU_MADE_A_PROGRAMMING_MISTAKE);
825
826 EIGEN_STATIC_ASSERT((std::is_same<typename internal::traits<Arg1Type>::StorageKind,
827 typename internal::traits<Arg2Type>::StorageKind>::value),
828 STORAGE_KIND_MUST_MATCH)
829 EIGEN_STATIC_ASSERT((std::is_same<typename internal::traits<Arg1Type>::StorageKind,
830 typename internal::traits<Arg3Type>::StorageKind>::value),
831 STORAGE_KIND_MUST_MATCH)
832 EIGEN_STATIC_ASSERT(
833 (std::is_same<typename internal::traits<Arg1Type>::Index, typename internal::traits<Arg2Type>::Index>::value),
834 STORAGE_INDEX_MUST_MATCH)
835 EIGEN_STATIC_ASSERT(
836 (std::is_same<typename internal::traits<Arg1Type>::Index, typename internal::traits<Arg3Type>::Index>::value),
837 STORAGE_INDEX_MUST_MATCH)
838
839 EIGEN_INTERNAL_ASSERT_TENSOR_SHAPE(m_arg1Impl, m_arg2Impl);
840 EIGEN_INTERNAL_ASSERT_TENSOR_SHAPE(m_arg1Impl, m_arg3Impl);
841 }
842
843 using Index = typename XprType::Index;
844 using Scalar = typename XprType::Scalar;
845 using CoeffReturnType = typename internal::traits<XprType>::Scalar;
846 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
847 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
848 using Dimensions = typename TensorEvaluator<Arg1Type, Device>::Dimensions;
849 using Storage = StorageMemory<CoeffReturnType, Device>;
850 using EvaluatorPointerType = typename Storage::Type;
851 static constexpr int NumDims = internal::array_size<Dimensions>::value;
852
853 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
854 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
855 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
856
857 using Arg1TensorBlock = typename TensorEvaluator<const Arg1Type, Device>::TensorBlock;
858 using Arg2TensorBlock = typename TensorEvaluator<const Arg2Type, Device>::TensorBlock;
859 using Arg3TensorBlock = typename TensorEvaluator<const Arg3Type, Device>::TensorBlock;
860
861 // Rebuilds the ternary expression over the arguments' block expressions,
862 // carrying the functor; mirrors TensorSelectOpBlockFactory.
863 struct TensorCwiseTernaryOpBlockFactory {
864 TernaryOp func;
865
866 template <typename Arg1XprType, typename Arg2XprType, typename Arg3XprType>
867 struct XprType {
868 using type = TensorCwiseTernaryOp<TernaryOp, const Arg1XprType, const Arg2XprType, const Arg3XprType>;
869 };
870
871 template <typename Arg1XprType, typename Arg2XprType, typename Arg3XprType>
872 typename XprType<Arg1XprType, Arg2XprType, Arg3XprType>::type expr(const Arg1XprType& arg1, const Arg2XprType& arg2,
873 const Arg3XprType& arg3) const {
874 return typename XprType<Arg1XprType, Arg2XprType, Arg3XprType>::type(arg1, arg2, arg3, func);
875 }
876 };
877
878 using TensorBlock = internal::TensorTernaryExprBlock<TensorCwiseTernaryOpBlockFactory, Arg1TensorBlock,
879 Arg2TensorBlock, Arg3TensorBlock>;
880 //===--------------------------------------------------------------------===//
881
882 EIGEN_DEVICE_FUNC const Dimensions& dimensions() const {
883 // TODO: use arg2 or arg3 dimensions if they are known at compile time.
884 return m_arg1Impl.dimensions();
885 }
886
887 EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(EvaluatorPointerType) {
888 m_arg1Impl.evalSubExprsIfNeeded(nullptr);
889 // The executor does not call cleanup() when evaluation throws, so release the operands evaluated first.
890 EIGEN_TRY { m_arg2Impl.evalSubExprsIfNeeded(nullptr); }
891 EIGEN_CATCH(...) {
892 m_arg1Impl.cleanup();
893 EIGEN_THROW;
894 }
895 EIGEN_TRY { m_arg3Impl.evalSubExprsIfNeeded(nullptr); }
896 EIGEN_CATCH(...) {
897 m_arg1Impl.cleanup();
898 m_arg2Impl.cleanup();
899 EIGEN_THROW;
900 }
901 return true;
902 }
903 EIGEN_STRONG_INLINE void cleanup() {
904 m_arg1Impl.cleanup();
905 m_arg2Impl.cleanup();
906 m_arg3Impl.cleanup();
907 }
908
909 EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const {
910 return m_functor(m_arg1Impl.coeff(index), m_arg2Impl.coeff(index), m_arg3Impl.coeff(index));
911 }
912 template <int LoadMode>
913 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
914 return m_functor.packetOp(m_arg1Impl.template packet<LoadMode>(index), m_arg2Impl.template packet<LoadMode>(index),
915 m_arg3Impl.template packet<LoadMode>(index));
916 }
917
918 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
919 const double functor_cost = internal::functor_traits<TernaryOp>::Cost;
920 return m_arg1Impl.costPerCoeff(vectorized) + m_arg2Impl.costPerCoeff(vectorized) +
921 m_arg3Impl.costPerCoeff(vectorized) + TensorOpCost(0, 0, functor_cost, vectorized, PacketSize);
922 }
923
924 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
925 static constexpr double functor_cost = internal::functor_traits<TernaryOp>::Cost;
926 return internal::TensorBlockResourceRequirements::merge(
927 m_arg1Impl.getResourceRequirements(),
928 internal::TensorBlockResourceRequirements::merge(m_arg2Impl.getResourceRequirements(),
929 m_arg3Impl.getResourceRequirements()))
930 .addCostPerCoeff(TensorOpCost(0, 0, functor_cost, PacketAccess, PacketSize));
931 }
932
933 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
934 bool /*root_of_expr_ast*/ = false) const {
935 // The forwarded destination buffer might alias one of the inputs; drop it
936 // like the binary evaluator does.
937 desc.DropDestinationBuffer();
938 return TensorBlock(m_arg1Impl.block(desc, scratch), m_arg2Impl.block(desc, scratch),
939 m_arg3Impl.block(desc, scratch), TensorCwiseTernaryOpBlockFactory{m_functor});
940 }
941
942 EIGEN_DEVICE_FUNC EvaluatorPointerType data() const { return nullptr; }
943
944 private:
945 const TernaryOp m_functor;
946 TensorEvaluator<Arg1Type, Device> m_arg1Impl;
947 TensorEvaluator<Arg2Type, Device> m_arg2Impl;
948 TensorEvaluator<Arg3Type, Device> m_arg3Impl;
949};
950
951// -------------------- SelectOp --------------------
952
953template <typename IfArgType, typename ThenArgType, typename ElseArgType, typename Device>
954struct TensorEvaluator<const TensorSelectOp<IfArgType, ThenArgType, ElseArgType>, Device> {
955 using XprType = TensorSelectOp<IfArgType, ThenArgType, ElseArgType>;
956 using Scalar = typename XprType::Scalar;
957
958 using TernarySelectOp = internal::scalar_boolean_select_op<typename internal::traits<ThenArgType>::Scalar,
959 typename internal::traits<ElseArgType>::Scalar,
960 typename internal::traits<IfArgType>::Scalar>;
961 static constexpr bool TernaryPacketAccess =
962 TensorEvaluator<ThenArgType, Device>::PacketAccess && TensorEvaluator<ElseArgType, Device>::PacketAccess &&
963 TensorEvaluator<IfArgType, Device>::PacketAccess && internal::functor_traits<TernarySelectOp>::PacketAccess;
964
965 static constexpr int Layout = TensorEvaluator<IfArgType, Device>::Layout;
966 enum {
967 IsAligned = TensorEvaluator<ThenArgType, Device>::IsAligned && TensorEvaluator<ElseArgType, Device>::IsAligned,
968 PacketAccess =
969 (TensorEvaluator<ThenArgType, Device>::PacketAccess && TensorEvaluator<ElseArgType, Device>::PacketAccess) ||
970 TernaryPacketAccess,
971 BlockAccess = TensorEvaluator<IfArgType, Device>::BlockAccess &&
972 TensorEvaluator<ThenArgType, Device>::BlockAccess &&
973 TensorEvaluator<ElseArgType, Device>::BlockAccess,
974 PreferBlockAccess = TensorEvaluator<IfArgType, Device>::PreferBlockAccess ||
975 TensorEvaluator<ThenArgType, Device>::PreferBlockAccess ||
976 TensorEvaluator<ElseArgType, Device>::PreferBlockAccess,
977 CoordAccess = false, // to be implemented
978 RawAccess = false
979 };
980
981 EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device)
982 : m_condImpl(op.ifExpression(), device),
983 m_thenImpl(op.thenExpression(), device),
984 m_elseImpl(op.elseExpression(), device),
985 m_condSize(internal::array_prod(m_condImpl.dimensions())) {
986 EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<IfArgType, Device>::Layout) ==
987 static_cast<int>(TensorEvaluator<ThenArgType, Device>::Layout)),
988 YOU_MADE_A_PROGRAMMING_MISTAKE);
989 EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<IfArgType, Device>::Layout) ==
990 static_cast<int>(TensorEvaluator<ElseArgType, Device>::Layout)),
991 YOU_MADE_A_PROGRAMMING_MISTAKE);
992 EIGEN_INTERNAL_ASSERT_TENSOR_SHAPE(m_condImpl, m_thenImpl);
993 EIGEN_INTERNAL_ASSERT_TENSOR_SHAPE(m_thenImpl, m_elseImpl);
994 }
995
996 using Index = typename XprType::Index;
997 using CoeffReturnType = typename internal::traits<XprType>::Scalar;
998 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
999 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
1000 using CondScalar = std::remove_const_t<typename internal::traits<IfArgType>::Scalar>;
1001 using CondPacket = typename PacketType<CondScalar, Device>::type;
1002 static constexpr int CondPacketSize = PacketType<CondScalar, Device>::size;
1003 static constexpr bool VectorizedCondCast =
1004 std::is_same<CondScalar, bool>::value && TensorEvaluator<IfArgType, Device>::PacketAccess &&
1005 internal::type_casting_traits<CondScalar, std::remove_const_t<Scalar>>::VectorizedCast;
1006 // How condPacket() builds the mask: 0 = coefficient by coefficient, 1 = from one condition packet, 2 = from two,
1007 // 3 = from one wider condition packet while it stays inside the condition.
1008 static constexpr int CondPacketMode = !VectorizedCondCast ? 0
1009 : CondPacketSize == PacketSize ? 1
1010 : PacketSize == 2 * CondPacketSize ? 2
1011 : CondPacketSize > PacketSize ? 3
1012 : 0;
1013 using Dimensions = typename TensorEvaluator<IfArgType, Device>::Dimensions;
1014 using Storage = StorageMemory<CoeffReturnType, Device>;
1015 using EvaluatorPointerType = typename Storage::Type;
1016
1017 static constexpr int NumDims = internal::array_size<Dimensions>::value;
1018
1019 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
1020 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
1021 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
1022
1023 using IfArgTensorBlock = typename TensorEvaluator<const IfArgType, Device>::TensorBlock;
1024 using ThenArgTensorBlock = typename TensorEvaluator<const ThenArgType, Device>::TensorBlock;
1025 using ElseArgTensorBlock = typename TensorEvaluator<const ElseArgType, Device>::TensorBlock;
1026
1027 struct TensorSelectOpBlockFactory {
1028 template <typename IfArgXprType, typename ThenArgXprType, typename ElseArgXprType>
1029 struct XprType {
1030 using type = TensorSelectOp<const IfArgXprType, const ThenArgXprType, const ElseArgXprType>;
1031 };
1032
1033 template <typename IfArgXprType, typename ThenArgXprType, typename ElseArgXprType>
1034 typename XprType<IfArgXprType, ThenArgXprType, ElseArgXprType>::type expr(const IfArgXprType& if_expr,
1035 const ThenArgXprType& then_expr,
1036 const ElseArgXprType& else_expr) const {
1037 return typename XprType<IfArgXprType, ThenArgXprType, ElseArgXprType>::type(if_expr, then_expr, else_expr);
1038 }
1039 };
1040
1041 using TensorBlock = internal::TensorTernaryExprBlock<TensorSelectOpBlockFactory, IfArgTensorBlock, ThenArgTensorBlock,
1042 ElseArgTensorBlock>;
1043 //===--------------------------------------------------------------------===//
1044
1045 EIGEN_DEVICE_FUNC const Dimensions& dimensions() const {
1046 // TODO: use then or else impl instead if they happen to be known at compile time.
1047 return m_condImpl.dimensions();
1048 }
1049
1050 EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(EvaluatorPointerType) {
1051 m_condImpl.evalSubExprsIfNeeded(nullptr);
1052 // The executor does not call cleanup() when evaluation throws, so release the operands evaluated first.
1053 EIGEN_TRY { m_thenImpl.evalSubExprsIfNeeded(nullptr); }
1054 EIGEN_CATCH(...) {
1055 m_condImpl.cleanup();
1056 EIGEN_THROW;
1057 }
1058 EIGEN_TRY { m_elseImpl.evalSubExprsIfNeeded(nullptr); }
1059 EIGEN_CATCH(...) {
1060 m_condImpl.cleanup();
1061 m_thenImpl.cleanup();
1062 EIGEN_THROW;
1063 }
1064 return true;
1065 }
1066
1067#ifdef EIGEN_USE_THREADS
1068 template <typename EvalSubExprsCallback>
1069 EIGEN_STRONG_INLINE void evalSubExprsIfNeededAsync(EvaluatorPointerType, EvalSubExprsCallback done) {
1070 m_condImpl.evalSubExprsIfNeededAsync(nullptr, [this, done](bool) {
1071 m_thenImpl.evalSubExprsIfNeededAsync(
1072 nullptr, [this, done](bool) { m_elseImpl.evalSubExprsIfNeededAsync(nullptr, [done](bool) { done(true); }); });
1073 });
1074 }
1075#endif // EIGEN_USE_THREADS
1076
1077 EIGEN_STRONG_INLINE void cleanup() {
1078 m_condImpl.cleanup();
1079 m_thenImpl.cleanup();
1080 m_elseImpl.cleanup();
1081 }
1082
1083 EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const {
1084 return m_condImpl.coeff(index) ? m_thenImpl.coeff(index) : m_elseImpl.coeff(index);
1085 }
1086
1087 // Tag dispatch rather than enable_if overloads: MSVC 19.29 folds the non-dependent part of each enable_if
1088 // condition and then rejects the overloads whose conditions fold alike as redeclarations (C2535). Modes 1-3 are
1089 // member templates so that explicitly instantiating the evaluator does not instantiate pcasts its packet types lack.
1090 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType condPacket(Index index) const {
1091 return condPacket(index, std::integral_constant<int, CondPacketMode>());
1092 }
1093
1094 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType condPacket(Index index, std::integral_constant<int, 0>) const {
1095 return scalarCondPacket(index);
1096 }
1097
1098 template <typename Unused = void>
1099 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType condPacket(Index index, std::integral_constant<int, 1>) const {
1100 return internal::pcast<CondPacket, PacketReturnType>(m_condImpl.template packet<Unaligned>(index));
1101 }
1102
1103 template <typename Unused = void>
1104 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType condPacket(Index index, std::integral_constant<int, 2>) const {
1105 return internal::pcast<CondPacket, PacketReturnType>(m_condImpl.template packet<Unaligned>(index),
1106 m_condImpl.template packet<Unaligned>(index + CondPacketSize));
1107 }
1108
1109 template <typename Unused = void>
1110 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType condPacket(Index index, std::integral_constant<int, 3>) const {
1111 if (index + CondPacketSize <= m_condSize) {
1112 return internal::pcast<CondPacket, PacketReturnType>(m_condImpl.template packet<Unaligned>(index));
1113 }
1114 return scalarCondPacket(index);
1115 }
1116
1117 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType scalarCondPacket(Index index) const {
1118 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment)
1119 std::remove_const_t<Scalar> arr[PacketSize];
1120 EIGEN_UNROLL_LOOP
1121 for (Index i = 0; i < PacketSize; ++i) {
1122 arr[i] = m_condImpl.coeff(index + i) ? Scalar(true) : Scalar(false);
1123 }
1124 return internal::pload<PacketReturnType>(arr);
1125 }
1126
1127 template <int LoadMode, bool UseTernary = TernaryPacketAccess, std::enable_if_t<!UseTernary, bool> = true>
1128 EIGEN_DEVICE_FUNC PacketReturnType packet(Index index) const {
1129 return TernarySelectOp().template packetOp<PacketReturnType>(
1130 m_thenImpl.template packet<LoadMode>(index), m_elseImpl.template packet<LoadMode>(index), condPacket(index));
1131 }
1132
1133 template <int LoadMode, bool UseTernary = TernaryPacketAccess, std::enable_if_t<UseTernary, bool> = true>
1134 EIGEN_DEVICE_FUNC PacketReturnType packet(Index index) const {
1135 return TernarySelectOp().template packetOp<PacketReturnType>(m_thenImpl.template packet<LoadMode>(index),
1136 m_elseImpl.template packet<LoadMode>(index),
1137 m_condImpl.template packet<LoadMode>(index));
1138 }
1139
1140 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
1141 const bool cond_vectorized = vectorized && (TernaryPacketAccess || CondPacketMode != 0);
1142 return m_condImpl.costPerCoeff(cond_vectorized) +
1143 (vectorized ? (m_thenImpl.costPerCoeff(vectorized) + m_elseImpl.costPerCoeff(vectorized) +
1144 TensorOpCost(0, 0, internal::functor_traits<TernarySelectOp>::Cost, vectorized, PacketSize))
1145 : m_thenImpl.costPerCoeff(vectorized).cwiseMax(m_elseImpl.costPerCoeff(vectorized)));
1146 }
1147
1148 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
1149 auto then_req = m_thenImpl.getResourceRequirements();
1150 auto else_req = m_elseImpl.getResourceRequirements();
1151
1152 auto merged_req = internal::TensorBlockResourceRequirements::merge(then_req, else_req);
1153 merged_req.cost_per_coeff =
1154 PacketAccess ? (then_req.cost_per_coeff + else_req.cost_per_coeff +
1155 TensorOpCost(0, 0, internal::functor_traits<TernarySelectOp>::Cost, PacketAccess, PacketSize))
1156 : then_req.cost_per_coeff.cwiseMax(else_req.cost_per_coeff);
1157
1158 return internal::TensorBlockResourceRequirements::merge(m_condImpl.getResourceRequirements(), merged_req);
1159 }
1160
1161 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
1162 bool /*root_of_expr_ast*/ = false) const {
1163 // It's unsafe to pass destination buffer to underlying expressions, because
1164 // output might be aliased with one of the inputs.
1165 desc.DropDestinationBuffer();
1166
1167 return TensorBlock(m_condImpl.block(desc, scratch), m_thenImpl.block(desc, scratch),
1168 m_elseImpl.block(desc, scratch), TensorSelectOpBlockFactory());
1169 }
1170
1171 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EvaluatorPointerType data() const { return nullptr; }
1172
1173#ifdef EIGEN_USE_SYCL
1174 // binding placeholder accessors to a command group handler for SYCL
1175 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void bind(cl::sycl::handler& cgh) const {
1176 m_condImpl.bind(cgh);
1177 m_thenImpl.bind(cgh);
1178 m_elseImpl.bind(cgh);
1179 }
1180#endif
1181 private:
1182 TensorEvaluator<IfArgType, Device> m_condImpl;
1183 TensorEvaluator<ThenArgType, Device> m_thenImpl;
1184 TensorEvaluator<ElseArgType, Device> m_elseImpl;
1185 const Index m_condSize;
1186};
1187
1188} // end namespace Eigen
1189
1190#if defined(EIGEN_USE_SYCL) && defined(SYCL_COMPILER_IS_DPCPP)
1191template <typename Derived, typename Device>
1192struct cl::sycl::is_device_copyable<
1193 Eigen::TensorEvaluator<Derived, Device>,
1194 std::enable_if_t<!std::is_trivially_copyable<Eigen::TensorEvaluator<Derived, Device>>::value>> : std::true_type {};
1195#endif
1196
1197#endif // EIGEN_TENSOR_TENSOR_EVALUATOR_H
Tensor binary expression.
Definition TensorExpr.h:153
Tensor nullary expression.
Definition TensorExpr.h:41
Tensor unary expression.
Definition TensorExpr.h:90
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47