11#ifndef EIGEN_VISITOR_H
12#define EIGEN_VISITOR_H
15#include "./InternalHeaderCheck.h"
22template <
typename Visitor,
typename =
void>
23struct visitor_already_initialized : std::false_type {};
24template <
typename Visitor>
25struct visitor_already_initialized<Visitor, void_t<decltype(functor_traits<Visitor>::AlreadyInitialized)>>
26 : bool_constant<functor_traits<Visitor>::AlreadyInitialized> {};
28template <
typename Visitor,
typename Derived,
int UnrollCount,
29 bool Vectorize = (Derived::PacketAccess && functor_traits<Visitor>::PacketAccess),
bool LinearAccess =
false,
30 bool ShortCircuitEvaluation =
false>
33template <
typename Visitor,
bool ShortCircuitEvaluation = false>
34struct short_circuit_eval_impl {
36 static constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool run(
const Visitor&) {
return false; }
38template <
typename Visitor>
39struct short_circuit_eval_impl<Visitor, true> {
41 static constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool run(
const Visitor& visitor) {
return visitor.done(); }
45template <
typename Visitor,
typename Derived,
int UnrollCount,
bool Vectorize,
bool ShortCircuitEvaluation>
46struct visitor_impl<Visitor, Derived, UnrollCount, Vectorize, false, ShortCircuitEvaluation> {
48 using Scalar =
typename Derived::Scalar;
49 using Packet =
typename packet_traits<Scalar>::type;
50 static constexpr bool RowMajor = Derived::IsRowMajor;
51 static constexpr int RowsAtCompileTime = Derived::RowsAtCompileTime;
52 static constexpr int ColsAtCompileTime = Derived::ColsAtCompileTime;
53 static constexpr int PacketSize = packet_traits<Scalar>::size;
54 static constexpr int InnerSizeAtCompileTime =
RowMajor ? ColsAtCompileTime : RowsAtCompileTime;
55 static constexpr int OuterSizeAtCompileTime =
RowMajor ? RowsAtCompileTime : ColsAtCompileTime;
56 static constexpr int PacketOpsPerOuter =
57 Vectorize && InnerSizeAtCompileTime >= PacketSize ? InnerSizeAtCompileTime / PacketSize : 0;
58 static constexpr int FirstScalarInner = PacketOpsPerOuter * PacketSize;
59 static constexpr int ScalarOpsPerOuter = InnerSizeAtCompileTime - FirstScalarInner;
60 static constexpr int OpsPerOuter = PacketOpsPerOuter + ScalarOpsPerOuter;
61 static constexpr int OpCount = UnrollCount == 0 ? 0 : OuterSizeAtCompileTime * OpsPerOuter;
64 static constexpr bool IsPacketOp() {
65 return OpsPerOuter != 0 && (Op % OpsPerOuter) < PacketOpsPerOuter;
69 static constexpr int CoeffIndex() {
70 return OpsPerOuter == 0 ? 0
71 : (Op / OpsPerOuter) * InnerSizeAtCompileTime +
72 (IsPacketOp<Op>() ? ((Op % OpsPerOuter) * PacketSize)
73 : (FirstScalarInner + (Op % OpsPerOuter) - PacketOpsPerOuter));
76 template <int Op, std::enable_if_t<!IsPacketOp<Op>(),
bool> =
true>
77 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void visit(
const Derived& mat, Visitor& visitor) {
78 constexpr int K = CoeffIndex<Op>();
79 constexpr int R =
RowMajor ? (K / ColsAtCompileTime) : (K % RowsAtCompileTime);
80 constexpr int C =
RowMajor ? (K % ColsAtCompileTime) : (K / RowsAtCompileTime);
81 EIGEN_IF_CONSTEXPR (Op == 0) {
82 visitor.init(mat.coeff(R, C), R, C);
84 visitor(mat.coeff(R, C), R, C);
88 template <int Op, std::enable_if_t<IsPacketOp<Op>(),
bool> =
true>
89 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void visit(
const Derived& mat, Visitor& visitor) {
90 constexpr int K = CoeffIndex<Op>();
91 constexpr int R =
RowMajor ? (K / ColsAtCompileTime) : (K % RowsAtCompileTime);
92 constexpr int C =
RowMajor ? (K % ColsAtCompileTime) : (K / RowsAtCompileTime);
93 Packet P = mat.template packet<Packet>(R, C);
94 EIGEN_IF_CONSTEXPR (Op == 0) {
95 visitor.initpacket(P, R, C);
97 visitor.packet(P, R, C);
101 template <
int... Ops>
102 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void run_impl(
const Derived& mat, Visitor& visitor,
103 std::integer_sequence<int, Ops...>) {
104 int unused[] = {0, (visit<Ops>(mat, visitor), 0)...};
105 EIGEN_UNUSED_VARIABLE(unused);
108 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void run(
const Derived& mat, Visitor& visitor) {
109 run_impl(mat, visitor, std::make_integer_sequence<int, OpCount>{});
114template <
typename Visitor,
typename Derived,
int UnrollCount,
bool Vectorize,
bool ShortCircuitEvaluation>
115struct visitor_impl<Visitor, Derived, UnrollCount, Vectorize, true, ShortCircuitEvaluation> {
117 using Scalar =
typename Derived::Scalar;
118 using Packet =
typename packet_traits<Scalar>::type;
119 static constexpr int PacketSize = packet_traits<Scalar>::size;
120 static constexpr int PacketOps = Vectorize ? UnrollCount / PacketSize : 0;
121 static constexpr int FirstScalar = PacketOps * PacketSize;
122 static constexpr int ScalarOps = UnrollCount - FirstScalar;
123 static constexpr int OpCount = PacketOps + ScalarOps;
126 static constexpr bool IsPacketOp() {
127 return Op < PacketOps;
131 static constexpr int CoeffIndex() {
132 return IsPacketOp<Op>() ? (Op * PacketSize) : (FirstScalar + Op - PacketOps);
135 template <int Op, std::enable_if_t<!IsPacketOp<Op>(),
bool> =
true>
136 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void visit(
const Derived& mat, Visitor& visitor) {
137 constexpr int K = CoeffIndex<Op>();
138 EIGEN_IF_CONSTEXPR (Op == 0) {
139 visitor.init(mat.coeff(K), K);
141 visitor(mat.coeff(K), K);
145 template <int Op, std::enable_if_t<IsPacketOp<Op>(),
bool> =
true>
146 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void visit(
const Derived& mat, Visitor& visitor) {
147 constexpr int K = CoeffIndex<Op>();
148 Packet P = mat.template packet<Packet>(K);
149 EIGEN_IF_CONSTEXPR (Op == 0) {
150 visitor.initpacket(P, K);
152 visitor.packet(P, K);
156 template <
int... Ops>
157 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void run_impl(
const Derived& mat, Visitor& visitor,
158 std::integer_sequence<int, Ops...>) {
159 int unused[] = {0, (visit<Ops>(mat, visitor), 0)...};
160 EIGEN_UNUSED_VARIABLE(unused);
163 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void run(
const Derived& mat, Visitor& visitor) {
164 run_impl(mat, visitor, std::make_integer_sequence<int, OpCount>{});
169template <
typename Visitor,
typename Derived,
bool ShortCircuitEvaluation>
170struct visitor_impl<Visitor, Derived, Dynamic, false, false, ShortCircuitEvaluation> {
171 using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>;
172 static constexpr bool RowMajor = Derived::IsRowMajor;
174 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void run(
const Derived& mat, Visitor& visitor) {
175 const Index innerSize =
RowMajor ? mat.cols() : mat.rows();
176 const Index outerSize =
RowMajor ? mat.rows() : mat.cols();
177 if (innerSize == 0 || outerSize == 0)
return;
179 visitor.init(mat.coeff(0, 0), 0, 0);
180 if (short_circuit::run(visitor))
return;
181 for (Index i = 1; i < innerSize; ++i) {
184 visitor(mat.coeff(r, c), r, c);
185 if EIGEN_PREDICT_FALSE (short_circuit::run(visitor))
return;
188 for (Index j = 1; j < outerSize; j++) {
189 for (Index i = 0; i < innerSize; ++i) {
192 visitor(mat.coeff(r, c), r, c);
193 if EIGEN_PREDICT_FALSE (short_circuit::run(visitor))
return;
200template <
typename Visitor,
typename Derived,
bool ShortCircuitEvaluation>
201struct visitor_impl<Visitor, Derived, Dynamic, true, false, ShortCircuitEvaluation> {
202 using Scalar =
typename Derived::Scalar;
203 using Packet =
typename packet_traits<Scalar>::type;
204 static constexpr int PacketSize = packet_traits<Scalar>::size;
205 using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>;
206 static constexpr bool RowMajor = Derived::IsRowMajor;
207 static constexpr bool AlreadyInitialized = visitor_already_initialized<Visitor>::value;
209 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void run(
const Derived& mat, Visitor& visitor) {
210 const Index innerSize =
RowMajor ? mat.cols() : mat.rows();
211 const Index outerSize =
RowMajor ? mat.rows() : mat.cols();
212 if (innerSize == 0 || outerSize == 0)
return;
213 EIGEN_IF_CONSTEXPR (!AlreadyInitialized) {
215 if (innerSize < PacketSize) {
216 visitor.init(mat.coeff(0, 0), 0, 0);
219 Packet p = mat.template packet<Packet>(0, 0);
220 visitor.initpacket(p, 0, 0);
223 if EIGEN_PREDICT_FALSE (short_circuit::run(visitor))
return;
224 for (; i + PacketSize - 1 < innerSize; i += PacketSize) {
227 Packet p = mat.template packet<Packet>(r, c);
228 visitor.packet(p, r, c);
229 if EIGEN_PREDICT_FALSE (short_circuit::run(visitor))
return;
231 for (; i < innerSize; ++i) {
234 visitor(mat.coeff(r, c), r, c);
235 if EIGEN_PREDICT_FALSE (short_circuit::run(visitor))
return;
238 const Index packetEnd = innerSize - innerSize % PacketSize;
239 for (Index j = AlreadyInitialized ? 0 : 1; j < outerSize; j++) {
241 for (; AlreadyInitialized ? i < packetEnd : i + PacketSize - 1 < innerSize; i += PacketSize) {
244 Packet p = mat.template packet<Packet>(r, c);
245 visitor.packet(p, r, c);
246 if EIGEN_PREDICT_FALSE (short_circuit::run(visitor))
return;
248 for (; i < innerSize; ++i) {
251 visitor(mat.coeff(r, c), r, c);
252 if EIGEN_PREDICT_FALSE (short_circuit::run(visitor))
return;
259template <
typename Visitor,
typename Derived,
bool ShortCircuitEvaluation>
260struct visitor_impl<Visitor, Derived, Dynamic, false, true, ShortCircuitEvaluation> {
261 using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>;
263 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void run(
const Derived& mat, Visitor& visitor) {
264 const Index size = mat.size();
265 if (size == 0)
return;
266 visitor.init(mat.coeff(0), 0);
267 if EIGEN_PREDICT_FALSE (short_circuit::run(visitor))
return;
268 for (Index k = 1; k < size; k++) {
269 visitor(mat.coeff(k), k);
270 if EIGEN_PREDICT_FALSE (short_circuit::run(visitor))
return;
276template <
typename Visitor,
typename Derived,
bool ShortCircuitEvaluation>
277struct visitor_impl<Visitor, Derived, Dynamic, true, true, ShortCircuitEvaluation> {
278 using Scalar =
typename Derived::Scalar;
279 using Packet =
typename packet_traits<Scalar>::type;
280 static constexpr int PacketSize = packet_traits<Scalar>::size;
281 using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>;
282 static constexpr bool AlreadyInitialized = visitor_already_initialized<Visitor>::value;
284 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void run(
const Derived& mat, Visitor& visitor) {
285 const Index size = mat.size();
286 if (size == 0)
return;
287 const Index packetEnd = size - size % PacketSize;
289 EIGEN_IF_CONSTEXPR (!AlreadyInitialized) {
290 if (size < PacketSize) {
291 visitor.init(mat.coeff(0), 0);
294 Packet p = mat.template packet<Packet>(k);
295 visitor.initpacket(p, k);
298 if EIGEN_PREDICT_FALSE (short_circuit::run(visitor))
return;
300 for (; k < packetEnd; k += PacketSize) {
301 Packet p = mat.template packet<Packet>(k);
302 visitor.packet(p, k);
303 if EIGEN_PREDICT_FALSE (short_circuit::run(visitor))
return;
305 for (; k < size; k++) {
306 visitor(mat.coeff(k), k);
307 if EIGEN_PREDICT_FALSE (short_circuit::run(visitor))
return;
313template <
typename XprType>
314class visitor_evaluator {
316 using Evaluator = evaluator<XprType>;
317 using Scalar =
typename XprType::Scalar;
318 using Packet =
typename packet_traits<Scalar>::type;
319 using CoeffReturnType = std::remove_const_t<typename XprType::CoeffReturnType>;
321 static constexpr bool PacketAccess =
static_cast<bool>(Evaluator::Flags &
PacketAccessBit);
322 static constexpr bool LinearAccess =
static_cast<bool>(Evaluator::Flags &
LinearAccessBit);
323 static constexpr bool IsRowMajor =
static_cast<bool>(XprType::IsRowMajor);
324 static constexpr int RowsAtCompileTime = XprType::RowsAtCompileTime;
325 static constexpr int ColsAtCompileTime = XprType::ColsAtCompileTime;
326 static constexpr int XprAlignment = Evaluator::Alignment;
327 static constexpr int CoeffReadCost = Evaluator::CoeffReadCost;
329 EIGEN_DEVICE_FUNC
explicit visitor_evaluator(
const XprType& xpr) : m_evaluator(xpr), m_xpr(xpr) {}
331 EIGEN_DEVICE_FUNC
constexpr Index rows() const noexcept {
return m_xpr.rows(); }
332 EIGEN_DEVICE_FUNC
constexpr Index cols() const noexcept {
return m_xpr.cols(); }
333 EIGEN_DEVICE_FUNC
constexpr Index size() const noexcept {
return m_xpr.size(); }
335 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col)
const {
336 return m_evaluator.coeff(row, col);
338 template <
typename Packet,
int Alignment = Unaligned>
339 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(Index row, Index col)
const {
340 return m_evaluator.template packet<Alignment, Packet>(row, col);
342 template <
typename Packet>
343 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetSegment(Index row, Index col, Index begin, Index count)
const {
344 return m_evaluator.template packetSegment<Unaligned, Packet>(row, col, begin, count);
347 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index)
const {
return m_evaluator.coeff(index); }
348 template <
typename Packet,
int Alignment = XprAlignment>
349 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(Index index)
const {
350 return m_evaluator.template packet<Alignment, Packet>(index);
352 template <
typename Packet>
353 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetSegment(Index index, Index begin, Index count)
const {
354 return m_evaluator.template packetSegment<Unaligned, Packet>(index, begin, count);
358 Evaluator m_evaluator;
359 const XprType& m_xpr;
362template <
typename T,
typename =
void>
363struct visitor_has_linear_access : std::false_type {};
366struct visitor_has_linear_access<T, void_t<decltype(functor_traits<T>::LinearAccess)>>
367 : bool_constant<static_cast<bool>(functor_traits<T>::LinearAccess)> {};
369template <
typename Derived,
typename Visitor,
bool ShortCircuitEvaluation>
371 using Evaluator = visitor_evaluator<Derived>;
378 static constexpr int InnerSizeAtCompileTime = IsRowMajor ? ColsAtCompileTime : RowsAtCompileTime;
379 static constexpr int OuterSizeAtCompileTime = IsRowMajor ? RowsAtCompileTime : ColsAtCompileTime;
383 static constexpr bool LinearAccess = (!ShortCircuitEvaluation || visitor_already_initialized<Visitor>::value) &&
384 Evaluator::LinearAccess && visitor_has_linear_access<Visitor>::value;
385 static constexpr bool Vectorize = Evaluator::PacketAccess &&
static_cast<bool>(functor_traits<Visitor>::PacketAccess);
387 static constexpr int PacketSize = packet_traits<Scalar>::size;
388 static constexpr int VectorOps =
389 Vectorize ? (LinearAccess ? (SizeAtCompileTime / PacketSize)
390 : (OuterSizeAtCompileTime * (InnerSizeAtCompileTime / PacketSize)))
392 static constexpr int ScalarOps = SizeAtCompileTime - (VectorOps * PacketSize);
394 static constexpr int TotalOps = VectorOps + ScalarOps;
396 static constexpr int UnrollCost = int(Evaluator::CoeffReadCost) + int(functor_traits<Visitor>::Cost);
397 static constexpr bool Unroll = (SizeAtCompileTime != Dynamic) && ((TotalOps * UnrollCost) <= EIGEN_UNROLLING_LIMIT);
398 static constexpr int UnrollCount = Unroll ? int(SizeAtCompileTime) : Dynamic;
400 using impl = visitor_impl<Visitor, Evaluator, UnrollCount, Vectorize, LinearAccess, ShortCircuitEvaluation>;
402 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void run(
const DenseBase<Derived>& mat, Visitor& visitor) {
403 Evaluator evaluator(mat.derived());
404 impl::run(evaluator, visitor);
408template <
typename Scalar>
409struct count_visitor {
410 using result_type = Index;
411 using Packet =
typename packet_traits<Scalar>::type;
412 EIGEN_DEVICE_FUNC
inline void init(
const Scalar& value, Index, Index) { res = value != Scalar(0) ? 1 : 0; }
413 EIGEN_DEVICE_FUNC
inline void init(
const Scalar& value, Index) { res = value != Scalar(0) ? 1 : 0; }
414 EIGEN_DEVICE_FUNC
inline Index count_redux(
const Packet& p)
const {
return predux_count(p); }
415 EIGEN_DEVICE_FUNC
inline void initpacket(
const Packet& p, Index, Index) { res = count_redux(p); }
416 EIGEN_DEVICE_FUNC
inline void initpacket(
const Packet& p, Index) { res = count_redux(p); }
417 EIGEN_DEVICE_FUNC
inline void operator()(
const Scalar& value, Index, Index) {
418 if (value != Scalar(0)) res++;
420 EIGEN_DEVICE_FUNC
inline void operator()(
const Scalar& value, Index) {
421 if (value != Scalar(0)) res++;
423 EIGEN_DEVICE_FUNC
inline void packet(
const Packet& p, Index, Index) { res += count_redux(p); }
424 EIGEN_DEVICE_FUNC
inline void packet(
const Packet& p, Index) { res += count_redux(p); }
428template <
typename Scalar>
429struct functor_traits<count_visitor<Scalar>> {
431 Cost = NumTraits<Scalar>::AddCost,
433 PacketAccess = packet_traits<Scalar>::HasCmp && packet_traits<Scalar>::HasAdd
458template <
typename Derived>
459template <
typename Visitor>
461 using impl = internal::visit_impl<Derived, Visitor,
false>;
462 impl::run(derived(), visitor);
469template <
typename Derived>
471 using Visitor = internal::count_visitor<Scalar>;
472 using impl = internal::visit_impl<Derived, Visitor,
false>;
474 impl::run(derived(), visitor);
void visit(Visitor &func) const
Definition Visitor.h:460
@ SizeAtCompileTime
Definition DenseBase.h:109
@ IsRowMajor
Definition DenseBase.h:167
@ ColsAtCompileTime
Definition DenseBase.h:103
@ RowsAtCompileTime
Definition DenseBase.h:97
typename internal::traits< Derived >::Scalar Scalar
Definition DenseBase.h:63
Index count() const
Definition Visitor.h:470
@ RowMajor
Definition Constants.h:321
constexpr unsigned int PacketAccessBit
Definition Constants.h:98
constexpr unsigned int LinearAccessBit
Definition Constants.h:134