Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
Predicator.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// SPDX-FileCopyrightText: The Eigen Authors
5// SPDX-License-Identifier: MPL-2.0
6
7#ifndef EIGEN_PREDICATOR_H
8#define EIGEN_PREDICATOR_H
9
10// IWYU pragma: private
11#include "./InternalHeaderCheck.h"
12
13namespace Eigen {
14
15namespace internal {
16
17// Predicate reductions all_of and any_of. A predicate tests a scalar with operator() and a packet with
18// packetOp, which returns a truth packet: each lane is zero where the predicate is false and, where it is true, the
19// lane mask pcmp_* produces (all ones, or one for bool packets). Truth packets combine with pand and por, so the
20// traversals below reduce and branch once per block of packets rather than once per packet. Results do not depend on
21// the grouping, but a short-circuit may read past the deciding coefficient, up to the end of its block.
22template <typename Scalar>
23struct nonzero_predicate {
24 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const Scalar& x) const { return x != Scalar(0); }
25 template <typename Packet>
26 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& x) const {
27 return pandnot(ptrue(x), pcmp_eq(x, pzero(x)));
28 }
29};
30template <typename Scalar>
31struct functor_traits<nonzero_predicate<Scalar>> {
32 static constexpr bool PacketAccess = packet_traits<Scalar>::HasCmp;
33 // Accepts that ARMv7 NEON reads a subnormal float as zero: packets compare there already, and grouped scalar tests
34 // may be vectorized onto it. Testing float one at a time instead cost 0.7-0.85x at sizes with a scalar tail on
35 // Cortex-A72.
36 static constexpr bool GroupScalars = true;
37};
38
39template <typename Scalar>
40struct isfinite_predicate {
41 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const Scalar& x) const { return (numext::isfinite)(x); }
42 template <typename Packet>
43 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& x) const {
44 return pisfinite(x);
45 }
46};
47template <typename Scalar>
48struct functor_traits<isfinite_predicate<Scalar>> {
49 static constexpr bool PacketAccess = packet_traits<Scalar>::HasCmp;
50 static constexpr bool GroupScalars = true;
51};
52
53// Separate specializations keep custom scalars that support only one of isApprox and isMuchSmallerThan compiling.
54template <typename Scalar, bool Approximate>
55struct fuzzy_constant_predicate {
56 using RealScalar = typename NumTraits<Scalar>::Real;
57 Scalar value;
58 RealScalar precision;
59 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const Scalar& x) const {
60 return internal::isApprox(x, value, precision);
61 }
62 // Requires 0 <= precision <= 1 and a finite value. Then min(|x|, |v|) p = min(|x| p, |v| p) exactly, and |x| p is
63 // x copysign(p, v) where x has the sign of v; where it does not, |x - v| >= max(|x|, |v|) > p min(|x|, |v|) unless
64 // x = v = 0, and x copysign(p, v) <= 0 rejects it as well.
65 template <typename Packet>
66 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& x) const {
67 const Packet bound = pset1<Packet>(numext::abs(value) * precision);
68 const Packet scale = pset1<Packet>(numext::copysign(precision, value));
69 return pcmp_le(pabs(psub(x, pset1<Packet>(value))), pmin(pmul(x, scale), bound));
70 }
71 EIGEN_DEVICE_FUNC bool packetOpApplies() const {
72 return precision >= RealScalar(0) && precision <= RealScalar(1) && (numext::isfinite)(value);
73 }
74};
75template <typename Scalar>
76struct fuzzy_constant_predicate<Scalar, false> {
77 Scalar value;
78 typename NumTraits<Scalar>::Real precision;
79 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const Scalar& x) const {
80 return internal::isMuchSmallerThan(x, Scalar(1), precision);
81 }
82 template <typename Packet>
83 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& x) const {
84 return pcmp_le(pabs(x), pset1<Packet>(precision));
85 }
86};
87template <typename Scalar, bool Approximate>
88struct functor_traits<fuzzy_constant_predicate<Scalar, Approximate>> {
89 // ARMv7 NEON flushes subnormal float operands and results; scalar VFP does not.
90 // Explicitly flushing would change exact-zero checks; normal operands can also have a subnormal difference.
91 static constexpr bool PacketAccess =
92 (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value) &&
93 !(EIGEN_ARCH_ARM && std::is_same<Scalar, float>::value) && packet_traits<Scalar>::HasAbs &&
94 packet_traits<Scalar>::HasCmp &&
95 (!Approximate ||
96 (packet_traits<Scalar>::HasSub && packet_traits<Scalar>::HasMin && packet_traits<Scalar>::HasMul));
97 // Grouped scalar compares may be vectorized onto the unit PacketAccess excludes.
98 static constexpr bool GroupScalars = PacketAccess;
99 // The approximate test saturates the vector units, so larger blocks barely speed up full scans but read further
100 // past a deciding coefficient: 2 packets per exit is never slower than master's exit per packet.
101 static constexpr int BlockPackets = Approximate ? 2 : 8;
102};
103
104// A predicate's scalar test alone, tested one coefficient at a time.
105template <typename Predicate>
106struct scalar_predicate {
107 Predicate pred;
108 template <typename Scalar>
109 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const Scalar& x) const {
110 return pred(x);
111 }
112};
113template <typename Predicate>
114struct functor_traits<scalar_predicate<Predicate>> {
115 static constexpr bool PacketAccess = false;
116};
117
118// Searches for a coefficient on which the predicate is Target: any_of searches for true, all_of for false. Packets
119// combine truth masks and test the combined mask; scalars search for a true test_scalar by OR, which vectorizes
120// better than an AND of test results.
121struct packet_predicate {
122 template <typename Predicate, typename Packet>
123 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet apply(const Predicate& pred, const Packet& x) {
124 return pred.packetOp(x);
125 }
126};
127template <bool Target>
128struct predicate_search;
129template <>
130struct predicate_search<true> : packet_predicate {
131 template <typename Packet>
132 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet combine(const Packet& a, const Packet& b) {
133 return por(a, b);
134 }
135 template <typename Packet>
136 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool found(const Packet& m) {
137 return predux_any(m);
138 }
139 // Only the lanes set in a lane mask, as for a segment.
140 template <typename Packet>
141 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool found_in(const Packet& m, const Packet& lanes) {
142 return predux_any(pand(m, lanes));
143 }
144 template <typename Predicate, typename Scalar>
145 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool test_scalar(const Predicate& pred, const Scalar& x) {
146 return pred(x);
147 }
148};
149template <>
150struct predicate_search<false> : packet_predicate {
151 template <typename Packet>
152 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet combine(const Packet& a, const Packet& b) {
153 return pand(a, b);
154 }
155 // pandnot rather than pnot, which some targets implement only bytewise.
156 template <typename Packet>
157 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool found(const Packet& m) {
158 return predux_any(pandnot(ptrue(m), m));
159 }
160 template <typename Packet>
161 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool found_in(const Packet& m, const Packet& lanes) {
162 return predux_any(pandnot(lanes, m));
163 }
164 template <typename Predicate, typename Scalar>
165 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool test_scalar(const Predicate& pred, const Scalar& x) {
166 return !pred(x);
167 }
168};
169
170// all_of(nonzero) searches for zero lanes, x == 0 being the exact complement of x != 0: a single compare per packet
171// and no complement of the combined mask, which some compilers otherwise lower through narrowing and widening.
172// Scalars still test the predicate: a coefficient is found where it is false.
173struct zero_lane_search : predicate_search<true> {
174 template <typename Predicate, typename Packet>
175 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet apply(const Predicate&, const Packet& x) {
176 return pcmp_eq(x, pzero(x));
177 }
178 template <typename Predicate, typename Scalar>
179 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool test_scalar(const Predicate& pred, const Scalar& x) {
180 return !pred(x);
181 }
182};
183// any_of on bool lanes reduces them directly, treating every nonzero byte as true as the scalar reduction does.
184struct bool_lane_search : predicate_search<true> {
185 template <typename Predicate, typename Packet>
186 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet apply(const Predicate&, const Packet& x) {
187 return x;
188 }
189};
190// any_of(nonzero) on integer lanes ORs the coefficients themselves and compares the OR with zero once per block: it
191// is zero exactly where every coefficient is. Floating-point lanes keep the compare per packet: their OR can be a
192// signaling NaN, which a floating-point compare would flag as an invalid operation.
193struct or_lane_search : predicate_search<true> {
194 template <typename Predicate, typename Packet>
195 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet apply(const Predicate&, const Packet& x) {
196 return x;
197 }
198 template <typename Packet>
199 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool found(const Packet& m) {
200 return predux_any(pandnot(ptrue(m), pcmp_eq(m, pzero(m))));
201 }
202 template <typename Packet>
203 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool found_in(const Packet& m, const Packet& lanes) {
204 return found(pand(m, lanes));
205 }
206};
207template <bool Target, typename Predicate>
208struct predicate_search_for {
209 using type = predicate_search<Target>;
210};
211template <typename Scalar>
212struct predicate_search_for<false, nonzero_predicate<Scalar>> {
213 using type = zero_lane_search;
214};
215template <typename Scalar>
216struct predicate_search_for<true, nonzero_predicate<Scalar>> {
217 using type = std::conditional_t<std::is_integral<Scalar>::value, or_lane_search, predicate_search<true>>;
218};
219template <>
220struct predicate_search_for<true, nonzero_predicate<bool>> {
221 using type = bool_lane_search;
222};
223
224// Coefficient access along one inner vector, or along the whole expression with linear access.
225template <typename Evaluator, bool Linear = Evaluator::LinearAccess>
226struct predicate_segment {
227 using Scalar = typename Evaluator::Scalar;
228 static constexpr bool FixedSize = Evaluator::RowsAtCompileTime > 0 && Evaluator::ColsAtCompileTime > 0;
229 static constexpr bool PlainLoad = int(Evaluator::CoeffReadCost) <= int(NumTraits<Scalar>::ReadCost);
230 const Evaluator& eval;
231 Index length;
232 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE predicate_segment(const Evaluator& evaluator, Index, Index size)
233 : eval(evaluator), length(size) {}
234 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index size() const { return length; }
235 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Evaluator::CoeffReturnType coeff(Index i) const {
236 return eval.coeff(i);
237 }
238 template <typename Packet>
239 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(Index i) const {
240 return eval.template packet<Packet>(i);
241 }
242 template <typename Packet>
243 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetSegment(Index i, Index count) const {
244 return eval.template packetSegment<Packet>(i, 0, count);
245 }
246};
247template <typename Evaluator>
248struct predicate_segment<Evaluator, false> {
249 using Scalar = typename Evaluator::Scalar;
250 static constexpr bool FixedSize =
251 (Evaluator::IsRowMajor ? Evaluator::ColsAtCompileTime : Evaluator::RowsAtCompileTime) > 0;
252 static constexpr bool PlainLoad = int(Evaluator::CoeffReadCost) <= int(NumTraits<Scalar>::ReadCost);
253 const Evaluator& eval;
254 Index outer;
255 Index length;
256 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE predicate_segment(const Evaluator& evaluator, Index j, Index size)
257 : eval(evaluator), outer(j), length(size) {}
258 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index size() const { return length; }
259 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename Evaluator::CoeffReturnType coeff(Index i) const {
260 return Evaluator::IsRowMajor ? eval.coeff(outer, i) : eval.coeff(i, outer);
261 }
262 template <typename Packet>
263 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(Index i) const {
264 return Evaluator::IsRowMajor ? eval.template packet<Packet>(outer, i) : eval.template packet<Packet>(i, outer);
265 }
266 template <typename Packet>
267 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetSegment(Index i, Index count) const {
268 return Evaluator::IsRowMajor ? eval.template packetSegment<Packet>(outer, i, 0, count)
269 : eval.template packetSegment<Packet>(i, outer, 0, count);
270 }
271};
272
273// Whether a predicate's scalar tests may be grouped; predicates that do not say are tested one at a time.
274template <typename Predicate, typename = void>
275struct predicate_groups_scalars : std::false_type {};
276template <typename Predicate>
277struct predicate_groups_scalars<Predicate, void_t<decltype(functor_traits<Predicate>::GroupScalars)>>
278 : bool_constant<static_cast<bool>(functor_traits<Predicate>::GroupScalars)> {};
279
280// Packets tested between exits on the long path; predicates that do not say test blocks of 8.
281template <typename Predicate, typename = void>
282struct predicate_block_packets : std::integral_constant<int, 8> {};
283template <typename Predicate>
284struct predicate_block_packets<Predicate, void_t<decltype(functor_traits<Predicate>::BlockPackets)>>
285 : std::integral_constant<int, functor_traits<Predicate>::BlockPackets> {};
286
287// Scalar searches OR their tests without a branch per test; spelling a | b on the calls directly draws clang's
288// -Wbitwise-instead-of-logical.
289EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predicate_or(bool a, bool b) { return a | b; }
290
291// At most three coefficients from i on, as straight-line tests.
292template <typename Search, typename Segment, typename Predicate>
293EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool predicate_search_upto3(const Segment& seg, Index i, Predicate pred) {
294 const Index size = seg.size();
295 if (i == size) return false;
296 bool b = Search::test_scalar(pred, seg.coeff(i));
297 if (i + 1 < size) b = predicate_or(b, Search::test_scalar(pred, seg.coeff(i + 1)));
298 if (i + 2 < size) b = predicate_or(b, Search::test_scalar(pred, seg.coeff(i + 2)));
299 return b;
300}
301
302template <typename Search, typename Segment, typename Predicate>
303EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool predicate_search_scalar(const Segment& seg, Index i, Predicate pred) {
304 const Index size = seg.size();
305 for (; i + 3 < size; i += 4) {
306 bool b = predicate_or(
307 predicate_or(Search::test_scalar(pred, seg.coeff(i)), Search::test_scalar(pred, seg.coeff(i + 1))),
308 predicate_or(Search::test_scalar(pred, seg.coeff(i + 2)), Search::test_scalar(pred, seg.coeff(i + 3))));
309 if EIGEN_PREDICT_FALSE (b) return true;
310 }
311 return predicate_search_upto3<Search>(seg, i, pred);
312}
313
314// One coefficient at a time with an exit after each, a shape compilers do not vectorize.
315template <typename Search, typename Segment, typename Predicate>
316EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predicate_search_sequential(const Segment& seg, Index i, Predicate pred) {
317 for (const Index size = seg.size(); i < size; ++i)
318 if (Search::test_scalar(pred, seg.coeff(i))) return true;
319 return false;
320}
321
322// Scalar tests from i on: grouped only where the predicate allows it, since grouped scalar tests may be vectorized
323// onto the unit a predicate excludes (ARMv7 NEON, which flushes subnormals, turns grouped float compares into vceq).
324template <typename Search, typename Segment, typename Predicate>
325EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool predicate_search_tail(const Segment& seg, Index i, Predicate pred) {
326 if (predicate_groups_scalars<Predicate>::value) return predicate_search_scalar<Search>(seg, i, pred);
327 return predicate_search_sequential<Search>(seg, i, pred);
328}
329
330// Scalar blocks OR their tests into an unsigned integer as wide as the scalar. gcc vectorizes that reduction but not an
331// OR of bools, and a wider accumulator widens every test.
332template <typename Scalar, typename Unsigned = typename numext::get_integer_by_size<sizeof(Scalar)>::unsigned_type>
333struct predicate_block_accumulator {
334 using type = Unsigned;
335};
336template <typename Scalar>
337struct predicate_block_accumulator<Scalar, void> {
338 using type = bool;
339};
340
341template <typename Search, typename Packet, bool Vectorize>
342struct predicate_search_segment {
343 static constexpr Index ShortSize = 0;
344 // Without packets, blocks of 64 tests without an exit, which compilers vectorize, and one exit check per block;
345 // starting each block from the identity keeps its trip count regular once inlined. Only where the predicate allows
346 // grouped tests, as for predicate_search_tail.
347 template <typename Segment, typename Predicate>
348 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run(const Segment& seg, Predicate pred) {
349 if (!predicate_groups_scalars<Predicate>::value) return predicate_search_sequential<Search>(seg, 0, pred);
350 const Index size = seg.size();
351 // The shortest segments first, so they skip the block and group guards.
352 if (EIGEN_PREDICT_FALSE(size < 4)) return predicate_search_upto3<Search>(seg, 0, pred);
353 Index i = 0;
354 using Acc = typename predicate_block_accumulator<typename Segment::Scalar>::type;
355 for (; i + 64 <= size; i += 64) {
356 Acc b = 0;
357 for (Index k = 0; k < 64; ++k) b |= Acc(Search::test_scalar(pred, seg.coeff(i + k)));
358 if EIGEN_PREDICT_FALSE (b != Acc(0)) return true;
360 return predicate_search_scalar<Search>(seg, i, pred);
361 }
362 template <typename Segment, typename Predicate>
363 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run_short(const Segment& seg, Predicate pred) {
364 return run(seg, pred);
365 }
366};
367// first(count) is a packet whose first count lanes are all ones and whose other lanes are zero. table is one packet of
368// 0xff bytes followed by one packet of zero bytes; an unaligned load that starts Size - count lanes into it reads count
369// lanes of ones, then zeros. Bytes rather than Scalars, so that one table serves lanes of any width.
370template <typename Packet, typename Bytes = std::make_index_sequence<2 * sizeof(Packet)>>
371struct predicate_lane_table;
372template <typename Packet, std::size_t... Byte>
373struct predicate_lane_table<Packet, std::index_sequence<Byte...>> {
374 using Scalar = typename unpacket_traits<Packet>::type;
375 static constexpr Index Size = unpacket_traits<Packet>::size;
376 // Aligned to the packet, so that every load start is aligned to its Scalar.
377 alignas(sizeof(Packet)) static constexpr unsigned char table[2 * sizeof(Packet)] = {
378 (Byte < sizeof(Packet) ? 0xff : 0)...};
379 static EIGEN_STRONG_INLINE Packet first(Index count) {
380 return ploadu<Packet>(reinterpret_cast<const Scalar*>(table + (Size - count) * sizeof(Scalar)));
381 }
382};
383#if EIGEN_COMP_CXXVER < 17
384template <typename Packet, std::size_t... Byte>
385alignas(sizeof(Packet)) constexpr unsigned char predicate_lane_table<Packet, std::index_sequence<Byte...>>::table[];
386#endif
387
388template <typename Search, typename Packet>
389struct predicate_search_segment<Search, Packet, true> {
390 // One helper per packet, taking its operands by reference and returning the mask by value: if the early inliner
391 // leaves it as a call, the caller needs no addressable packet temporaries.
392 template <typename Segment, typename Predicate>
393 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet test(const Segment& seg, Index i, const Predicate& pred) {
394 return Search::apply(pred, seg.template packet<Packet>(i));
395 }
396 static constexpr Index PacketSize = unpacket_traits<Packet>::size;
397 // Segments shorter than ShortSize take run_short.
398 static constexpr Index ShortSize = 4 * PacketSize;
399
400 // Up to three packets (plus a ragged tail), the common short call: test them directly.
401 template <typename Segment, typename Predicate>
402 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run_short(const Segment& seg, Predicate pred) {
403 const Index size = seg.size();
404 Index i = 0;
405 if (PacketSize <= size) {
406 if (size < 2 * PacketSize) {
407 if (Search::found(test(seg, 0, pred))) return true;
408 i = PacketSize;
409 } else if (size < 3 * PacketSize) {
410 if (Search::found(Search::combine(test(seg, 0, pred), test(seg, PacketSize, pred)))) return true;
411 i = 2 * PacketSize;
412 } else {
413 if (Search::found(Search::combine(Search::combine(test(seg, 0, pred), test(seg, PacketSize, pred)),
414 test(seg, 2 * PacketSize, pred))))
415 return true;
416 i = 3 * PacketSize;
417 }
418 if (EIGEN_PREDICT_TRUE(i == size)) return false;
419 } else if (has_packet_segment<Packet>::value && Segment::PlainLoad && (Segment::FixedSize || size >= 4)) {
420 return run_partial(seg, pred, bool_constant<has_packet_segment<Packet>::value>());
421 } else if (size < 4 && predicate_groups_scalars<Predicate>::value) {
422 // Shorter than one packet and than a group of four: only these sizes reach this check.
423 return predicate_search_upto3<Search>(seg, 0, pred);
424 }
425 return predicate_search_tail<Search>(seg, i, pred);
426 }
427
428 // A segment shorter than one packet, of a compile-time length or of at least four coefficients: one masked load,
429 // with the lanes past the segment masked off. Shorter runtime lengths keep their scalar tests, which are faster than
430 // building the masks; a compile-time length folds them. Only for a segment that loads its coefficients
431 // (Segment::PlainLoad): an expression computes on the zeros past the segment too, and 0 / 0 there would raise
432 // FE_INVALID, which the scalar tests of the same coefficients do not.
433 template <typename Segment, typename Predicate>
434 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run_partial(const Segment& seg, Predicate pred, std::true_type) {
435 const Index size = seg.size();
436 return Search::found_in(Search::apply(pred, seg.template packetSegment<Packet>(0, size)),
437 predicate_lane_table<Packet>::first(size));
438 }
439 template <typename Segment, typename Predicate>
440 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run_partial(const Segment& seg, Predicate pred, std::false_type) {
441 return predicate_search_tail<Search>(seg, 0, pred);
442 }
443
444 template <int Block, typename Segment, typename Predicate>
445 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run_blocks(const Segment& seg, Predicate pred) {
446 const Index size = seg.size();
447 Index i = 0;
448 for (; i + Block * PacketSize <= size; i += Block * PacketSize) {
449 Packet m = test(seg, i, pred);
450 for (int k = 1; k < Block; ++k) m = Search::combine(m, test(seg, i + k * PacketSize, pred));
451 if EIGEN_PREDICT_FALSE (Search::found(m)) return true;
452 }
453 if (Block > 1 && i + PacketSize <= size) {
454 Packet m = test(seg, i, pred);
455 for (i += PacketSize; i + PacketSize <= size; i += PacketSize) m = Search::combine(m, test(seg, i, pred));
456 if (Search::found(m)) return true;
457 }
458 if (EIGEN_PREDICT_TRUE(i == size)) return false;
459 return predicate_search_tail<Search>(seg, i, pred);
461
462 template <typename Segment, typename Predicate>
463 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run(const Segment& seg, Predicate pred) {
464 const Index size = seg.size();
465 if (size < ShortSize) return run_short(seg, pred);
466 EIGEN_IF_CONSTEXPR (predicate_block_packets<Predicate>::value < 8) {
467 return run_blocks<predicate_block_packets<Predicate>::value>(seg, pred);
468 }
469 Index i = 0;
470 // Longer segments: the block loop is laid out off the fall-through path.
471 if (EIGEN_PREDICT_FALSE(8 * PacketSize <= size)) {
472 do {
473 Packet m = Search::combine(
474 Search::combine(Search::combine(test(seg, i + 0 * PacketSize, pred), test(seg, i + 1 * PacketSize, pred)),
475 Search::combine(test(seg, i + 2 * PacketSize, pred), test(seg, i + 3 * PacketSize, pred))),
476 Search::combine(Search::combine(test(seg, i + 4 * PacketSize, pred), test(seg, i + 5 * PacketSize, pred)),
477 Search::combine(test(seg, i + 6 * PacketSize, pred), test(seg, i + 7 * PacketSize, pred))));
478 if EIGEN_PREDICT_FALSE (Search::found(m)) return true;
479 i += 8 * PacketSize;
480 } while (i + 8 * PacketSize <= size);
481 }
482 if (i + 4 * PacketSize <= size) {
483 Packet m =
484 Search::combine(Search::combine(test(seg, i + 0 * PacketSize, pred), test(seg, i + 1 * PacketSize, pred)),
485 Search::combine(test(seg, i + 2 * PacketSize, pred), test(seg, i + 3 * PacketSize, pred)));
486 if (Search::found(m)) return true;
487 i += 4 * PacketSize;
488 }
489 if (i + PacketSize <= size) {
490 Packet m = test(seg, i, pred);
491 for (i += PacketSize; i + PacketSize <= size; i += PacketSize) m = Search::combine(m, test(seg, i, pred));
492 if (Search::found(m)) return true;
493 }
494 if (EIGEN_PREDICT_TRUE(i == size)) return false;
495 return predicate_search_tail<Search>(seg, i, pred);
496 }
497};
498
499template <typename Derived, typename Predicate>
500struct predicate_reduction {
501 using Evaluator = visitor_evaluator<Derived>;
502 using Scalar = typename Derived::Scalar;
503 static constexpr bool LinearAccess = Evaluator::LinearAccess;
504 // A segment's compile-time length, so a short fixed-size segment uses a packet that fits it, as assignment does.
505 static constexpr int SegmentSizeAtCompileTime =
506 LinearAccess ? int(Derived::SizeAtCompileTime)
507 : int(Evaluator::IsRowMajor ? Derived::ColsAtCompileTime : Derived::RowsAtCompileTime);
508 using Packet = typename find_largest_packet<Scalar, SegmentSizeAtCompileTime>::type;
509 static constexpr bool Vectorize = Evaluator::PacketAccess && packet_traits<Scalar>::Vectorizable &&
510 (unpacket_traits<Packet>::size > 1) &&
511 static_cast<bool>(functor_traits<Predicate>::PacketAccess);
512 using Segment = predicate_segment<Evaluator>;
513
514 template <bool Target>
515 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool search(const Derived& xpr, Predicate pred) {
516 using impl = predicate_search_segment<typename predicate_search_for<Target, Predicate>::type, Packet, Vectorize>;
517 Evaluator eval(xpr);
518 EIGEN_IF_CONSTEXPR (LinearAccess) {
519 return impl::run(Segment(eval, 0, eval.size()), pred);
520 } else {
521 // The inner size is read once: through the evaluator's reference to the expression, compilers reload it for
522 // every inner vector.
523 const Index innerSize = Evaluator::IsRowMajor ? eval.cols() : eval.rows();
524 const Index outerSize = Evaluator::IsRowMajor ? eval.rows() : eval.cols();
525 // Choosing the segment path once keeps the length dispatch out of the loop over short inner vectors.
526 if (innerSize < impl::ShortSize) {
527 for (Index j = 0; j < outerSize; ++j)
528 if (impl::run_short(Segment(eval, j, innerSize), pred)) return true;
529 } else {
530 for (Index j = 0; j < outerSize; ++j)
531 if (impl::run(Segment(eval, j, innerSize), pred)) return true;
532 }
533 return false;
534 }
535 }
536};
537
539template <typename Derived, typename Predicate>
540EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool all_of(const Derived& xpr, const Predicate& pred) {
541 return !predicate_reduction<Derived, Predicate>::template search<false>(xpr, pred);
542}
543
545template <typename Derived, typename Predicate>
546EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_of(const Derived& xpr, const Predicate& pred) {
547 return predicate_reduction<Derived, Predicate>::template search<true>(xpr, pred);
548}
549
550// The packet test of fuzzy_constant_predicate holds only where packetOpApplies; elsewhere it tests scalars.
551template <typename Derived, typename Predicate>
552EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool fuzzy_constant_all_of(const Derived& xpr, const Predicate& pred,
553 std::true_type) {
554 if (EIGEN_PREDICT_FALSE(!pred.packetOpApplies())) return all_of(xpr, scalar_predicate<Predicate>{pred});
555 return all_of(xpr, pred);
556}
557template <typename Derived, typename Predicate>
558EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool fuzzy_constant_all_of(const Derived& xpr, const Predicate& pred,
559 std::false_type) {
560 return all_of(xpr, pred);
561}
562
563template <typename Derived, bool AlwaysTrue = NumTraits<typename traits<Derived>::Scalar>::IsInteger>
564struct all_finite_impl {
565 static EIGEN_DEVICE_FUNC inline bool run(const Derived& /*derived*/) { return true; }
566};
567#if !defined(__FINITE_MATH_ONLY__) || !(__FINITE_MATH_ONLY__)
568template <typename Derived>
569struct all_finite_impl<Derived, false> {
570 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool run(const Derived& derived) {
571 return all_of(derived, isfinite_predicate<typename traits<Derived>::Scalar>());
572 }
573};
574#endif
575
576} // end namespace internal
577
585template <typename Derived>
586EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool DenseBase<Derived>::all() const {
587 return internal::all_of(derived(), internal::nonzero_predicate<Scalar>());
588}
589
594template <typename Derived>
595EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool DenseBase<Derived>::any() const {
596 return internal::any_of(derived(), internal::nonzero_predicate<Scalar>());
597}
598
599template <typename Derived>
600EIGEN_DEVICE_FUNC inline bool DenseBase<Derived>::hasNaN() const {
601 return derived().cwiseTypedNotEqual(derived()).any();
602}
603
608template <typename Derived>
609EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool DenseBase<Derived>::allFinite() const {
610 return internal::all_finite_impl<Derived>::run(derived());
611}
612
614template <typename Derived>
615EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool DenseBase<Derived>::isApproxToConstant(const Scalar& val,
616 const RealScalar& prec) const {
617 using Predicate = internal::fuzzy_constant_predicate<Scalar, true>;
618 return internal::fuzzy_constant_all_of(
619 derived(), Predicate{val, prec},
620 std::integral_constant<bool, internal::functor_traits<Predicate>::PacketAccess>());
621}
622
631template <typename Derived>
632EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool DenseBase<Derived>::isZero(const RealScalar& prec) const {
633 return internal::all_of(derived(), internal::fuzzy_constant_predicate<Scalar, false>{Scalar(0), prec});
634}
635
636} // end namespace Eigen
637
638#endif // EIGEN_PREDICATOR_H
bool isApproxToConstant(const Scalar &value, const RealScalar &prec=NumTraits< Scalar >::dummy_precision()) const
Definition Predicator.h:615
bool isZero(const RealScalar &prec=NumTraits< Scalar >::dummy_precision()) const
Definition Predicator.h:632
bool all() const
Definition Predicator.h:586
bool any() const
Definition Predicator.h:595
bool allFinite() const
Definition Predicator.h:609
typename internal::traits< ArrayWrapper< ExpressionType > >::Scalar Scalar
Definition DenseBase.h:63