Visitor.h

// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
//
// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
// SPDX-License-Identifier: MPL-2.0
 
#ifndef EIGEN_VISITOR_H
#define EIGEN_VISITOR_H
 
// IWYU pragma: private
#include "./InternalHeaderCheck.h"
 
namespace Eigen {
 
namespace internal {
 
// Opt in only when coefficient updates are valid without init/initpacket.
template <typename Visitor, typename = void>
struct visitor_already_initialized : std::false_type {};
template <typename Visitor>
struct visitor_already_initialized<Visitor, void_t<decltype(functor_traits<Visitor>::AlreadyInitialized)>>
    : bool_constant<functor_traits<Visitor>::AlreadyInitialized> {};
 
template <typename Visitor, typename Derived, int UnrollCount,
          bool Vectorize = (Derived::PacketAccess && functor_traits<Visitor>::PacketAccess), bool LinearAccess = false,
          bool ShortCircuitEvaluation = false>
struct visitor_impl;
 
template <typename Visitor, bool ShortCircuitEvaluation = false>
struct short_circuit_eval_impl {
  // if short circuit evaluation is not used, do nothing
  static constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(const Visitor&) { return false; }
};
template <typename Visitor>
struct short_circuit_eval_impl<Visitor, true> {
  // if short circuit evaluation is used, check the visitor
  static constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(const Visitor& visitor) { return visitor.done(); }
};
 
// unrolled inner-outer traversal
template <typename Visitor, typename Derived, int UnrollCount, bool Vectorize, bool ShortCircuitEvaluation>
struct visitor_impl<Visitor, Derived, UnrollCount, Vectorize, false, ShortCircuitEvaluation> {
  // don't use short circuit evaluation for unrolled version
  using Scalar = typename Derived::Scalar;
  using Packet = typename packet_traits<Scalar>::type;
  static constexpr bool RowMajor = Derived::IsRowMajor;
  static constexpr int RowsAtCompileTime = Derived::RowsAtCompileTime;
  static constexpr int ColsAtCompileTime = Derived::ColsAtCompileTime;
  static constexpr int PacketSize = packet_traits<Scalar>::size;
  static constexpr int InnerSizeAtCompileTime = RowMajor ? ColsAtCompileTime : RowsAtCompileTime;
  static constexpr int OuterSizeAtCompileTime = RowMajor ? RowsAtCompileTime : ColsAtCompileTime;
  static constexpr int PacketOpsPerOuter =
      Vectorize && InnerSizeAtCompileTime >= PacketSize ? InnerSizeAtCompileTime / PacketSize : 0;
  static constexpr int FirstScalarInner = PacketOpsPerOuter * PacketSize;
  static constexpr int ScalarOpsPerOuter = InnerSizeAtCompileTime - FirstScalarInner;
  static constexpr int OpsPerOuter = PacketOpsPerOuter + ScalarOpsPerOuter;
  static constexpr int OpCount = UnrollCount == 0 ? 0 : OuterSizeAtCompileTime * OpsPerOuter;
 
  template <int Op>
  static constexpr bool IsPacketOp() {
    return OpsPerOuter != 0 && (Op % OpsPerOuter) < PacketOpsPerOuter;
  }
 
  template <int Op>
  static constexpr int CoeffIndex() {
    return OpsPerOuter == 0 ? 0
                            : (Op / OpsPerOuter) * InnerSizeAtCompileTime +
                                  (IsPacketOp<Op>() ? ((Op % OpsPerOuter) * PacketSize)
                                                    : (FirstScalarInner + (Op % OpsPerOuter) - PacketOpsPerOuter));
  }
 
  template <int Op, std::enable_if_t<!IsPacketOp<Op>(), bool> = true>
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void visit(const Derived& mat, Visitor& visitor) {
    constexpr int K = CoeffIndex<Op>();
    constexpr int R = RowMajor ? (K / ColsAtCompileTime) : (K % RowsAtCompileTime);
    constexpr int C = RowMajor ? (K % ColsAtCompileTime) : (K / RowsAtCompileTime);
    EIGEN_IF_CONSTEXPR (Op == 0) {
      visitor.init(mat.coeff(R, C), R, C);
    } else {
      visitor(mat.coeff(R, C), R, C);
    }
  }
 
  template <int Op, std::enable_if_t<IsPacketOp<Op>(), bool> = true>
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void visit(const Derived& mat, Visitor& visitor) {
    constexpr int K = CoeffIndex<Op>();
    constexpr int R = RowMajor ? (K / ColsAtCompileTime) : (K % RowsAtCompileTime);
    constexpr int C = RowMajor ? (K % ColsAtCompileTime) : (K / RowsAtCompileTime);
    Packet P = mat.template packet<Packet>(R, C);
    EIGEN_IF_CONSTEXPR (Op == 0) {
      visitor.initpacket(P, R, C);
    } else {
      visitor.packet(P, R, C);
    }
  }
 
  template <int... Ops>
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run_impl(const Derived& mat, Visitor& visitor,
                                                             std::integer_sequence<int, Ops...>) {
    int unused[] = {0, (visit<Ops>(mat, visitor), 0)...};
    EIGEN_UNUSED_VARIABLE(unused);
  }
 
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
    run_impl(mat, visitor, std::make_integer_sequence<int, OpCount>{});
  }
};
 
// unrolled linear traversal
template <typename Visitor, typename Derived, int UnrollCount, bool Vectorize, bool ShortCircuitEvaluation>
struct visitor_impl<Visitor, Derived, UnrollCount, Vectorize, true, ShortCircuitEvaluation> {
  // don't use short circuit evaluation for unrolled version
  using Scalar = typename Derived::Scalar;
  using Packet = typename packet_traits<Scalar>::type;
  static constexpr int PacketSize = packet_traits<Scalar>::size;
  static constexpr int PacketOps = Vectorize ? UnrollCount / PacketSize : 0;
  static constexpr int FirstScalar = PacketOps * PacketSize;
  static constexpr int ScalarOps = UnrollCount - FirstScalar;
  static constexpr int OpCount = PacketOps + ScalarOps;
 
  template <int Op>
  static constexpr bool IsPacketOp() {
    return Op < PacketOps;
  }
 
  template <int Op>
  static constexpr int CoeffIndex() {
    return IsPacketOp<Op>() ? (Op * PacketSize) : (FirstScalar + Op - PacketOps);
  }
 
  template <int Op, std::enable_if_t<!IsPacketOp<Op>(), bool> = true>
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void visit(const Derived& mat, Visitor& visitor) {
    constexpr int K = CoeffIndex<Op>();
    EIGEN_IF_CONSTEXPR (Op == 0) {
      visitor.init(mat.coeff(K), K);
    } else {
      visitor(mat.coeff(K), K);
    }
  }
 
  template <int Op, std::enable_if_t<IsPacketOp<Op>(), bool> = true>
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void visit(const Derived& mat, Visitor& visitor) {
    constexpr int K = CoeffIndex<Op>();
    Packet P = mat.template packet<Packet>(K);
    EIGEN_IF_CONSTEXPR (Op == 0) {
      visitor.initpacket(P, K);
    } else {
      visitor.packet(P, K);
    }
  }
 
  template <int... Ops>
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run_impl(const Derived& mat, Visitor& visitor,
                                                             std::integer_sequence<int, Ops...>) {
    int unused[] = {0, (visit<Ops>(mat, visitor), 0)...};
    EIGEN_UNUSED_VARIABLE(unused);
  }
 
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
    run_impl(mat, visitor, std::make_integer_sequence<int, OpCount>{});
  }
};
 
// dynamic scalar outer-inner traversal
template <typename Visitor, typename Derived, bool ShortCircuitEvaluation>
struct visitor_impl<Visitor, Derived, Dynamic, /*Vectorize=*/false, /*LinearAccess=*/false, ShortCircuitEvaluation> {
  using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>;
  static constexpr bool RowMajor = Derived::IsRowMajor;
 
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
    const Index innerSize = RowMajor ? mat.cols() : mat.rows();
    const Index outerSize = RowMajor ? mat.rows() : mat.cols();
    if (innerSize == 0 || outerSize == 0) return;
    {
      visitor.init(mat.coeff(0, 0), 0, 0);
      if (short_circuit::run(visitor)) return;
      for (Index i = 1; i < innerSize; ++i) {
        Index r = RowMajor ? 0 : i;
        Index c = RowMajor ? i : 0;
        visitor(mat.coeff(r, c), r, c);
        if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
      }
    }
    for (Index j = 1; j < outerSize; j++) {
      for (Index i = 0; i < innerSize; ++i) {
        Index r = RowMajor ? j : i;
        Index c = RowMajor ? i : j;
        visitor(mat.coeff(r, c), r, c);
        if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
      }
    }
  }
};
 
// dynamic vectorized outer-inner traversal
template <typename Visitor, typename Derived, bool ShortCircuitEvaluation>
struct visitor_impl<Visitor, Derived, Dynamic, /*Vectorize=*/true, /*LinearAccess=*/false, ShortCircuitEvaluation> {
  using Scalar = typename Derived::Scalar;
  using Packet = typename packet_traits<Scalar>::type;
  static constexpr int PacketSize = packet_traits<Scalar>::size;
  using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>;
  static constexpr bool RowMajor = Derived::IsRowMajor;
  static constexpr bool AlreadyInitialized = visitor_already_initialized<Visitor>::value;
 
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
    const Index innerSize = RowMajor ? mat.cols() : mat.rows();
    const Index outerSize = RowMajor ? mat.rows() : mat.cols();
    if (innerSize == 0 || outerSize == 0) return;
    EIGEN_IF_CONSTEXPR (!AlreadyInitialized) {
      Index i = 0;
      if (innerSize < PacketSize) {
        visitor.init(mat.coeff(0, 0), 0, 0);
        i = 1;
      } else {
        Packet p = mat.template packet<Packet>(0, 0);
        visitor.initpacket(p, 0, 0);
        i = PacketSize;
      }
      if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
      for (; i + PacketSize - 1 < innerSize; i += PacketSize) {
        Index r = RowMajor ? 0 : i;
        Index c = RowMajor ? i : 0;
        Packet p = mat.template packet<Packet>(r, c);
        visitor.packet(p, r, c);
        if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
      }
      for (; i < innerSize; ++i) {
        Index r = RowMajor ? 0 : i;
        Index c = RowMajor ? i : 0;
        visitor(mat.coeff(r, c), r, c);
        if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
      }
    }
    const Index packetEnd = innerSize - innerSize % PacketSize;
    for (Index j = AlreadyInitialized ? 0 : 1; j < outerSize; j++) {
      Index i = 0;
      for (; AlreadyInitialized ? i < packetEnd : i + PacketSize - 1 < innerSize; i += PacketSize) {
        Index r = RowMajor ? j : i;
        Index c = RowMajor ? i : j;
        Packet p = mat.template packet<Packet>(r, c);
        visitor.packet(p, r, c);
        if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
      }
      for (; i < innerSize; ++i) {
        Index r = RowMajor ? j : i;
        Index c = RowMajor ? i : j;
        visitor(mat.coeff(r, c), r, c);
        if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
      }
    }
  }
};
 
// dynamic scalar linear traversal
template <typename Visitor, typename Derived, bool ShortCircuitEvaluation>
struct visitor_impl<Visitor, Derived, Dynamic, /*Vectorize=*/false, /*LinearAccess=*/true, ShortCircuitEvaluation> {
  using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>;
 
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
    const Index size = mat.size();
    if (size == 0) return;
    visitor.init(mat.coeff(0), 0);
    if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
    for (Index k = 1; k < size; k++) {
      visitor(mat.coeff(k), k);
      if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
    }
  }
};
 
// dynamic vectorized linear traversal
template <typename Visitor, typename Derived, bool ShortCircuitEvaluation>
struct visitor_impl<Visitor, Derived, Dynamic, /*Vectorize=*/true, /*LinearAccess=*/true, ShortCircuitEvaluation> {
  using Scalar = typename Derived::Scalar;
  using Packet = typename packet_traits<Scalar>::type;
  static constexpr int PacketSize = packet_traits<Scalar>::size;
  using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>;
  static constexpr bool AlreadyInitialized = visitor_already_initialized<Visitor>::value;
 
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
    const Index size = mat.size();
    if (size == 0) return;
    const Index packetEnd = size - size % PacketSize;
    Index k = 0;
    EIGEN_IF_CONSTEXPR (!AlreadyInitialized) {
      if (size < PacketSize) {
        visitor.init(mat.coeff(0), 0);
        k = 1;
      } else {
        Packet p = mat.template packet<Packet>(k);
        visitor.initpacket(p, k);
        k = PacketSize;
      }
      if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
    }
    for (; k < packetEnd; k += PacketSize) {
      Packet p = mat.template packet<Packet>(k);
      visitor.packet(p, k);
      if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
    }
    for (; k < size; k++) {
      visitor(mat.coeff(k), k);
      if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
    }
  }
};
 
// evaluator adaptor
template <typename XprType>
class visitor_evaluator {
 public:
  using Evaluator = evaluator<XprType>;
  using Scalar = typename XprType::Scalar;
  using Packet = typename packet_traits<Scalar>::type;
  using CoeffReturnType = std::remove_const_t<typename XprType::CoeffReturnType>;
 
  static constexpr bool PacketAccess = static_cast<bool>(Evaluator::Flags & PacketAccessBit);
  static constexpr bool LinearAccess = static_cast<bool>(Evaluator::Flags & LinearAccessBit);
  static constexpr bool IsRowMajor = static_cast<bool>(XprType::IsRowMajor);
  static constexpr int RowsAtCompileTime = XprType::RowsAtCompileTime;
  static constexpr int ColsAtCompileTime = XprType::ColsAtCompileTime;
  static constexpr int XprAlignment = Evaluator::Alignment;
  static constexpr int CoeffReadCost = Evaluator::CoeffReadCost;
 
  EIGEN_DEVICE_FUNC explicit visitor_evaluator(const XprType& xpr) : m_evaluator(xpr), m_xpr(xpr) {}
 
  EIGEN_DEVICE_FUNC constexpr Index rows() const noexcept { return m_xpr.rows(); }
  EIGEN_DEVICE_FUNC constexpr Index cols() const noexcept { return m_xpr.cols(); }
  EIGEN_DEVICE_FUNC constexpr Index size() const noexcept { return m_xpr.size(); }
  // outer-inner access
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
    return m_evaluator.coeff(row, col);
  }
  template <typename Packet, int Alignment = Unaligned>
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(Index row, Index col) const {
    return m_evaluator.template packet<Alignment, Packet>(row, col);
  }
  // linear access
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_evaluator.coeff(index); }
  template <typename Packet, int Alignment = XprAlignment>
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(Index index) const {
    return m_evaluator.template packet<Alignment, Packet>(index);
  }
 
 protected:
  Evaluator m_evaluator;
  const XprType& m_xpr;
};
 
template <typename T, typename = void>
struct visitor_has_linear_access : std::false_type {};
 
template <typename T>
struct visitor_has_linear_access<T, void_t<decltype(functor_traits<T>::LinearAccess)>>
    : bool_constant<static_cast<bool>(functor_traits<T>::LinearAccess)> {};
 
template <typename Derived, typename Visitor, bool ShortCircuitEvaluation>
struct visit_impl {
  using Evaluator = visitor_evaluator<Derived>;
  using Scalar = typename DenseBase<Derived>::Scalar;
 
  static constexpr bool IsRowMajor = DenseBase<Derived>::IsRowMajor;
  static constexpr int SizeAtCompileTime = DenseBase<Derived>::SizeAtCompileTime;
  static constexpr int RowsAtCompileTime = DenseBase<Derived>::RowsAtCompileTime;
  static constexpr int ColsAtCompileTime = DenseBase<Derived>::ColsAtCompileTime;
  static constexpr int InnerSizeAtCompileTime = IsRowMajor ? ColsAtCompileTime : RowsAtCompileTime;
  static constexpr int OuterSizeAtCompileTime = IsRowMajor ? RowsAtCompileTime : ColsAtCompileTime;
 
  // Linear packets can make an early scalar short-circuit exit more expensive.
  // Preinitialized visitors opt into starting directly with packet traversal.
  static constexpr bool LinearAccess = (!ShortCircuitEvaluation || visitor_already_initialized<Visitor>::value) &&
                                       Evaluator::LinearAccess && visitor_has_linear_access<Visitor>::value;
  static constexpr bool Vectorize = Evaluator::PacketAccess && static_cast<bool>(functor_traits<Visitor>::PacketAccess);
 
  static constexpr int PacketSize = packet_traits<Scalar>::size;
  static constexpr int VectorOps =
      Vectorize ? (LinearAccess ? (SizeAtCompileTime / PacketSize)
                                : (OuterSizeAtCompileTime * (InnerSizeAtCompileTime / PacketSize)))
                : 0;
  static constexpr int ScalarOps = SizeAtCompileTime - (VectorOps * PacketSize);
  // treat vector op and scalar op as same cost for unroll logic
  static constexpr int TotalOps = VectorOps + ScalarOps;
 
  static constexpr int UnrollCost = int(Evaluator::CoeffReadCost) + int(functor_traits<Visitor>::Cost);
  static constexpr bool Unroll = (SizeAtCompileTime != Dynamic) && ((TotalOps * UnrollCost) <= EIGEN_UNROLLING_LIMIT);
  static constexpr int UnrollCount = Unroll ? int(SizeAtCompileTime) : Dynamic;
 
  using impl = visitor_impl<Visitor, Evaluator, UnrollCount, Vectorize, LinearAccess, ShortCircuitEvaluation>;
 
  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const DenseBase<Derived>& mat, Visitor& visitor) {
    Evaluator evaluator(mat.derived());
    impl::run(evaluator, visitor);
  }
};
 
}  // end namespace internal

template <typename Derived>
template <typename Visitor>
EIGEN_DEVICE_FUNC void DenseBase<Derived>::visit(Visitor& visitor) const {
  using impl = internal::visit_impl<Derived, Visitor, /*ShortCircuitEvaluation*/ false>;
  impl::run(derived(), visitor);
}
 
namespace internal {
 
template <typename Scalar, bool Approximate>
struct fuzzy_constant_visitor {
  Scalar value;
  typename NumTraits<Scalar>::Real precision;
  bool result = true;
 
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(const Scalar& x, Index, Index = 0) {
    result = result && (Approximate ? internal::isApprox(x, value, precision)
                                    : internal::isMuchSmallerThan(x, Scalar(1), precision));
  }
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void init(const Scalar& x, Index r, Index c = 0) { (*this)(x, r, c); }
  EIGEN_DEVICE_FUNC bool done() const { return !result; }
  template <typename Packet>
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void packet(const Packet& x, Index, Index = 0) {
    const Packet p = pset1<Packet>(precision);
    Packet mask;
    EIGEN_IF_CONSTEXPR (Approximate) {
      const Packet v = pset1<Packet>(value);
      mask = pcmp_le(pabs(psub(x, v)), pmul(pmin(pabs(x), pabs(v)), p));
    } else {
      mask = pcmp_le(pabs(x), p);
    }
    // Reduce the comparison mask directly, without comparing its lanes to zero again.
    result = result && !predux_any(pandnot(ptrue(mask), mask));
  }
  template <typename Packet>
  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void initpacket(const Packet& x, Index r, Index c = 0) {
    packet(x, r, c);
  }
};
 
template <typename Scalar, bool Approximate>
struct functor_traits<fuzzy_constant_visitor<Scalar, Approximate>> {
  static constexpr bool AlreadyInitialized = true;
  static constexpr bool LinearAccess = true;
  static constexpr int Cost = 4 * NumTraits<Scalar>::AddCost + NumTraits<Scalar>::MulCost;
  // ARMv7 NEON flushes subnormal float operands and results; scalar VFP does not.
  // Explicitly flushing would change exact-zero checks; normal operands can also have a subnormal difference.
  static constexpr bool PacketAccess =
      (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value) &&
      !(EIGEN_ARCH_ARM && std::is_same<Scalar, float>::value) && packet_traits<Scalar>::HasAbs &&
      packet_traits<Scalar>::HasCmp &&
      (!Approximate ||
       (packet_traits<Scalar>::HasSub && packet_traits<Scalar>::HasMin && packet_traits<Scalar>::HasMul));
};
 
template <typename Scalar>
using use_fuzzy_constant_visitor =
    bool_constant<std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value>;
 
template <bool Approximate, typename Derived,
          std::enable_if_t<use_fuzzy_constant_visitor<typename Derived::Scalar>::value, int> = 0>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool fuzzy_constant_all(const Derived& matrix,
                                                              const typename Derived::Scalar& value,
                                                              const typename Derived::RealScalar& precision) {
  fuzzy_constant_visitor<typename Derived::Scalar, Approximate> visitor{value, precision};
  visit_impl<Derived, decltype(visitor), true>::run(matrix, visitor);
  return visitor.result;
}
 
// Separate overloads keep custom scalars from instantiating unused comparisons in C++14.
template <bool Approximate, typename Derived,
          std::enable_if_t<!use_fuzzy_constant_visitor<typename Derived::Scalar>::value && Approximate, int> = 0>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool fuzzy_constant_all(const Derived& matrix,
                                                              const typename Derived::Scalar& value,
                                                              const typename Derived::RealScalar& precision) {
  for (Index j = 0; j < matrix.cols(); ++j)
    for (Index i = 0; i < matrix.rows(); ++i)
      if (!internal::isApprox(matrix.coeff(i, j), value, precision)) return false;
  return true;
}
 
template <bool Approximate, typename Derived,
          std::enable_if_t<!use_fuzzy_constant_visitor<typename Derived::Scalar>::value && !Approximate, int> = 0>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool fuzzy_constant_all(const Derived& matrix, const typename Derived::Scalar&,
                                                              const typename Derived::RealScalar& precision) {
  using Scalar = typename Derived::Scalar;
  for (Index j = 0; j < matrix.cols(); ++j)
    for (Index i = 0; i < matrix.rows(); ++i)
      if (!internal::isMuchSmallerThan(matrix.coeff(i, j), Scalar(1), precision)) return false;
  return true;
}
 
template <typename Scalar>
struct all_visitor {
  using result_type = bool;
  using Packet = typename packet_traits<Scalar>::type;
  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = (value != Scalar(0)); }
  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = (value != Scalar(0)); }
  EIGEN_DEVICE_FUNC inline bool all_predux(const Packet& p) const { return predux_all(p); }
  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = all_predux(p); }
  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = all_predux(p); }
  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) { res = res && (value != Scalar(0)); }
  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index) { res = res && (value != Scalar(0)); }
  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index, Index) { res = res && all_predux(p); }
  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index) { res = res && all_predux(p); }
  EIGEN_DEVICE_FUNC inline bool done() const { return !res; }
  bool res = true;
};
template <typename Scalar>
struct functor_traits<all_visitor<Scalar>> {
  enum { Cost = NumTraits<Scalar>::ReadCost, LinearAccess = true, PacketAccess = packet_traits<Scalar>::HasCmp };
};
 
template <typename Scalar>
struct any_visitor {
  using result_type = bool;
  using Packet = typename packet_traits<Scalar>::type;
  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = (value != Scalar(0)); }
  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = (value != Scalar(0)); }
  EIGEN_DEVICE_FUNC inline bool any_predux(const Packet& p) const {
    return predux_any(pandnot(ptrue(p), pcmp_eq(p, pzero(p))));
  }
  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = any_predux(p); }
  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = any_predux(p); }
  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) { res = res || (value != Scalar(0)); }
  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index) { res = res || (value != Scalar(0)); }
  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index, Index) { res = res || any_predux(p); }
  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index) { res = res || any_predux(p); }
  EIGEN_DEVICE_FUNC inline bool done() const { return res; }
  bool res = false;
};
template <>
EIGEN_DEVICE_FUNC inline bool any_visitor<bool>::any_predux(const Packet& p) const {
  return predux(p);
}
template <typename Scalar>
struct functor_traits<any_visitor<Scalar>> {
  enum { Cost = NumTraits<Scalar>::ReadCost, LinearAccess = true, PacketAccess = packet_traits<Scalar>::HasCmp };
};
 
template <typename Scalar>
struct count_visitor {
  using result_type = Index;
  using Packet = typename packet_traits<Scalar>::type;
  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = value != Scalar(0) ? 1 : 0; }
  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = value != Scalar(0) ? 1 : 0; }
  EIGEN_DEVICE_FUNC inline Index count_redux(const Packet& p) const { return predux_count(p); }
  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = count_redux(p); }
  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = count_redux(p); }
  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) {
    if (value != Scalar(0)) res++;
  }
  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index) {
    if (value != Scalar(0)) res++;
  }
  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index, Index) { res += count_redux(p); }
  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index) { res += count_redux(p); }
  Index res = 0;
};
 
template <typename Scalar>
struct functor_traits<count_visitor<Scalar>> {
  enum {
    Cost = NumTraits<Scalar>::AddCost,
    LinearAccess = true,
    PacketAccess = packet_traits<Scalar>::HasCmp && packet_traits<Scalar>::HasAdd
  };
};
 
// Reduces pisfinite masks directly: their lanes are all-ones or zero, so the coefficients are all finite exactly when
// no lane of the complement is set. predux_all, which all() on isFiniteTyped() uses, compares each lane against zero
// instead, which a compiler can only fold away when it can prove the operand is a mask.
template <typename Scalar>
struct all_finite_visitor {
  using result_type = bool;
  using Packet = typename packet_traits<Scalar>::type;
  EIGEN_DEVICE_FUNC inline bool finite_predux(const Packet& p) const { return !predux_any(pnot(pisfinite(p))); }
  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = (numext::isfinite)(value); }
  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = (numext::isfinite)(value); }
  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = finite_predux(p); }
  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = finite_predux(p); }
  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) {
    res = res && (numext::isfinite)(value);
  }
  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index) { res = res && (numext::isfinite)(value); }
  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index, Index) { res = res && finite_predux(p); }
  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index) { res = res && finite_predux(p); }
  EIGEN_DEVICE_FUNC inline bool done() const { return !res; }
  bool res = true;
};
template <typename Scalar>
struct functor_traits<all_finite_visitor<Scalar>> {
  enum {
    Cost = NumTraits<Scalar>::ReadCost + NumTraits<Scalar>::MulCost,
    LinearAccess = true,
    PacketAccess = packet_traits<Scalar>::HasCmp
  };
};
 
template <typename Derived, bool AlwaysTrue = NumTraits<typename traits<Derived>::Scalar>::IsInteger>
struct all_finite_impl {
  static EIGEN_DEVICE_FUNC inline bool run(const Derived& /*derived*/) { return true; }
};
#if !defined(__FINITE_MATH_ONLY__) || !(__FINITE_MATH_ONLY__)
template <typename Derived>
struct all_finite_impl<Derived, false> {
  static EIGEN_DEVICE_FUNC inline bool run(const Derived& derived) {
    using Visitor = all_finite_visitor<typename traits<Derived>::Scalar>;
    Visitor visitor;
    visit_impl<Derived, Visitor, /*ShortCircuitEvaluation*/ true>::run(derived, visitor);
    return visitor.res;
  }
};
#endif
 
}  // end namespace internal

template <typename Derived>
EIGEN_DEVICE_FUNC inline bool DenseBase<Derived>::all() const {
  using Visitor = internal::all_visitor<Scalar>;
  using impl = internal::visit_impl<Derived, Visitor, /*ShortCircuitEvaluation*/ true>;
  Visitor visitor;
  impl::run(derived(), visitor);
  return visitor.res;
}

template <typename Derived>
EIGEN_DEVICE_FUNC inline bool DenseBase<Derived>::any() const {
  using Visitor = internal::any_visitor<Scalar>;
  using impl = internal::visit_impl<Derived, Visitor, /*ShortCircuitEvaluation*/ true>;
  Visitor visitor;
  impl::run(derived(), visitor);
  return visitor.res;
}

template <typename Derived>
EIGEN_DEVICE_FUNC Index DenseBase<Derived>::count() const {
  using Visitor = internal::count_visitor<Scalar>;
  using impl = internal::visit_impl<Derived, Visitor, /*ShortCircuitEvaluation*/ false>;
  Visitor visitor;
  impl::run(derived(), visitor);
  return visitor.res;
}
 
template <typename Derived>
EIGEN_DEVICE_FUNC inline bool DenseBase<Derived>::hasNaN() const {
  return derived().cwiseTypedNotEqual(derived()).any();
}

template <typename Derived>
EIGEN_DEVICE_FUNC inline bool DenseBase<Derived>::allFinite() const {
  return internal::all_finite_impl<Derived>::run(derived());
}

template <typename Derived>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool DenseBase<Derived>::isApproxToConstant(const Scalar& val,
                                                                                  const RealScalar& prec) const {
  typename internal::nested_eval<Derived, 1>::type self(derived());
  return internal::fuzzy_constant_all<true>(self, val, prec);
}

template <typename Derived>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool DenseBase<Derived>::isZero(const RealScalar& prec) const {
  typename internal::nested_eval<Derived, 1>::type self(derived());
  return internal::fuzzy_constant_all<false>(self, Scalar(0), prec);
}
 
}  // end namespace Eigen
 
#endif  // EIGEN_VISITOR_H