15#include "./InternalHeaderCheck.h"
22template <
typename RealScalar>
23struct stable_norm_accumulator {
24 using type = RealScalar;
28struct stable_norm_accumulator<half> {
33struct stable_norm_accumulator<bfloat16> {
37template <
typename RealScalar,
typename Accumulator>
38struct stable_normalization_normal_min {
39 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Accumulator run() {
40 return static_cast<Accumulator
>((numext::numeric_limits<RealScalar>::min)());
45template <
typename Accumulator>
46struct stable_normalization_normal_min<half, Accumulator> {
47 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Accumulator run() {
return Accumulator(1) / Accumulator(16384); }
50template <
typename Accumulator>
51struct stable_normalization_normal_min<bfloat16, Accumulator> {
52 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Accumulator run() {
53 return static_cast<Accumulator
>((numext::numeric_limits<float>::min)());
57template <
typename RealScalar,
typename Accumulator>
58EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool stable_normalization_inv_scale(
const Accumulator& value,
59 Accumulator& invScale) {
60 safe_scaling_factors<Accumulator> factors;
61 const Accumulator normalMin = stable_normalization_normal_min<RealScalar, Accumulator>::run();
62 if (!safe_scaling<Accumulator>::try_compute_ceiling_factors_with_normal_reciprocal(value, normalMin, factors))
64 invScale = factors.invScale;
68template <
typename Accumulator>
69EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool stable_normalization_combined_factor(
const Accumulator& invScale,
70 const Accumulator& sqrtNorm,
71 Accumulator& factor) {
73 if (sqrtNorm < Accumulator(1) && invScale > Accumulator(NumTraits<Accumulator>::highest()) * sqrtNorm)
return false;
74 Accumulator localSqrtNorm = sqrtNorm;
75 pbarrier(localSqrtNorm);
76 factor = invScale / localSqrtNorm;
77 return factor >= stable_normalization_normal_min<Accumulator, Accumulator>::run();
80template <
typename VectorType,
typename Accumulator,
82 (int(inner_stride_at_compile_time<VectorType>::value) != 1)>
83struct stable_normalization_dispatch {
84 using Scalar =
typename traits<VectorType>::Scalar;
85 using RealScalar =
typename NumTraits<Scalar>::Real;
87 using HasWritableRealView = bool_constant<!NumTraits<Scalar>::IsComplex || complex_array_access<Scalar>::value>;
89 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Accumulator max_abs(
const VectorType& vec) {
90 return redux_with_identity(vec.realView().template cast<Accumulator>().cwiseAbs(),
91 scalar_max_op<Accumulator, Accumulator, PropagateNaN>(), 0);
94 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Accumulator scaled_squared_norm(
const VectorType& vec,
95 const Accumulator& factor) {
96 return (vec.realView().template cast<Accumulator>() * factor).squaredNorm();
99 template <
typename ResultType>
100 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void assign_scaled(ResultType& result,
const VectorType& vec,
101 const Accumulator& factor) {
102 assign_scaled_impl(result, vec, factor, HasWritableRealView());
105 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void scale_in_place(VectorType& vec,
const Accumulator& factor) {
106 assign_scaled_impl(vec, vec, factor, HasWritableRealView());
110 template <
typename ResultType>
111 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void assign_scaled_impl(ResultType& result,
const VectorType& vec,
112 const Accumulator& factor, std::true_type) {
113 result.realView() = (vec.realView().
template cast<Accumulator>() * factor).template cast<RealScalar>();
116 template <
typename ResultType>
117 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void assign_scaled_impl(ResultType& result,
const VectorType& vec,
118 const Accumulator& factor, std::false_type) {
119 result = vec *
static_cast<RealScalar
>(factor);
124template <
typename VectorType,
typename Accumulator>
125struct stable_normalization_dispatch<VectorType, Accumulator, true> {
126 using Scalar =
typename traits<VectorType>::Scalar;
127 using RealScalar =
typename NumTraits<Scalar>::Real;
129 using PlainVector = Matrix<Scalar, Dynamic, 1>;
130 using ConstContiguousMap = Map<const PlainVector, evaluator<VectorType>::Alignment>;
131 using ContiguousMap = Map<PlainVector, evaluator<VectorType>::Alignment>;
133 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
bool is_contiguous(
const VectorType& vec) {
134 return vec.innerStride() == 1 && (vec.outerSize() == 1 || vec.outerStride() == vec.innerSize());
137 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Accumulator max_abs(
const VectorType& vec) {
138 if (is_contiguous(vec)) {
139 const ConstContiguousMap contiguous(vec.data(), vec.size());
140 return stable_normalization_dispatch<ConstContiguousMap, Accumulator, false>::max_abs(contiguous);
142 return stable_normalization_dispatch<VectorType, Accumulator, false>::max_abs(vec);
145 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Accumulator scaled_squared_norm(
const VectorType& vec,
146 const Accumulator& factor) {
147 if (is_contiguous(vec)) {
148 const ConstContiguousMap contiguous(vec.data(), vec.size());
149 return stable_normalization_dispatch<ConstContiguousMap, Accumulator, false>::scaled_squared_norm(contiguous,
152 return stable_normalization_dispatch<VectorType, Accumulator, false>::scaled_squared_norm(vec, factor);
155 template <
typename ResultType>
156 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void assign_scaled(ResultType& result,
const VectorType& vec,
157 const Accumulator& factor) {
158 if (is_contiguous(vec)) {
159 const ConstContiguousMap contiguous(vec.data(), vec.size());
160 Map<PlainVector> output(result.data(), result.size());
161 stable_normalization_dispatch<ConstContiguousMap, Accumulator, false>::assign_scaled(output, contiguous, factor);
164 stable_normalization_dispatch<VectorType, Accumulator, false>::assign_scaled(result, vec, factor);
167 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void scale_in_place(VectorType& vec,
const Accumulator& factor) {
168 if (is_contiguous(vec)) {
169 ContiguousMap contiguous(vec.data(), vec.size());
170 stable_normalization_dispatch<ContiguousMap, Accumulator, false>::scale_in_place(contiguous, factor);
173 stable_normalization_dispatch<VectorType, Accumulator, false>::scale_in_place(vec, factor);
178template <
typename VectorType,
typename Accumulator>
179EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE
void stable_normalization_scale_in_place(VectorType& vec,
180 const Accumulator& factor) {
181 stable_normalization_dispatch<VectorType, Accumulator>::scale_in_place(vec, factor);
184template <
typename VectorType,
typename Divisor>
185EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE
void stable_normalization_divide_in_place(VectorType& vec,
const Divisor& divisor) {
189template <
typename VectorType,
typename Accumulator>
190EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void stable_normalization_with_division(VectorType& vec,
195 const Accumulator sqrtMax = numext::sqrt(
maxCoeff);
196 const RealScalar scale1 =
static_cast<RealScalar
>(sqrtMax);
197 const RealScalar scale2 =
static_cast<RealScalar
>(
maxCoeff / sqrtMax);
198 stable_normalization_divide_in_place(vec, scale1);
199 stable_normalization_divide_in_place(vec, scale2);
201 if (z > Accumulator(0)) {
202 stable_normalization_scale_in_place(vec, Accumulator(1) / numext::sqrt(z));
206template <
typename VectorType,
typename Accumulator,
207 bool = use_subnormal_preserving_scaling<Accumulator, typename traits<VectorType>::Scalar>::value>
208struct stable_normalization_subnormal_recovery {
209 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(VectorType&) {}
212template <
typename VectorType,
typename Accumulator>
213struct stable_normalization_subnormal_recovery<VectorType, Accumulator, true> {
214 using RealScalar =
typename NumTraits<typename traits<VectorType>::Scalar>::Real;
216 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(VectorType& vec) {
217 using Binary = binary_floating_point_traits<RealScalar>;
218 using Bits =
typename Binary::Bits;
219 decltype(
auto) components = vec.realView();
221 for (Index col = 0; col < components.cols(); ++col) {
222 for (Index row = 0; row < components.rows(); ++row) {
223 const Bits bits = Binary::magnitude(components.coeff(row, col));
224 if (bits > maxBits) maxBits = bits;
227 if (maxBits == 0 || maxBits >= Binary::kExponentUnit)
return;
229 const Accumulator maxAbs = numext::bit_cast<RealScalar>(maxBits);
230 const auto factors = safe_scaling<Accumulator>::compute_ceiling_factors(maxAbs);
231 safe_scaling<Accumulator>::scale_in_place(vec, maxAbs, factors);
232 const Accumulator squaredNorm = components.template cast<Accumulator>().squaredNorm();
233 if (squaredNorm > Accumulator(0)) {
234 stable_normalization_divide_in_place(vec, numext::sqrt(squaredNorm));
245template <
typename Xpr>
246struct squared_norm_runtime_unit_stride {
247 using Scalar =
typename traits<Xpr>::Scalar;
248 static constexpr bool value =
249 bool(traits<Xpr>::Flags &
DirectAccessBit) && bool(packet_traits<Scalar>::Vectorizable) &&
250 !bool(internal::is_same<Scalar, bool>::value) && (int(inner_stride_at_compile_time<Xpr>::value) != 1);
253template <typename Derived, typename Scalar = typename traits<Derived>::Scalar,
typename Enable =
void>
254struct squared_norm_impl {
255 using Real =
typename NumTraits<Scalar>::Real;
256 static EIGEN_DEVICE_FUNC
constexpr EIGEN_STRONG_INLINE Real run(
const Derived& a) {
257 return a.realView().cwiseAbs2().sum();
261template <
typename Derived>
262struct squared_norm_impl<Derived, bool, void> {
263 static EIGEN_DEVICE_FUNC
constexpr EIGEN_STRONG_INLINE
bool run(
const Derived& a) {
return a.any(); }
269template <
typename Derived,
typename Scalar>
270struct squared_norm_impl<Derived, Scalar, std::enable_if_t<squared_norm_runtime_unit_stride<Derived>::value>> {
271 using Real =
typename NumTraits<Scalar>::Real;
272 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Real run(
const Derived& a) {
273 if (a.innerStride() == 1 && (a.outerSize() == 1 || a.outerStride() == a.innerSize())) {
274 using PlainVector = Matrix<Scalar, Dynamic, 1>;
276 return contiguous.realView().cwiseAbs2().sum();
278 return a.realView().cwiseAbs2().sum();
295template <
typename Derived>
296template <
typename OtherDerived>
297EIGEN_DEVICE_FUNC
constexpr EIGEN_STRONG_INLINE
299 typename internal::traits<OtherDerived>::Scalar>::ReturnType
301 return internal::inner_product_dispatch<Derived, OtherDerived, true>::run(derived(), other.derived());
312template <
typename Derived>
315 return internal::squared_norm_impl<Derived>::run(derived());
324template <
typename Derived>
339template <
typename Derived>
342 using Nested_ =
typename internal::nested_eval<Derived, 2>::type;
343 Nested_ n(derived());
344 RealScalar z = n.squaredNorm();
346 if (z > RealScalar(0))
347 return n / numext::sqrt(z);
360template <
typename Derived>
364 if (z > RealScalar(0)) derived() /= numext::sqrt(z);
379template <
typename Derived>
380EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const typename MatrixBase<Derived>::PlainObject
382 using Nested_ =
typename internal::nested_eval<Derived, 3>::type;
383 using NestedClean = internal::remove_all_t<Nested_>;
384 using Accumulator =
typename internal::stable_norm_accumulator<RealScalar>::type;
385 using Dispatch = internal::stable_normalization_dispatch<NestedClean, Accumulator>;
386 Nested_ vec(derived());
387 if (EIGEN_PREDICT_FALSE(vec.size() == 0))
return vec;
391 const Accumulator w = Dispatch::max_abs(vec);
392 const Accumulator highest =
static_cast<Accumulator
>(NumTraits<RealScalar>::highest());
393 if (EIGEN_PREDICT_FALSE(!(w > Accumulator(0)))) {
395 internal::stable_normalization_subnormal_recovery<PlainObject, Accumulator>::run(
normalized);
398 if (EIGEN_PREDICT_FALSE(!(w <= highest)))
return vec;
400 Accumulator invScale;
401 if (EIGEN_PREDICT_TRUE((internal::stable_normalization_inv_scale<RealScalar>(w, invScale)))) {
402 const Accumulator z = Dispatch::scaled_squared_norm(vec, invScale);
403 if (z > Accumulator(0)) {
404 const Accumulator sqrt_z = numext::sqrt(z);
407 if (EIGEN_PREDICT_TRUE(internal::stable_normalization_combined_factor(invScale, sqrt_z, factor))) {
408 Dispatch::assign_scaled(
normalized, vec, factor);
411 Dispatch::assign_scaled(
normalized, vec, invScale);
412 internal::stable_normalization_divide_in_place(
normalized,
static_cast<RealScalar
>(sqrt_z));
418 internal::stable_normalization_subnormal_recovery<PlainObject, Accumulator>::run(
normalized);
423 internal::stable_normalization_with_division(
normalized, w);
438template <
typename Derived>
440 using Accumulator =
typename internal::stable_norm_accumulator<RealScalar>::type;
441 using Dispatch = internal::stable_normalization_dispatch<Derived, Accumulator>;
442 if (EIGEN_PREDICT_FALSE(size() == 0))
return;
444 const Accumulator w = Dispatch::max_abs(derived());
445 const Accumulator highest =
static_cast<Accumulator
>(NumTraits<RealScalar>::highest());
446 if (EIGEN_PREDICT_FALSE(!(w > Accumulator(0)))) {
447 internal::stable_normalization_subnormal_recovery<Derived, Accumulator>::run(derived());
450 if (EIGEN_PREDICT_FALSE(!(w <= highest)))
return;
452 Accumulator invScale;
453 if (EIGEN_PREDICT_TRUE((internal::stable_normalization_inv_scale<RealScalar>(w, invScale)))) {
454 const Accumulator z = Dispatch::scaled_squared_norm(derived(), invScale);
455 if (z > Accumulator(0)) {
456 const Accumulator sqrt_z = numext::sqrt(z);
458 if (EIGEN_PREDICT_TRUE(internal::stable_normalization_combined_factor(invScale, sqrt_z, factor))) {
459 Dispatch::scale_in_place(derived(), factor);
461 internal::stable_normalization_scale_in_place(derived(), invScale);
462 internal::stable_normalization_divide_in_place(derived(),
static_cast<RealScalar
>(sqrt_z));
465 internal::stable_normalization_subnormal_recovery<Derived, Accumulator>::run(derived());
470 internal::stable_normalization_with_division(derived(), w);
477template <
typename Derived,
int p>
478struct lpNorm_selector {
481 return numext::pow(m.
cwiseAbs().array().pow(p).sum(), RealScalar(1) / p);
485template <
typename Derived>
486struct lpNorm_selector<Derived, 1> {
487 EIGEN_DEVICE_FUNC
static inline typename NumTraits<typename traits<Derived>::Scalar>::Real run(
488 const MatrixBase<Derived>& m) {
489 return m.cwiseAbs().sum();
493template <
typename Derived>
494struct lpNorm_selector<Derived, 2> {
495 EIGEN_DEVICE_FUNC
static inline typename NumTraits<typename traits<Derived>::Scalar>::Real run(
496 const MatrixBase<Derived>& m) {
501template <
typename Derived>
502struct lpNorm_selector<Derived, Infinity> {
503 using RealScalar =
typename NumTraits<typename traits<Derived>::Scalar>::Real;
504 EIGEN_DEVICE_FUNC
static inline RealScalar run(
const MatrixBase<Derived>& m) {
506 return redux_with_identity(m.cwiseAbs(), scalar_max_op<RealScalar, RealScalar, PropagateNaN>(), 0);
526template <
typename Derived>
528#ifndef EIGEN_PARSED_BY_DOXYGEN
531EIGEN_DEVICE_FUNC MatrixBase<Derived>::RealScalar
534 return internal::lpNorm_selector<Derived, p>::run(*
this);
542template <
typename Derived>
545 return derived().matrix().template
lpNorm<p>();
561template <
typename Derived>
563 typename internal::nested_eval<Derived, 1>::type self(derived());
564 for (Index i = 0; i < cols(); ++i) {
565 if (!internal::isApprox(self.col(i).squaredNorm(),
static_cast<RealScalar
>(1), prec))
return false;
566 for (Index j = 0; j < i; ++j)
567 if (!internal::isMuchSmallerThan(self.col(i).dot(self.col(j)),
static_cast<Scalar>(1), prec))
return false;
internal::traits< Derived >::Scalar maxCoeff() const
Definition Redux.h:841
typename internal::traits< Derived >::Scalar Scalar
Definition DenseBase.h:63
RealScalar lpNorm() const
Definition Dot.h:544
constexpr CastXpr< NewType >::Type cast() const
Definition DenseBase.h:66
A matrix or vector expression mapping an existing array of data.
Definition Map.h:97
Base class for all dense matrices, vectors, and expressions.
Definition MatrixBase.h:53
void stableNormalize()
Definition Dot.h:439
constexpr ScalarBinaryOpTraits< typenameinternal::traits< Derived >::Scalar, typenameinternal::traits< OtherDerived >::Scalar >::ReturnType dot(const MatrixBase< OtherDerived > &other) const
Definition Dot.h:300
const PlainObject stableNormalized() const
Definition Dot.h:381
const PlainObject normalized() const
Definition Dot.h:340
RealScalar lpNorm() const
Definition Dot.h:533
bool isUnitary(const RealScalar &prec=NumTraits< Scalar >::dummy_precision()) const
Definition Dot.h:562
const CwiseUnaryOp< internal::scalar_abs_op< Scalar >, const Derived > cwiseAbs() const
constexpr RealScalar squaredNorm() const
Definition Dot.h:314
void normalize()
Definition Dot.h:361
RealScalar norm() const
Definition Dot.h:326
constexpr unsigned int DirectAccessBit
Definition Constants.h:160
Holds information about the various numeric (i.e. scalar) types allowed by Eigen.
Definition NumTraits.h:267
Determines whether the given binary operation of two numeric types is allowed and what the scalar ret...
Definition XprHelper.h:1080