4#ifndef EIGEN_SAFE_SCALING_H
5#define EIGEN_SAFE_SCALING_H
8#include "./InternalHeaderCheck.h"
18template <
typename Scalar>
19struct supports_power_of_two_scaling
20 : bool_constant<(std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value) &&
21 std::numeric_limits<Scalar>::is_iec559 && std::numeric_limits<Scalar>::radix == 2 &&
22 (sizeof(Scalar) == sizeof(numext::uint32_t) || sizeof(Scalar) == sizeof(numext::uint64_t))> {};
24#if !defined(EIGEN_GPU_COMPILE_PHASE)
26struct supports_power_of_two_scaling<long double> : bool_constant<std::numeric_limits<long double>::radix == 2> {};
30struct supports_power_of_two_scaling<half> : true_type {};
33struct supports_power_of_two_scaling<bfloat16> : true_type {};
35template <typename Scalar, bool = supports_power_of_two_scaling<Scalar>::value>
38template <
typename Scalar>
39struct safe_scaling_factors {
40 Scalar scale = Scalar(1);
42 Scalar invScale = Scalar(1);
50template <
typename Scalar>
51EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE
typename binary_floating_point_traits<Scalar>::Bits scale_binary_bits_by_exponent(
52 const typename binary_floating_point_traits<Scalar>::Bits valueBits,
int exponent) {
53 using Binary = binary_floating_point_traits<Scalar>;
54 using Bits =
typename Binary::Bits;
55 constexpr int kInfinityExponent = int(Binary::kExponentMask >> Binary::kFractionBits);
57 const Bits valueExponentBits = valueBits & Binary::kExponentMask;
58 const Bits fraction = valueBits & Binary::kFractionMask;
59 if (valueExponentBits == Binary::kExponentMask || (valueExponentBits == 0 && fraction == 0))
return valueBits;
60 const Bits sign = valueBits & Binary::kSignBit;
62 Bits significand = fraction;
63 int resultExponent = int(valueExponentBits >> Binary::kFractionBits);
64 if (resultExponent == 0) {
66 while (significand < Binary::kExponentUnit) {
72 constexpr int kSaturatingExponent = kInfinityExponent + Binary::kFractionBits + 2;
73 resultExponent += numext::mini(numext::maxi(exponent, -kSaturatingExponent), kSaturatingExponent);
74 if (resultExponent >= kInfinityExponent)
return sign | Binary::kExponentMask;
75 if (resultExponent > 0)
76 return sign | (Bits(resultExponent) << Binary::kFractionBits) | (significand & Binary::kFractionMask);
81 const int shift = 1 - resultExponent;
82 if (shift > Binary::kFractionBits + 1)
return sign;
83 const Bits mantissa = significand | Binary::kExponentUnit;
84 const Bits halfway = Bits(1) << (shift - 1);
85 const Bits remainder = mantissa & ((halfway << 1) - 1);
86 Bits rounded = mantissa >> shift;
87 if (remainder > halfway || (remainder == halfway && (rounded & Bits(1)) != 0)) ++rounded;
88 return sign | rounded;
92template <
typename Scalar>
93EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename binary_floating_point_traits<Scalar>::Bits
94scale_binary_bits_by_power_of_two(
const typename binary_floating_point_traits<Scalar>::Bits valueBits,
95 const typename binary_floating_point_traits<Scalar>::Bits factorExponentBits) {
96 using Binary = binary_floating_point_traits<Scalar>;
97 return scale_binary_bits_by_exponent<Scalar>(
98 valueBits,
int(factorExponentBits >> Binary::kFractionBits) - Binary::kExponentBias);
103template <
typename Scalar>
104EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar scale_binary_by_exponent(
const Scalar& value,
int exponent) {
105 using Binary = binary_floating_point_traits<Scalar>;
106 return numext::bit_cast<Scalar>(scale_binary_bits_by_exponent<Scalar>(Binary::bits(value), exponent));
109template <
typename Scalar>
110EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<Scalar> scale_binary_by_exponent(
const std::complex<Scalar>& value,
112 return std::complex<Scalar>(scale_binary_by_exponent(value.real(), exponent),
113 scale_binary_by_exponent(value.imag(), exponent));
122template <
typename Scalar>
123EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE
int binary_frexp_exponent_of_magnitude(
124 const typename binary_floating_point_traits<Scalar>::Bits magnitude) {
125 using Binary = binary_floating_point_traits<Scalar>;
126 using Bits =
typename Binary::Bits;
127 if (magnitude == 0)
return 0;
128 const Bits exponentBits = magnitude & Binary::kExponentMask;
129 if (exponentBits != 0)
return int(exponentBits >> Binary::kFractionBits) - Binary::kExponentBias + 1;
131 return log_2_impl<Bits>::run_floor(magnitude) + 1 + std::numeric_limits<Scalar>::min_exponent -
132 std::numeric_limits<Scalar>::digits;
135template <
typename Scalar>
136EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
int binary_frexp_exponent(
const Scalar& value) {
137 return binary_frexp_exponent_of_magnitude<Scalar>(binary_floating_point_traits<Scalar>::magnitude(value));
141template <
typename Scalar>
142EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE
typename binary_floating_point_traits<Scalar>::Bits larger_magnitude_bits(
143 const typename binary_floating_point_traits<Scalar>::Bits a,
144 const typename binary_floating_point_traits<Scalar>::Bits b) {
145 return a < b ? b : a;
149template <
typename Scalar>
150EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE
bool is_below_normal_magnitude_bits(
151 const typename binary_floating_point_traits<Scalar>::Bits magnitude) {
152 return magnitude < binary_floating_point_traits<Scalar>::kExponentUnit;
156template <
typename Scalar>
157EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar binary_frexp(
const Scalar& value,
int& exponent) {
158 exponent = binary_frexp_exponent(value);
159 return scale_binary_by_exponent(value, -exponent);
165template <
typename Scalar>
166EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar scale_binary_by_power_of_two(
const Scalar& value,
const Scalar& factor) {
167 using Binary = binary_floating_point_traits<Scalar>;
168 using Bits =
typename Binary::Bits;
169 const Bits factorExponentBits = Binary::bits(factor);
170 eigen_internal_assert(factorExponentBits != 0 && factorExponentBits < Binary::kExponentMask &&
171 (factorExponentBits & ~Binary::kExponentMask) == 0);
172 return numext::bit_cast<Scalar>(scale_binary_bits_by_power_of_two<Scalar>(Binary::bits(value), factorExponentBits));
175template <
typename Scalar>
176EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<Scalar> scale_binary_by_power_of_two(
177 const std::complex<Scalar>& value,
const Scalar& factor) {
178 return std::complex<Scalar>(scale_binary_by_power_of_two(value.real(), factor),
179 scale_binary_by_power_of_two(value.imag(), factor));
183template <
typename Scalar>
184struct scale_by_exponent_op {
185 EIGEN_DEVICE_FUNC
explicit scale_by_exponent_op(
int exponent) : m_exponent(exponent) {}
187 template <
typename CoeffScalar>
188 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffScalar operator()(
const CoeffScalar& value)
const {
189 return scale_binary_by_exponent(value, m_exponent);
195template <
typename Scalar>
196struct functor_traits<scale_by_exponent_op<Scalar>> {
197 static constexpr int Cost = 10 * NumTraits<Scalar>::MulCost;
198 static constexpr bool PacketAccess =
false;
199 static constexpr bool IsRepeatable =
true;
202template <
typename FactorScalar>
203struct scale_by_power_of_two_op {
204 EIGEN_DEVICE_FUNC
explicit scale_by_power_of_two_op(
const FactorScalar& factor) : m_factor(factor) {}
206 template <
typename CoeffScalar>
207 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffScalar operator()(
const CoeffScalar& value)
const {
208 return scale_binary_by_power_of_two(value, m_factor);
211 FactorScalar m_factor;
214template <
typename FactorScalar>
215struct functor_traits<scale_by_power_of_two_op<FactorScalar>> {
217 static constexpr int Cost = 10 * NumTraits<FactorScalar>::MulCost;
218 static constexpr bool PacketAccess =
false;
219 static constexpr bool IsRepeatable =
true;
222template <
typename FactorScalar,
typename CoeffScalar>
223struct use_subnormal_preserving_scaling
225 (std::is_same<FactorScalar, float>::value &&
226 (std::is_same<CoeffScalar, float>::value || std::is_same<CoeffScalar, std::complex<float>>::value)) ||
227 (std::is_same<FactorScalar, double>::value &&
228 (std::is_same<CoeffScalar, double>::value || std::is_same<CoeffScalar, std::complex<double>>::value))> {};
232template <
typename Scalar>
233EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool is_subnormal_magnitude_impl(
const Scalar& value, true_type) {
235 const int exponent = binary_frexp_exponent(value);
236 return exponent != 0 && exponent < std::numeric_limits<Scalar>::min_exponent;
238template <
typename Scalar>
239EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool is_subnormal_magnitude_impl(
const Scalar& value, false_type) {
240 const Scalar magnitude = numext::abs(value);
241 return magnitude > Scalar(0) && magnitude < (std::numeric_limits<Scalar>::min)();
244template <
typename Scalar>
245EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool is_subnormal_magnitude(
const Scalar& value) {
246 return is_subnormal_magnitude_impl(value, bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>());
253template <
typename Scalar>
254EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool is_zero_or_subnormal_magnitude_impl(
const Scalar& value, true_type) {
255 return is_below_normal_magnitude_bits<Scalar>(binary_floating_point_traits<Scalar>::magnitude(value));
257template <
typename Scalar>
258EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool is_zero_or_subnormal_magnitude_impl(
const Scalar& value, false_type) {
259 return numext::abs(value) < (std::numeric_limits<Scalar>::min)();
261template <
typename Scalar>
262EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool is_zero_or_subnormal_magnitude(
const Scalar& value) {
263 return is_zero_or_subnormal_magnitude_impl(value,
264 bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>());
269template <
typename Scalar>
270EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar max_preserving_subnormals_impl(
const Scalar& a,
const Scalar& b,
272 using Binary = binary_floating_point_traits<Scalar>;
273 return numext::bit_cast<Scalar>(larger_magnitude_bits<Scalar>(Binary::magnitude(a), Binary::magnitude(b)));
275template <
typename Scalar>
276EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar max_preserving_subnormals_impl(
const Scalar& a,
const Scalar& b,
278 return numext::maxi(a, b);
280template <
typename Scalar>
281EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar max_preserving_subnormals(
const Scalar& a,
const Scalar& b) {
282 return max_preserving_subnormals_impl(a, b, bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>());
288template <
typename Scalar>
289EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar abs_preserving_subnormals_impl(
const Scalar& value, true_type) {
290 return numext::bit_cast<Scalar>(binary_floating_point_traits<Scalar>::magnitude(value));
292template <
typename Scalar>
293EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar abs_preserving_subnormals_impl(
const Scalar& value, false_type) {
294 return numext::abs(value);
296template <
typename Scalar>
297EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar abs_preserving_subnormals(
const Scalar& value) {
298 return abs_preserving_subnormals_impl(value,
299 bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>());
302template <
typename Scalar>
303EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool is_negative_preserving_subnormals_impl(
const Scalar& value, true_type) {
304 using Binary = binary_floating_point_traits<Scalar>;
305 return (Binary::bits(value) & Binary::kSignBit) != 0 && !is_zero_magnitude_bits<Scalar>(Binary::magnitude(value));
307template <
typename Scalar>
308EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool is_negative_preserving_subnormals_impl(
const Scalar& value, false_type) {
309 return value < Scalar(0);
311template <
typename Scalar>
312EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool is_negative_preserving_subnormals(
const Scalar& value) {
313 return is_negative_preserving_subnormals_impl(
314 value, bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>());
319template <
typename Scalar,
typename Derived>
320EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE Index index_of_largest_magnitude(
const DenseBase<Derived>& src) {
321 using Binary = binary_floating_point_traits<Scalar>;
322 using Bits =
typename Binary::Bits;
323 const evaluator<Derived> coeffs(src.derived());
325 Bits largest = Binary::magnitude(coeffs.coeff(0));
326 for (Index i = 1; i < src.size(); ++i) {
327 const Bits magnitude = Binary::magnitude(coeffs.coeff(i));
328 if (magnitude > largest) {
339template <
typename Scalar>
340EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar ldexp_preserving_subnormals_impl(
const Scalar& value,
int exponent,
342 return scale_binary_by_exponent(value, exponent);
344template <
typename Scalar>
345EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar ldexp_preserving_subnormals_impl(
const Scalar& value,
int exponent,
347 return numext::ldexp(value, exponent);
349template <
typename Scalar>
350EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar ldexp_preserving_subnormals(
const Scalar& value,
int exponent) {
351 return ldexp_preserving_subnormals_impl(value, exponent,
352 bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>());
355template <
typename Scalar>
356EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
int frexp_exponent_preserving_subnormals_impl(
const Scalar& value, true_type) {
357 return binary_frexp_exponent(value);
359template <
typename Scalar>
360EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
int frexp_exponent_preserving_subnormals_impl(
const Scalar& value, false_type) {
362 EIGEN_USING_STD(frexp);
363 frexp(value, &exponent);
366template <
typename Scalar>
367EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
int frexp_exponent_preserving_subnormals(
const Scalar& value) {
368 return frexp_exponent_preserving_subnormals_impl(
369 value, bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>());
372template <
typename Scalar>
373EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar frexp_preserving_subnormals(
const Scalar& value,
int& exponent) {
374 exponent = frexp_exponent_preserving_subnormals(value);
375 return ldexp_preserving_subnormals(value, -exponent);
378template <
typename Scalar,
bool IsPowerOfTwo_>
379struct safe_scaling_operations {
381 using Factors = safe_scaling_factors<Scalar>;
383 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
bool needs_subnormal_recovery(
const Scalar& value) {
384 using Binary = binary_floating_point_traits<Scalar>;
385 using Bits =
typename Binary::Bits;
386 constexpr Bits kThreshold = Bits(subnormal_recovery_exponent() + Binary::kExponentBias) << Binary::kFractionBits;
387 return Binary::magnitude(value) - Bits(1) < kThreshold - Bits(1);
390 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
bool is_identity(
const Factors& factors) {
391 return factors.scale == Scalar(1) && factors.invScale == Scalar(1);
394 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
bool has_normal_reciprocal(
const Scalar&,
const Scalar&, false_type) {
398 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
bool has_normal_reciprocal(
const Scalar& value,
const Scalar& normalMin,
400 return value >= normalMin && value <= Scalar(1) / normalMin;
403 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
bool is_positive_finite(
const Scalar& value, false_type) {
404 return value > Scalar(0) && value <= NumTraits<Scalar>::highest();
407 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
bool is_positive_finite(
const Scalar& value, true_type) {
408 using Binary = binary_floating_point_traits<Scalar>;
409 const typename Binary::Bits bits = Binary::bits(value);
410 return bits > 0 && bits < Binary::kExponentMask;
413 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Factors select_factors(
const Scalar& maxCoeff) {
414 using IsIeeeBinary = bool_constant<std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value>;
415 if (!is_positive_finite(maxCoeff, IsIeeeBinary()))
return Factors{};
416 return safe_scaling<Scalar, IsPowerOfTwo_>::compute_floor_factors(maxCoeff);
419 template <
typename Src,
typename Func>
420 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void with_scaled_impl(
const Src& src,
const Scalar&,
421 const Factors& factors,
const Func& func,
423 if (is_identity(factors)) {
425 const Src* unscaled = &src;
426 EIGEN_OPTIMIZATION_BARRIER(unscaled);
428 }
else EIGEN_IF_CONSTEXPR (IsPowerOfTwo_)
429 func(src * factors.invScale);
431 func(src / factors.scale);
434 template <
typename Src,
typename Func>
435 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void with_scaled_impl(
const Src& src,
const Scalar& maxCoeff,
436 const Factors& factors,
const Func& func,
438 if (!is_identity(factors) && needs_subnormal_recovery(maxCoeff))
439 func(src.unaryExpr(scale_by_power_of_two_op<Scalar>(factors.invScale)));
441 with_scaled_impl(src, maxCoeff, factors, func, false_type());
444 template <
typename Dest,
typename Src>
445 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void unscale_to_impl(Dest& dest,
const Src& src,
const Scalar&,
446 const Factors& factors, false_type) {
447 unscale_to(dest, src, factors);
452 template <
typename Dest,
typename Src>
453 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void unscale_to_impl(Dest& dest,
const Src& src,
const Scalar& maxCoeff,
454 const Factors& factors, true_type) {
455 if (!is_identity(factors) && needs_subnormal_recovery(maxCoeff))
456 dest = src.unaryExpr(scale_by_power_of_two_op<Scalar>(factors.scale));
458 unscale_to_impl(dest, src, maxCoeff, factors, false_type());
461 template <
typename Src>
462 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Scalar recover_flushed_max_coeff_impl(
const Src&,
const Scalar& maxCoeff,
467 template <
typename Src>
468 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Scalar recover_flushed_max_coeff_impl(
const Src& src,
469 const Scalar& maxCoeff,
471 using Binary = binary_floating_point_traits<Scalar>;
472 using Bits =
typename Binary::Bits;
473 if (!is_zero_or_subnormal_magnitude(maxCoeff))
return maxCoeff;
475 const evaluator<Src> coeffs(src);
477 for (Index col = 0; col < src.cols(); ++col) {
478 for (Index row = 0; row < src.rows(); ++row) {
479 const typename Src::Scalar coeff = coeffs.coeff(row, col);
480 maxBits = numext::maxi(
481 maxBits, numext::maxi(Binary::magnitude(numext::real(coeff)), Binary::magnitude(numext::imag(coeff))));
484 return numext::bit_cast<Scalar>(maxBits);
487 template <
typename Derived>
488 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Index recover_flushed_max_coeff_index_impl(
const DenseBase<Derived>&,
489 Index position, false_type) {
493 template <
typename Derived>
494 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Index recover_flushed_max_coeff_index_impl(
const DenseBase<Derived>& src,
496 return index_of_largest_magnitude<Scalar>(src);
500 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Factors
501 compute_ceiling_factors_with_normal_reciprocal(
const Scalar& value) {
502 Factors factors = safe_scaling<Scalar, IsPowerOfTwo_>::compute_ceiling_factors(value);
503 EIGEN_IF_CONSTEXPR (!IsPowerOfTwo_) {
504 if (factors.invScale > NumTraits<Scalar>::highest()) {
505 factors.invScale = NumTraits<Scalar>::highest();
506 factors.scale = Scalar(1) / factors.invScale;
512 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
bool try_compute_ceiling_factors_with_normal_reciprocal(
513 const Scalar& value,
const Scalar& normalMin, Factors& factors) {
514 if (!IsPowerOfTwo_ &&
515 !has_normal_reciprocal(value, normalMin, bool_constant<std::is_floating_point<Scalar>::value>()))
517 factors = compute_ceiling_factors_with_normal_reciprocal(value);
523 template <
typename Src,
typename Func>
524 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Factors with_scaled(
const Src& src,
const Scalar& maxCoeff,
526 const Factors factors = select_factors(maxCoeff);
527 constexpr bool kPreserveSubnormalInputs =
528 IsPowerOfTwo_ && use_subnormal_preserving_scaling<Scalar, typename Src::Scalar>::value;
529 with_scaled_impl(src, maxCoeff, factors, func, bool_constant<kPreserveSubnormalInputs>());
533 template <
typename Dest,
typename Src>
534 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void unscale_to(Dest&& dest,
const Src& src,
const Factors& factors) {
535 if (factors.scale == Scalar(1)) {
536 const Src* unscaled = &src;
537 EIGEN_OPTIMIZATION_BARRIER(unscaled);
541 dest = src * factors.scale;
544 template <
typename ValueType>
545 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void unscale_in_place(ValueType& value,
const Factors& factors) {
546 if (factors.scale == Scalar(1))
return;
547 value *= factors.scale;
550 template <
typename MatrixType>
551 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void unscale_in_place(MatrixType& matrix,
const Scalar& maxCoeff,
552 const Factors& factors) {
553 if (factors.scale == Scalar(1))
return;
554 unscale_to(matrix, matrix, maxCoeff, factors);
557 template <
typename Dest,
typename Src>
558 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void unscale_to(Dest&& dest,
const Src& src,
const Scalar& maxCoeff,
559 const Factors& factors) {
560 constexpr bool kPreserveSubnormalOutputs =
561 IsPowerOfTwo_ && use_subnormal_preserving_scaling<Scalar, typename Src::Scalar>::value;
562 unscale_to_impl(dest, src, maxCoeff, factors, bool_constant<kPreserveSubnormalOutputs>());
568 EIGEN_DEVICE_FUNC
static constexpr int subnormal_recovery_exponent() {
569 return std::numeric_limits<Scalar>::min_exponent + std::numeric_limits<Scalar>::digits - 2;
576 template <
typename Src>
577 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Scalar recover_flushed_max_coeff(
const Src& src,
578 const Scalar& maxCoeff) {
579 constexpr bool kPreserveSubnormalInputs =
580 IsPowerOfTwo_ && use_subnormal_preserving_scaling<Scalar, typename Src::Scalar>::value;
581 return recover_flushed_max_coeff_impl(src, maxCoeff, bool_constant<kPreserveSubnormalInputs>());
586 template <
typename Src>
587 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Scalar max_abs_component(
const Src& src) {
588 if (src.size() == 0)
return Scalar(0);
589 return recover_flushed_max_coeff(src, src.realView().cwiseAbs().template maxCoeff<PropagateNaN>());
595 template <
typename Derived>
596 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Index recover_flushed_max_coeff_index(
const DenseBase<Derived>& src,
598 constexpr bool kPreserveSubnormalInputs =
599 IsPowerOfTwo_ && use_subnormal_preserving_scaling<Scalar, typename Derived::Scalar>::value;
600 return recover_flushed_max_coeff_index_impl(src, position, bool_constant<kPreserveSubnormalInputs>());
603 template <
typename MatrixType>
604 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Factors scale_in_place(MatrixType& matrix,
const Scalar& maxCoeff) {
605 const Factors factors = select_factors(maxCoeff);
606 scale_in_place(matrix, maxCoeff, factors);
610 template <
typename MatrixType>
611 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void scale_in_place(MatrixType& matrix,
const Scalar& maxCoeff,
612 const Factors& factors) {
613 if (!is_identity(factors)) scale_to(matrix, matrix, maxCoeff, factors);
616 template <
typename Dest,
typename Src>
617 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void scale_to(Dest& dest,
const Src& src,
const Scalar& maxCoeff,
618 const Factors& factors) {
619 constexpr bool kPreserveSubnormalInputs =
620 IsPowerOfTwo_ && use_subnormal_preserving_scaling<Scalar, typename Src::Scalar>::value;
622 src, maxCoeff, factors, [&](
const auto& scaled) { dest = scaled; }, bool_constant<kPreserveSubnormalInputs>());
625 template <
typename Dest,
typename Src>
626 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Factors scale_to(Dest& dest,
const Src& src,
const Scalar& maxCoeff) {
627 const Factors factors = select_factors(maxCoeff);
628 scale_to(dest, src, maxCoeff, factors);
633template <
typename Scalar,
bool>
634struct safe_scaling : safe_scaling_operations<Scalar, false> {
635 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE safe_scaling_factors<Scalar> compute_floor_factors(
const Scalar& value) {
636 return {value, Scalar(1)};
639 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE safe_scaling_factors<Scalar> compute_ceiling_factors(
640 const Scalar& value) {
641 return {value, Scalar(1) / value};
645template <
typename Scalar>
646struct safe_scaling<Scalar, true> : safe_scaling_operations<Scalar, true> {
648 using Binary = binary_floating_point_traits<Scalar>;
649 using Bits =
typename Binary::Bits;
651 template <
bool RoundUp>
652 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE safe_scaling_factors<Scalar> compute_power_of_two_factors(
653 const Scalar& value) {
654 constexpr Bits kMaxScale = Binary::kMaxFiniteExponentBits - Binary::kExponentUnit;
655 Bits scaleBits = RoundUp ? numext::bit_cast<Bits>(numext::ceil_power_of_two(value))
656 : Binary::bits(value) & Binary::kExponentMask;
657 if (scaleBits < Binary::kExponentUnit) scaleBits = Binary::kExponentUnit;
658 if (scaleBits > kMaxScale) scaleBits = kMaxScale;
659 const Bits invScaleBits = Bits(Binary::kMaxFiniteExponentBits - scaleBits);
660 return {numext::bit_cast<Scalar>(scaleBits), numext::bit_cast<Scalar>(invScaleBits)};
664 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE safe_scaling_factors<Scalar> compute_floor_factors(
const Scalar& value) {
665 return compute_power_of_two_factors<false>(value);
668 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE safe_scaling_factors<Scalar> compute_ceiling_factors(
669 const Scalar& value) {
670 return compute_power_of_two_factors<true>(value);
674#if !defined(EIGEN_GPU_COMPILE_PHASE)
676struct safe_scaling<long double, true> : safe_scaling_operations<long double, true> {
677 static EIGEN_STRONG_INLINE safe_scaling_factors<long double> compute_floor_factors(
const long double& value) {
678 EIGEN_USING_STD(frexp);
680 frexp(value, &exponent);
681 return compute_factors_from_exponent(exponent - 1);
684 static EIGEN_STRONG_INLINE safe_scaling_factors<long double> compute_ceiling_factors(
const long double& value) {
685 EIGEN_USING_STD(frexp);
687 const long double fraction = frexp(value, &exponent);
688 if (numext::equal_strict(fraction, 0.5L)) --exponent;
689 return compute_factors_from_exponent(exponent);
693 static EIGEN_STRONG_INLINE safe_scaling_factors<long double> compute_factors_from_exponent(
int exponent) {
694 constexpr int kMinNormalExponent = std::numeric_limits<long double>::min_exponent - 1;
695 constexpr int kMaxNormalExponent = std::numeric_limits<long double>::max_exponent - 1;
696 constexpr int kMinScaleExponent =
697 kMinNormalExponent > -kMaxNormalExponent ? kMinNormalExponent : -kMaxNormalExponent;
698 constexpr int kMaxScaleExponent =
699 kMaxNormalExponent < -kMinNormalExponent ? kMaxNormalExponent : -kMinNormalExponent;
700 if (exponent < kMinScaleExponent) exponent = kMinScaleExponent;
701 if (exponent > kMaxScaleExponent) exponent = kMaxScaleExponent;
703 const long double scale = numext::ldexp(1.0L, exponent);
704 return {scale, 1.0L / scale};
712template <
typename Scalar>
713EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename NumTraits<Scalar>::Real component_magnitude_impl(
const Scalar& z,
715 return max_preserving_subnormals_impl(numext::real(z), numext::imag(z), true_type());
717template <
typename Scalar>
718EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename NumTraits<Scalar>::Real component_magnitude_impl(
const Scalar& z,
720 return numext::maxi(numext::abs(numext::real(z)), numext::abs(numext::imag(z)));
722template <
typename Scalar>
723EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename NumTraits<Scalar>::Real component_magnitude(
const Scalar& z) {
724 using RealScalar =
typename NumTraits<Scalar>::Real;
725 constexpr bool kReadsRepresentation =
726 complex_array_access<Scalar>::value && use_subnormal_preserving_scaling<RealScalar, RealScalar>::value;
727 return component_magnitude_impl(z, bool_constant<kReadsRepresentation>());
730template <typename Scalar, bool IsComplex = NumTraits<Scalar>::IsComplex>
731struct exponent_scaling_impl {
732 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Scalar ldexp(
const Scalar& z,
int exponent) {
733 return Scalar(ldexp_preserving_subnormals(numext::real(z), exponent),
734 ldexp_preserving_subnormals(numext::imag(z), exponent));
737template <
typename Scalar>
738struct exponent_scaling_impl<Scalar, false> {
739 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Scalar ldexp(
const Scalar& x,
int exponent) {
740 return ldexp_preserving_subnormals(x, exponent);
746template <
typename Scalar,
typename Exponent>
747EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar apply_exponent(
const Scalar& z, Exponent exponent) {
748 constexpr Exponent kMaxExponent = Exponent(1) << 24;
749 return exponent_scaling_impl<Scalar>::ldexp(
750 z,
static_cast<int>(numext::mini(numext::maxi(exponent, -kMaxExponent), kMaxExponent)));
756template <
typename Scalar,
typename Exponent>
757EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar balance_exponent(
const Scalar& z, Exponent& exponent) {
758 const typename NumTraits<Scalar>::Real magnitude = component_magnitude(z);
759 if (numext::is_exactly_zero_no_flush(magnitude) || !(numext::isfinite)(magnitude))
return z;
760 const int e = frexp_exponent_preserving_subnormals(magnitude);
762 return exponent_scaling_impl<Scalar>::ldexp(z, -e);
767template <
typename Scalar>
768struct has_scaled_product
769 : bool_constant<supports_power_of_two_scaling<typename NumTraits<Scalar>::Real>::value &&
770 std::is_floating_point<typename NumTraits<Scalar>::Real>::value &&
771 (std::is_same<Scalar, typename NumTraits<Scalar>::Real>::value ||
772 std::is_same<Scalar, std::complex<typename NumTraits<Scalar>::Real>>::value)> {};
779template <typename Scalar, bool Scaled = has_scaled_product<Scalar>::value>
780class scaled_product {
782 using RealScalar =
typename NumTraits<Scalar>::Real;
783 using Exponent = numext::int64_t;
785 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void multiply(
const Scalar& factor) {
786 const Scalar product = m_mantissa * factor;
787 if (EIGEN_PREDICT_TRUE(in_range(product)))
788 m_mantissa = product;
793 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void divide(
const Scalar& divisor) {
794 const Scalar quotient = m_mantissa / divisor;
795 if (EIGEN_PREDICT_TRUE(in_range(quotient)))
796 m_mantissa = quotient;
798 rescale_quotient(divisor);
801 EIGEN_DEVICE_FUNC
void multiply(
const scaled_product& other) {
803 const Exponent exponent = other.m_exponent;
804 const Scalar mantissa = other.m_mantissa;
806 m_exponent += exponent;
809 EIGEN_DEVICE_FUNC Scalar value()
const {
810 return m_exponent == 0 ? m_mantissa : apply_exponent(m_mantissa, m_exponent);
814 EIGEN_DEVICE_FUNC RealScalar abs()
const {
815 Exponent exponent = m_exponent;
816 const RealScalar magnitude = numext::abs(balance_exponent(m_mantissa, exponent));
817 return apply_exponent(magnitude, exponent);
821 EIGEN_DEVICE_FUNC RealScalar logAbs()
const {
822 Exponent exponent = m_exponent;
823 RealScalar magnitude = numext::abs(balance_exponent(m_mantissa, exponent));
826 if (magnitude < RealScalar(0.75)) {
827 magnitude *= RealScalar(2);
830 return numext::log(magnitude) + RealScalar(exponent) * numext::log(RealScalar(2));
836 EIGEN_DEVICE_FUNC
static constexpr RealScalar upper_bound() {
837 return NumTraits<RealScalar>::highest() / RealScalar(2);
840 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
bool in_range(
const RealScalar& x) {
841 const RealScalar magnitude = numext::abs(x);
842 return magnitude >= (std::numeric_limits<RealScalar>::min)() && magnitude <= upper_bound();
848 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
bool in_range(
const std::complex<RealScalar>& z) {
849 const RealScalar kLowest = (std::numeric_limits<RealScalar>::min)() / NumTraits<RealScalar>::epsilon();
850 const RealScalar re = numext::abs(numext::real(z));
851 const RealScalar im = numext::abs(numext::imag(z));
852 return re <= upper_bound() && im <= upper_bound() && (re >= kLowest || im >= kLowest);
857 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE
void rescale(
const Scalar& factor) {
858 const Scalar balancedFactor = balance_exponent(factor, m_exponent);
859 m_mantissa = balance_exponent(m_mantissa, m_exponent) * balancedFactor;
863 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE
void rescale_quotient(
const Scalar& divisor) {
864 Exponent divisorExponent = 0;
865 const Scalar balancedDivisor = balance_exponent(divisor, divisorExponent);
866 m_mantissa = balance_exponent(m_mantissa, m_exponent) / balancedDivisor;
867 m_exponent -= divisorExponent;
870 Scalar m_mantissa = Scalar(1);
871 Exponent m_exponent = 0;
874template <
typename Scalar>
875class scaled_product<Scalar, false> {
877 using RealScalar =
typename NumTraits<Scalar>::Real;
879 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void multiply(
const Scalar& factor) { m_product *= factor; }
880 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void divide(
const Scalar& divisor) { m_product /= divisor; }
881 EIGEN_DEVICE_FUNC
void multiply(
const scaled_product& other) { m_product *= other.m_product; }
882 EIGEN_DEVICE_FUNC Scalar value()
const {
return m_product; }
883 EIGEN_DEVICE_FUNC RealScalar abs()
const {
return numext::abs(m_product); }
885 EIGEN_DEVICE_FUNC RealScalar logAbs()
const {
return numext::log(numext::abs(m_product)); }
888 Scalar m_product = Scalar(1);
895constexpr int kScaledProdBlock = 8;
897template <
typename Evaluator,
typename Scalar>
898EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index scaled_prod_blocks(
const Evaluator&, Index,
899 scaled_product<Scalar, false>&) {
904template <
typename Evaluator,
typename Scalar>
905EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index scaled_prod_blocks(
const Evaluator& coeffs, Index size,
906 scaled_product<Scalar, true>& product) {
907 using RealScalar =
typename NumTraits<Scalar>::Real;
908 const RealScalar upper = numext::ldexp(RealScalar(1), std::numeric_limits<RealScalar>::max_exponent / 10);
909 const RealScalar lower = RealScalar(1) / upper;
911 for (; i + kScaledProdBlock <= size; i += kScaledProdBlock) {
912 Scalar chains[4] = {Scalar(1), Scalar(1), Scalar(1), Scalar(1)};
913 RealScalar smallest = upper, largest = lower;
914 for (
int j = 0; j < kScaledProdBlock; ++j) {
915 const Scalar factor = coeffs.coeff(i + j);
916 const RealScalar magnitude = numext::maxi(numext::abs(numext::real(factor)), numext::abs(numext::imag(factor)));
917 smallest = numext::mini(smallest, magnitude);
918 largest = numext::maxi(largest, magnitude);
919 chains[j % 4] *= factor;
921 if (EIGEN_PREDICT_TRUE(smallest >= lower && largest <= upper)) {
922 product.multiply((chains[0] * chains[1]) * (chains[2] * chains[3]));
924 for (
int j = 0; j < kScaledProdBlock; ++j) product.multiply(coeffs.coeff(i + j));
931template <
typename Derived>
932EIGEN_DEVICE_FUNC scaled_product<typename Derived::Scalar> scaled_prod(
const DenseBase<Derived>& factors) {
933 EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
934 const evaluator<Derived> coeffs(factors.derived());
935 scaled_product<typename Derived::Scalar> product;
936 for (Index i = scaled_prod_blocks(coeffs, factors.size(), product); i < factors.size(); ++i)
937 product.multiply(coeffs.coeff(i));
942template <
typename Derived>
943EIGEN_DEVICE_FUNC scaled_product<typename Derived::Scalar> scaled_prod_squared(
const DenseBase<Derived>& factors) {
944 scaled_product<typename Derived::Scalar> product = scaled_prod(factors);
945 product.multiply(product);
951template <
typename Derived>
952EIGEN_DEVICE_FUNC
typename NumTraits<typename Derived::Scalar>::Real log_abs_prod(
const DenseBase<Derived>& factors,
954 return scaled_prod(factors).logAbs();
956template <
typename Derived>
957EIGEN_DEVICE_FUNC
typename NumTraits<typename Derived::Scalar>::Real log_abs_prod(
const DenseBase<Derived>& factors,
959 return factors.derived().array().abs().log().sum();
961template <
typename Derived>
962EIGEN_DEVICE_FUNC
typename NumTraits<typename Derived::Scalar>::Real log_abs_prod(
const DenseBase<Derived>& factors) {
963 return log_abs_prod(factors, bool_constant<has_scaled_product<typename Derived::Scalar>::value>());