Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
MathFunctions.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2007 Julien Pommier
5// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12/* The sin and cos functions of this file come from
13 * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/
14 */
15
16#ifndef EIGEN_MATH_FUNCTIONS_SSE_H
17#define EIGEN_MATH_FUNCTIONS_SSE_H
18
19// IWYU pragma: private
20#include "../../InternalHeaderCheck.h"
21
22namespace Eigen {
23
24namespace internal {
25
26EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(Packet4f)
27EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_DOUBLE(Packet2d)
28
29// Notice that for newer processors, it is counterproductive to use Newton
30// iteration for square root. In particular, Skylake and Zen2 processors
31// have approximately doubled throughput of the _mm_sqrt_ps instruction
32// compared to their predecessors.
33template <>
34EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f psqrt<Packet4f>(const Packet4f& x) {
35 return _mm_sqrt_ps(x);
36}
37template <>
38EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d psqrt<Packet2d>(const Packet2d& x) {
39 return _mm_sqrt_pd(x);
40}
41template <>
42EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet16b psqrt<Packet16b>(const Packet16b& x) {
43 return x;
44}
45
46#if EIGEN_FAST_MATH
47// Even on Skylake, using Newton iteration is a win for reciprocal square root. rsqrtps reads a subnormal x as zero,
48// giving +-inf, so the subnormal lanes, 1 <= bits(|x|) <= 0x007fffff, take 1 / sqrt(x) instead. Adding 0x7f800000
49// moves exactly that range above 0x7f800000 and wraps every larger value negative. With SSE this test of x alone is
50// faster than the test of x * estimate in prsqrt<Packet8f>, which wins with AVX. Under DAZ the fallback reads a
51// subnormal as zero too, and returns the estimate's +-inf.
52template <>
53EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f prsqrt<Packet4f>(const Packet4f& x) {
54 const Packet4f r = generic_rsqrt_newton_step<Packet4f, /*Steps=*/1>::run(x, _mm_rsqrt_ps(x));
55 const Packet4i bias = pset1<Packet4i>(0x7f800000);
56 const Packet4i abs_bits = pand(Packet4i(_mm_castps_si128(x)), pset1<Packet4i>(0x7fffffff));
57 const Packet4f redo = _mm_castsi128_ps(pcmp_lt(bias, padd(abs_bits, bias)));
58 return predux_any(redo) ? pselect(redo, pdiv(pset1<Packet4f>(1.0f), psqrt(x)), r) : r;
59}
60
61#ifdef EIGEN_VECTORIZE_FMA
62// Trying to speed up reciprocal using Newton-Raphson is counterproductive
63// unless FMA is available. Without FMA pdiv(pset1<Packet>(Scalar(1),a)) is
64// 30% faster.
65template <>
66EIGEN_STRONG_INLINE Packet4f preciprocal<Packet4f>(const Packet4f& x) {
67#ifdef EIGEN_VECTORIZE_AVX
68 // generic_reciprocal_newton_step, with its NaN-or-zero test as a single compare: r == 0 or unordered.
69 const Packet4f one = pset1<Packet4f>(1.0f);
70 const Packet4f r0 = _mm_rcp_ps(x);
71 const Packet4f refined = pmadd(r0, pnmadd(x, r0, one), r0);
72 const Packet4f redo = _mm_cmp_ps(refined, _mm_setzero_ps(), _CMP_EQ_UQ);
73 return predux_any(redo) ? pselect(redo, pdiv(one, x), refined) : refined;
74#else
75 return generic_reciprocal_newton_step<Packet4f, /*Steps=*/1>::run(x, _mm_rcp_ps(x));
76#endif
77}
78#endif
79
80#endif
81
82} // end namespace internal
83
84namespace numext {
85
86template <>
87EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float sqrt(const float& x) {
88 return internal::pfirst(internal::Packet4f(_mm_sqrt_ss(_mm_set_ss(x))));
89}
90
91template <>
92EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double sqrt(const double& x) {
93#if EIGEN_COMP_GNUC_STRICT
94 // This works around a GCC bug generating poor code for _mm_sqrt_pd
95 // See https://gitlab.com/libeigen/eigen/commit/8dca9f97e38970
96 return internal::pfirst(internal::Packet2d(__builtin_ia32_sqrtsd(_mm_set_sd(x))));
97#else
98 return internal::pfirst(internal::Packet2d(_mm_sqrt_pd(_mm_set_sd(x))));
99#endif
100}
101
102} // namespace numext
103
104} // end namespace Eigen
105
106#endif // EIGEN_MATH_FUNCTIONS_SSE_H