36#ifndef EIGEN_INVERSE_SIZE_4_H
37#define EIGEN_INVERSE_SIZE_4_H
40#include "../InternalHeaderCheck.h"
42#if EIGEN_COMP_GNUC_STRICT
45#pragma GCC push_options
46#pragma GCC optimize("no-fast-math")
54template <
typename ResultType>
55EIGEN_STRONG_INLINE
bool inverse_size4_has_unit_inner_stride(
const ResultType &result) {
56 return ResultType::InnerStrideAtCompileTime == 1 || result.innerStride() == 1;
59template <
typename Scalar,
typename ResultType>
62template <
typename MatrixType,
typename ResultType>
63struct compute_inverse_size4<Architecture::Target, float, MatrixType, ResultType> {
65 MatrixAlignment = traits<MatrixType>::Alignment,
66 ResultAlignment = traits<ResultType>::Alignment,
69 using ActualMatrixType =
70 std::conditional_t<(MatrixType::Flags &
LinearAccessBit),
const MatrixType &,
typename MatrixType::PlainObject>;
72 static void run(
const MatrixType &mat, ResultType &result) {
73 if (EIGEN_PREDICT_FALSE(!inverse_size4_has_unit_inner_stride(result))) {
74 run_strided_result(mat, result);
76 run_unit_inner_stride(mat, result);
80 static EIGEN_DONT_INLINE
void run_strided_result(
const MatrixType &mat, ResultType &result) {
81 inverse_size4_contiguous_result_t<float, ResultType> contiguous;
82 compute_inverse_size4<Architecture::Target, float, MatrixType,
decltype(contiguous)>::run(mat, contiguous);
86 static void run_unit_inner_stride(
const MatrixType &mat, ResultType &result) {
87 ActualMatrixType matrix(mat);
89 const float *data = matrix.data();
90 const Index stride = matrix.innerStride();
91 Packet4f L1 = ploadt<Packet4f, MatrixAlignment>(data);
92 Packet4f L2 = ploadt<Packet4f, MatrixAlignment>(data + stride * 4);
93 Packet4f L3 = ploadt<Packet4f, MatrixAlignment>(data + stride * 8);
94 Packet4f L4 = ploadt<Packet4f, MatrixAlignment>(data + stride * 12);
101 EIGEN_IF_CONSTEXPR (!StorageOrdersMatch) {
102 A = vec4f_unpacklo(L1, L2);
103 B = vec4f_unpacklo(L3, L4);
104 C = vec4f_unpackhi(L1, L2);
105 D = vec4f_unpackhi(L3, L4);
107 A = vec4f_movelh(L1, L2);
108 B = vec4f_movehl(L2, L1);
109 C = vec4f_movelh(L3, L4);
110 D = vec4f_movehl(L4, L3);
116 AB = pmul(vec4f_swizzle2(A, A, 3, 3, 0, 0), B);
117 AB = pnmadd(vec4f_swizzle2(A, A, 1, 1, 2, 2), vec4f_swizzle2(B, B, 2, 3, 0, 1), AB);
120 DC = pmul(vec4f_swizzle2(D, D, 3, 3, 0, 0), C);
121 DC = pnmadd(vec4f_swizzle2(D, D, 1, 1, 2, 2), vec4f_swizzle2(C, C, 2, 3, 0, 1), DC);
124 Packet4f dA, dB, dC, dD;
126 dA = pmul(vec4f_swizzle2(A, A, 3, 3, 1, 1), A);
127 dA = psub(dA, vec4f_movehl(dA, dA));
129 dB = pmul(vec4f_swizzle2(B, B, 3, 3, 1, 1), B);
130 dB = psub(dB, vec4f_movehl(dB, dB));
132 dC = pmul(vec4f_swizzle2(C, C, 3, 3, 1, 1), C);
133 dC = psub(dC, vec4f_movehl(dC, dC));
135 dD = pmul(vec4f_swizzle2(D, D, 3, 3, 1, 1), D);
136 dD = psub(dD, vec4f_movehl(dD, dD));
140 d = pmul(vec4f_swizzle2(DC, DC, 0, 2, 1, 3), AB);
141 d = padd(d, vec4f_movehl(d, d));
142 d = padd(d, vec4f_swizzle2(d, d, 1, 0, 0, 0));
147 Packet4f det = vec4f_duplane(psub(padd(d1, d2), d), 0);
150 Packet4f rd = preciprocal(det);
153 Packet4f iA, iB, iC, iD;
156 iD = pmul(vec4f_swizzle2(C, C, 0, 0, 2, 2), vec4f_movelh(AB, AB));
157 iD = pmadd(vec4f_swizzle2(C, C, 1, 1, 3, 3), vec4f_movehl(AB, AB), iD);
158 iD = pmsub(D, vec4f_duplane(dA, 0), iD);
161 iA = pmul(vec4f_swizzle2(B, B, 0, 0, 2, 2), vec4f_movelh(DC, DC));
162 iA = pmadd(vec4f_swizzle2(B, B, 1, 1, 3, 3), vec4f_movehl(DC, DC), iA);
163 iA = pmsub(A, vec4f_duplane(dD, 0), iA);
166 iB = pmul(D, vec4f_swizzle2(AB, AB, 3, 0, 3, 0));
167 iB = pnmadd(vec4f_swizzle2(D, D, 1, 0, 3, 2), vec4f_swizzle2(AB, AB, 2, 1, 2, 1), iB);
168 iB = pmsub(C, vec4f_duplane(dB, 0), iB);
171 iC = pmul(A, vec4f_swizzle2(DC, DC, 3, 0, 3, 0));
172 iC = pnmadd(vec4f_swizzle2(A, A, 1, 0, 3, 2), vec4f_swizzle2(DC, DC, 2, 1, 2, 1), iC);
173 iC = pmsub(B, vec4f_duplane(dC, 0), iC);
176 const float neg_zero = numext::bit_cast<float>(0x80000000u);
177 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet4f>::alignment)
178 const float sign_mask[4] = {0.0f, neg_zero, neg_zero, 0.0f};
179 const Packet4f p4f_sign_PNNP = pload<Packet4f>(sign_mask);
180 rd = pxor(rd, p4f_sign_PNNP);
186 Index res_stride = result.outerStride();
187 float *res = result.data();
189 pstoret<float, Packet4f, ResultAlignment>(res + 0, vec4f_swizzle2(iA, iB, 3, 1, 3, 1));
190 pstoret<float, Packet4f, ResultAlignment>(res + res_stride, vec4f_swizzle2(iA, iB, 2, 0, 2, 0));
191 pstoret<float, Packet4f, ResultAlignment>(res + 2 * res_stride, vec4f_swizzle2(iC, iD, 3, 1, 3, 1));
192 pstoret<float, Packet4f, ResultAlignment>(res + 3 * res_stride, vec4f_swizzle2(iC, iD, 2, 0, 2, 0));
196#if !defined(EIGEN_VECTORIZE_NEON) || EIGEN_ARCH_ARM64
199template <
typename MatrixType,
typename ResultType>
200struct compute_inverse_size4<Architecture::Target, double, MatrixType, ResultType> {
202 MatrixAlignment = traits<MatrixType>::Alignment,
203 ResultAlignment = traits<ResultType>::Alignment,
206 using ActualMatrixType =
207 std::conditional_t<(MatrixType::Flags &
LinearAccessBit),
const MatrixType &,
typename MatrixType::PlainObject>;
209 static void run(
const MatrixType &mat, ResultType &result) {
210 if (EIGEN_PREDICT_FALSE(!inverse_size4_has_unit_inner_stride(result))) {
211 run_strided_result(mat, result);
213 run_unit_inner_stride(mat, result);
217 static EIGEN_DONT_INLINE
void run_strided_result(
const MatrixType &mat, ResultType &result) {
218 inverse_size4_contiguous_result_t<double, ResultType> contiguous;
219 compute_inverse_size4<Architecture::Target, double, MatrixType,
decltype(contiguous)>::run(mat, contiguous);
223 static void run_unit_inner_stride(
const MatrixType &mat, ResultType &result) {
224 ActualMatrixType matrix(mat);
233 Packet2d A1, A2, B1, B2, C1, C2, D1, D2;
235 const double *data = matrix.data();
236 const Index stride = matrix.innerStride();
237 EIGEN_IF_CONSTEXPR (StorageOrdersMatch) {
238 A1 = ploadt<Packet2d, MatrixAlignment>(data + stride * 0);
239 B1 = ploadt<Packet2d, MatrixAlignment>(data + stride * 2);
240 A2 = ploadt<Packet2d, MatrixAlignment>(data + stride * 4);
241 B2 = ploadt<Packet2d, MatrixAlignment>(data + stride * 6);
242 C1 = ploadt<Packet2d, MatrixAlignment>(data + stride * 8);
243 D1 = ploadt<Packet2d, MatrixAlignment>(data + stride * 10);
244 C2 = ploadt<Packet2d, MatrixAlignment>(data + stride * 12);
245 D2 = ploadt<Packet2d, MatrixAlignment>(data + stride * 14);
248 A1 = ploadt<Packet2d, MatrixAlignment>(data + stride * 0);
249 C1 = ploadt<Packet2d, MatrixAlignment>(data + stride * 2);
250 A2 = ploadt<Packet2d, MatrixAlignment>(data + stride * 4);
251 C2 = ploadt<Packet2d, MatrixAlignment>(data + stride * 6);
253 A1 = vec2d_unpacklo(A1, A2);
254 A2 = vec2d_unpackhi(temp, A2);
257 C1 = vec2d_unpacklo(C1, C2);
258 C2 = vec2d_unpackhi(temp, C2);
260 B1 = ploadt<Packet2d, MatrixAlignment>(data + stride * 8);
261 D1 = ploadt<Packet2d, MatrixAlignment>(data + stride * 10);
262 B2 = ploadt<Packet2d, MatrixAlignment>(data + stride * 12);
263 D2 = ploadt<Packet2d, MatrixAlignment>(data + stride * 14);
266 B1 = vec2d_unpacklo(B1, B2);
267 B2 = vec2d_unpackhi(temp, B2);
270 D1 = vec2d_unpacklo(D1, D2);
271 D2 = vec2d_unpackhi(temp, D2);
275 Packet2d dA, dB, dC, dD;
277 dA = vec2d_swizzle2(A2, A2, 1);
279 dA = psub(dA, vec2d_duplane(dA, 1));
281 dB = vec2d_swizzle2(B2, B2, 1);
283 dB = psub(dB, vec2d_duplane(dB, 1));
285 dC = vec2d_swizzle2(C2, C2, 1);
287 dC = psub(dC, vec2d_duplane(dC, 1));
289 dD = vec2d_swizzle2(D2, D2, 1);
291 dD = psub(dD, vec2d_duplane(dD, 1));
293 Packet2d DC1, DC2, AB1, AB2;
296 AB1 = pmul(B1, vec2d_duplane(A2, 1));
297 AB2 = pmul(B2, vec2d_duplane(A1, 0));
298 AB1 = pnmadd(B2, vec2d_duplane(A1, 1), AB1);
299 AB2 = pnmadd(B1, vec2d_duplane(A2, 0), AB2);
302 DC1 = pmul(C1, vec2d_duplane(D2, 1));
303 DC2 = pmul(C2, vec2d_duplane(D1, 0));
304 DC1 = pnmadd(C2, vec2d_duplane(D1, 1), DC1);
305 DC2 = pnmadd(C1, vec2d_duplane(D2, 0), DC2);
315 d1 = pmul(AB1, vec2d_swizzle2(DC1, DC2, 0));
316 d2 = pmul(AB2, vec2d_swizzle2(DC1, DC2, 3));
318 rd = padd(rd, vec2d_duplane(rd, 1));
325 det = vec2d_duplane(det, 0);
326 rd = pdiv(pset1<Packet2d>(1.0), det);
329 Packet2d iA1, iA2, iB1, iB2, iC1, iC2, iD1, iD2;
332 iD1 = pmul(AB1, vec2d_duplane(C1, 0));
333 iD2 = pmul(AB1, vec2d_duplane(C2, 0));
334 iD1 = pmadd(AB2, vec2d_duplane(C1, 1), iD1);
335 iD2 = pmadd(AB2, vec2d_duplane(C2, 1), iD2);
336 dA = vec2d_duplane(dA, 0);
337 iD1 = pmsub(D1, dA, iD1);
338 iD2 = pmsub(D2, dA, iD2);
341 iA1 = pmul(DC1, vec2d_duplane(B1, 0));
342 iA2 = pmul(DC1, vec2d_duplane(B2, 0));
343 iA1 = pmadd(DC2, vec2d_duplane(B1, 1), iA1);
344 iA2 = pmadd(DC2, vec2d_duplane(B2, 1), iA2);
345 dD = vec2d_duplane(dD, 0);
346 iA1 = pmsub(A1, dD, iA1);
347 iA2 = pmsub(A2, dD, iA2);
350 iB1 = pmul(D1, vec2d_swizzle2(AB2, AB1, 1));
351 iB2 = pmul(D2, vec2d_swizzle2(AB2, AB1, 1));
352 iB1 = pnmadd(vec2d_swizzle2(D1, D1, 1), vec2d_swizzle2(AB2, AB1, 2), iB1);
353 iB2 = pnmadd(vec2d_swizzle2(D2, D2, 1), vec2d_swizzle2(AB2, AB1, 2), iB2);
354 dB = vec2d_duplane(dB, 0);
355 iB1 = pmsub(C1, dB, iB1);
356 iB2 = pmsub(C2, dB, iB2);
359 iC1 = pmul(A1, vec2d_swizzle2(DC2, DC1, 1));
360 iC2 = pmul(A2, vec2d_swizzle2(DC2, DC1, 1));
361 iC1 = pnmadd(vec2d_swizzle2(A1, A1, 1), vec2d_swizzle2(DC2, DC1, 2), iC1);
362 iC2 = pnmadd(vec2d_swizzle2(A2, A2, 1), vec2d_swizzle2(DC2, DC1, 2), iC2);
363 dC = vec2d_duplane(dC, 0);
364 iC1 = pmsub(B1, dC, iC1);
365 iC2 = pmsub(B2, dC, iC2);
368 const double neg_zero = numext::bit_cast<double>(0x8000000000000000ull);
369 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2d>::alignment)
const double sign_mask1[2] = {0.0, neg_zero};
370 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2d>::alignment)
const double sign_mask2[2] = {neg_zero, 0.0};
371 const Packet2d sign_PN = pload<Packet2d>(sign_mask1);
372 const Packet2d sign_NP = pload<Packet2d>(sign_mask2);
373 d1 = pxor(rd, sign_PN);
374 d2 = pxor(rd, sign_NP);
376 Index res_stride = result.outerStride();
377 double *res = result.data();
378 pstoret<double, Packet2d, ResultAlignment>(res + 0, pmul(vec2d_swizzle2(iA2, iA1, 3), d1));
379 pstoret<double, Packet2d, ResultAlignment>(res + res_stride, pmul(vec2d_swizzle2(iA2, iA1, 0), d2));
380 pstoret<double, Packet2d, ResultAlignment>(res + 2, pmul(vec2d_swizzle2(iB2, iB1, 3), d1));
381 pstoret<double, Packet2d, ResultAlignment>(res + res_stride + 2, pmul(vec2d_swizzle2(iB2, iB1, 0), d2));
382 pstoret<double, Packet2d, ResultAlignment>(res + 2 * res_stride, pmul(vec2d_swizzle2(iC2, iC1, 3), d1));
383 pstoret<double, Packet2d, ResultAlignment>(res + 3 * res_stride, pmul(vec2d_swizzle2(iC2, iC1, 0), d2));
384 pstoret<double, Packet2d, ResultAlignment>(res + 2 * res_stride + 2, pmul(vec2d_swizzle2(iD2, iD1, 3), d1));
385 pstoret<double, Packet2d, ResultAlignment>(res + 3 * res_stride + 2, pmul(vec2d_swizzle2(iD2, iD1, 0), d2));
392#if EIGEN_COMP_GNUC_STRICT
393#pragma GCC pop_options
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321
constexpr unsigned int LinearAccessBit
Definition Constants.h:134
constexpr unsigned int RowMajorBit
Definition Constants.h:71