Eigen  5.0.1
 
Loading...
Searching...
No Matches
GeneralBlockPanelKernel.h
1// IWYU pragma: private
2// SPDX-FileCopyrightText: The Eigen Authors
3// SPDX-License-Identifier: MPL-2.0
4#include "../../InternalHeaderCheck.h"
5
6namespace Eigen {
7namespace internal {
8
9#if EIGEN_ARCH_ARM && EIGEN_COMP_CLANG
10
11// Clang seems to excessively spill registers in the GEBP kernel on 32-bit arm.
12// Here we specialize gebp_traits to eliminate these register spills.
13// See #2138.
14template <>
15struct gebp_traits<float, float, false, false, Architecture::NEON, GEBPPacketFull>
16 : gebp_traits<float, float, false, false, Architecture::Generic, GEBPPacketFull> {
17 EIGEN_STRONG_INLINE void acc(const AccPacket& c, const ResPacket& alpha, ResPacket& r) const {
18 // This volatile inline ASM both acts as a barrier to prevent reordering,
19 // as well as enforces strict register use.
20 asm volatile("vmla.f32 %q[r], %q[c], %q[alpha]" : [r] "+w"(r) : [c] "w"(c), [alpha] "w"(alpha) :);
21 }
22
23 template <typename LaneIdType>
24 EIGEN_STRONG_INLINE void madd(const Packet4f& a, const Packet4f& b, Packet4f& c, Packet4f&, const LaneIdType&) const {
25 acc(a, b, c);
26 }
27
28 template <typename LaneIdType>
29 EIGEN_STRONG_INLINE void madd(const Packet4f& a, const QuadPacket<Packet4f>& b, Packet4f& c, Packet4f& tmp,
30 const LaneIdType& lane) const {
31 madd(a, b.get(lane), c, tmp, lane);
32 }
33};
34
35#endif // EIGEN_ARCH_ARM && EIGEN_COMP_CLANG
36
37#if EIGEN_ARCH_ARM64
38
39#ifndef EIGEN_NEON_GEBP_NR
40#define EIGEN_NEON_GEBP_NR 8
41#endif
42
43template <>
44struct gebp_traits<float, float, false, false, Architecture::NEON, GEBPPacketFull>
45 : gebp_traits<float, float, false, false, Architecture::Generic, GEBPPacketFull> {
46 typedef float RhsPacket;
47 typedef float32x4_t RhsPacketx4;
48 enum { nr = EIGEN_NEON_GEBP_NR };
49 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; }
50
51 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const { dest = vld1q_f32(b); }
52
53 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; }
54
55 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
56
57 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b, dest); }
58
59 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/,
60 const FixedInt<0>&) const {
61 c = vfmaq_n_f32(c, a, b);
62 }
63 // NOTE: Template parameter inference failed when compiled with Android NDK:
64 // "candidate template ignored: could not match 'FixedInt<N>' against 'Eigen::internal::FixedInt<0>".
65
66 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
67 const FixedInt<0>&) const {
68 madd_helper<0>(a, b, c);
69 }
70 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
71 const FixedInt<1>&) const {
72 madd_helper<1>(a, b, c);
73 }
74 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
75 const FixedInt<2>&) const {
76 madd_helper<2>(a, b, c);
77 }
78 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
79 const FixedInt<3>&) const {
80 madd_helper<3>(a, b, c);
81 }
82
83 private:
84 template <int LaneID>
85 EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const {
86#if EIGEN_GNUC_STRICT_LESS_THAN(9, 0, 0)
87 // 1. workaround gcc issue https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89101
88 // vfmaq_laneq_f32 is implemented through a costly dup, which was fixed in gcc9
89 // 2. workaround the gcc register split problem on arm64-neon
90 if (LaneID == 0)
91 asm("fmla %0.4s, %1.4s, %2.s[0]\n" : "+w"(c) : "w"(a), "w"(b) :);
92 else if (LaneID == 1)
93 asm("fmla %0.4s, %1.4s, %2.s[1]\n" : "+w"(c) : "w"(a), "w"(b) :);
94 else if (LaneID == 2)
95 asm("fmla %0.4s, %1.4s, %2.s[2]\n" : "+w"(c) : "w"(a), "w"(b) :);
96 else if (LaneID == 3)
97 asm("fmla %0.4s, %1.4s, %2.s[3]\n" : "+w"(c) : "w"(a), "w"(b) :);
98#else
99 c = vfmaq_laneq_f32(c, a, b, LaneID);
100#endif
101 }
102};
103
104template <>
105struct gebp_traits<double, double, false, false, Architecture::NEON>
106 : gebp_traits<double, double, false, false, Architecture::Generic> {
107 typedef double RhsPacket;
108 enum { nr = EIGEN_NEON_GEBP_NR };
109 struct RhsPacketx4 {
110 float64x2_t B_0, B_1;
111 };
112
113 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; }
114
115 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
116 dest.B_0 = vld1q_f64(b);
117 dest.B_1 = vld1q_f64(b + 2);
118 }
119
120 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b, dest); }
121
122 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
123
124 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b, dest); }
125
126 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/,
127 const FixedInt<0>&) const {
128 c = vfmaq_n_f64(c, a, b);
129 }
130
131 // NOTE: Template parameter inference failed when compiled with Android NDK:
132 // "candidate template ignored: could not match 'FixedInt<N>' against 'Eigen::internal::FixedInt<0>".
133
134 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
135 const FixedInt<0>&) const {
136 madd_helper<0>(a, b, c);
137 }
138 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
139 const FixedInt<1>&) const {
140 madd_helper<1>(a, b, c);
141 }
142 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
143 const FixedInt<2>&) const {
144 madd_helper<2>(a, b, c);
145 }
146 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
147 const FixedInt<3>&) const {
148 madd_helper<3>(a, b, c);
149 }
150
151 private:
152 template <int LaneID>
153 EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const {
154#if EIGEN_GNUC_STRICT_LESS_THAN(9, 0, 0)
155 // 1. workaround gcc issue https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89101
156 // vfmaq_laneq_f64 is implemented through a costly dup, which was fixed in gcc9
157 // 2. workaround the gcc register split problem on arm64-neon
158 if (LaneID == 0)
159 asm("fmla %0.2d, %1.2d, %2.d[0]\n" : "+w"(c) : "w"(a), "w"(b.B_0) :);
160 else if (LaneID == 1)
161 asm("fmla %0.2d, %1.2d, %2.d[1]\n" : "+w"(c) : "w"(a), "w"(b.B_0) :);
162 else if (LaneID == 2)
163 asm("fmla %0.2d, %1.2d, %2.d[0]\n" : "+w"(c) : "w"(a), "w"(b.B_1) :);
164 else if (LaneID == 3)
165 asm("fmla %0.2d, %1.2d, %2.d[1]\n" : "+w"(c) : "w"(a), "w"(b.B_1) :);
166#else
167 if (LaneID == 0)
168 c = vfmaq_laneq_f64(c, a, b.B_0, 0);
169 else if (LaneID == 1)
170 c = vfmaq_laneq_f64(c, a, b.B_0, 1);
171 else if (LaneID == 2)
172 c = vfmaq_laneq_f64(c, a, b.B_1, 0);
173 else if (LaneID == 3)
174 c = vfmaq_laneq_f64(c, a, b.B_1, 1);
175#endif
176 }
177};
178
179// The register at operand 3 of fmla for data type half must be v0~v15, the compiler may not
180// allocate a required register for the '%2' of inline asm 'fmla %0.8h, %1.8h, %2.h[id]',
181// so inline assembly can't be used here to avoid the bug that vfmaq_lane_f16 is implemented
182// through a costly dup in gcc compiler.
183#if EIGEN_HAS_ARM64_FP16 && EIGEN_COMP_CLANG
184
185template <>
186struct gebp_traits<half, half, false, false, Architecture::NEON>
187 : gebp_traits<half, half, false, false, Architecture::Generic> {
188 typedef half RhsPacket;
189 typedef float16x4_t RhsPacketx4;
190 typedef float16x4_t PacketHalf;
191 enum { nr = EIGEN_NEON_GEBP_NR };
192
193 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; }
194
195 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const { dest = vld1_f16(&b->x); }
196
197 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; }
198
199 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
200
201 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar*, RhsPacket&) const {
202 // If LHS is a Packet8h, we cannot correctly mimic a ploadquad of the RHS
203 // using a single scalar value.
204 eigen_assert(false && "Cannot loadRhsQuad for a scalar RHS.");
205 }
206
207 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/,
208 const FixedInt<0>&) const {
209#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
210 c = vfmaq_n_f16(c, a, b);
211#else
212 c = vcombine_f16(vcvt_f16_f32(vfmaq_n_f32(vcvt_f32_f16(vget_low_f16(c)), vcvt_f32_f16(vget_low_f16(a)), b)),
213 vcvt_f16_f32(vfmaq_n_f32(vcvt_f32_f16(vget_high_f16(c)), vcvt_f32_f16(vget_high_f16(a)), b)));
214#endif
215 }
216 EIGEN_STRONG_INLINE void madd(const PacketHalf& a, const RhsPacket& b, PacketHalf& c, RhsPacket& /*tmp*/,
217 const FixedInt<0>&) const {
218#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
219 c = vfma_n_f16(c, a, b);
220#else
221 c = vcvt_f16_f32(vfmaq_n_f32(vcvt_f32_f16(c), vcvt_f32_f16(a), b));
222#endif
223 }
224
225 // NOTE: Template parameter inference failed when compiled with Android NDK:
226 // "candidate template ignored: could not match 'FixedInt<N>' against 'Eigen::internal::FixedInt<0>".
227 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
228 const FixedInt<0>&) const {
229 madd_helper<0>(a, b, c);
230 }
231 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
232 const FixedInt<1>&) const {
233 madd_helper<1>(a, b, c);
234 }
235 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
236 const FixedInt<2>&) const {
237 madd_helper<2>(a, b, c);
238 }
239 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
240 const FixedInt<3>&) const {
241 madd_helper<3>(a, b, c);
242 }
243
244 private:
245 template <int LaneID>
246 EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const {
247#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
248 c = vfmaq_lane_f16(c, a, b, LaneID);
249#else
250 float32x2_t bf = get_half<LaneID>(vcvt_f32_f16(b));
251 c = vcombine_f16(
252 vcvt_f16_f32(vfmaq_lane_f32(vcvt_f32_f16(vget_low_f16(c)), vcvt_f32_f16(vget_low_f16(a)), bf, LaneID % 2)),
253 vcvt_f16_f32(vfmaq_lane_f32(vcvt_f32_f16(vget_high_f16(c)), vcvt_f32_f16(vget_high_f16(a)), bf, LaneID % 2)));
254#endif
255 }
256
257#if !EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
258 template <int LaneID>
259 EIGEN_STRONG_INLINE static std::enable_if_t<(LaneID <= 1), float32x2_t> get_half(float32x4_t vec) {
260 return vget_low_f32(vec);
261 }
262
263 template <int LaneID>
264 EIGEN_STRONG_INLINE static std::enable_if_t<(LaneID > 1), float32x2_t> get_half(float32x4_t vec) {
265 return vget_high_f32(vec);
266 }
267#endif
268};
269#endif // EIGEN_HAS_ARM64_FP16 && EIGEN_COMP_CLANG
270#endif // EIGEN_ARCH_ARM64
271
272} // namespace internal
273} // namespace Eigen