Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GeneralBlockPanelKernel.h
1// IWYU pragma: private
2// SPDX-FileCopyrightText: The Eigen Authors
3// SPDX-License-Identifier: MPL-2.0
4#include "../../InternalHeaderCheck.h"
5
6namespace Eigen {
7namespace internal {
8
9#if EIGEN_ARCH_ARM
10template <>
11struct gebp_traits<double, double, false, false, Architecture::NEON, GEBPPacketFull>
12 : gebp_traits<double, double, false, false, Architecture::Generic, GEBPPacketFull> {
13 // AArch32 has 32 scalar double registers, enough for a 3x4 tile despite having only 16 NEON registers.
14 static constexpr int mr = 3;
15 // Keep scalar-aligned loads: pbroadcast4's unaligned memcpy loads spill through the stack on ARM32 GCC.
16 using RhsPacketx4 = RhsPacket;
17};
18#endif
19
20#if EIGEN_ARCH_ARM && EIGEN_COMP_CLANG
21
22// Clang seems to excessively spill registers in the GEBP kernel on 32-bit arm.
23// Here we specialize gebp_traits to eliminate these register spills.
24// See #2138.
25template <>
26struct gebp_traits<float, float, false, false, Architecture::NEON, GEBPPacketFull>
27 : gebp_traits<float, float, false, false, Architecture::Generic, GEBPPacketFull> {
28 EIGEN_STRONG_INLINE void acc(const AccPacket& c, const ResPacket& alpha, ResPacket& r) const {
29 // This volatile inline ASM both acts as a barrier to prevent reordering,
30 // as well as enforces strict register use.
31 asm volatile("vmla.f32 %q[r], %q[c], %q[alpha]" : [r] "+w"(r) : [c] "w"(c), [alpha] "w"(alpha) :);
32 }
33
34 template <typename LaneIdType>
35 EIGEN_STRONG_INLINE void madd(const Packet4f& a, const Packet4f& b, Packet4f& c, Packet4f&, const LaneIdType&) const {
36 acc(a, b, c);
37 }
38
39 template <typename LaneIdType>
40 EIGEN_STRONG_INLINE void madd(const Packet4f& a, const QuadPacket<Packet4f>& b, Packet4f& c, Packet4f& tmp,
41 const LaneIdType& lane) const {
42 madd(a, b.get(lane), c, tmp, lane);
43 }
44};
45
46#endif // EIGEN_ARCH_ARM && EIGEN_COMP_CLANG
47
48#if EIGEN_ARCH_ARM64
49
50#ifndef EIGEN_NEON_GEBP_NR
51#define EIGEN_NEON_GEBP_NR 8
52#endif
53
54template <>
55struct gebp_traits<float, float, false, false, Architecture::NEON, GEBPPacketFull>
56 : gebp_traits<float, float, false, false, Architecture::Generic, GEBPPacketFull> {
57 using RhsPacket = float;
58 using RhsPacketx4 = float32x4_t;
59 enum { nr = EIGEN_NEON_GEBP_NR };
60 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; }
61
62 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const { dest = vld1q_f32(b); }
63
64 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; }
65
66 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
67
68 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b, dest); }
69
70 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/,
71 const FixedInt<0>&) const {
72 c = vfmaq_n_f32(c, a, b);
73 }
74 // NOTE: Template parameter inference failed when compiled with Android NDK:
75 // "candidate template ignored: could not match 'FixedInt<N>' against 'Eigen::internal::FixedInt<0>".
76
77 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
78 const FixedInt<0>&) const {
79 madd_helper<0>(a, b, c);
80 }
81 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
82 const FixedInt<1>&) const {
83 madd_helper<1>(a, b, c);
84 }
85 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
86 const FixedInt<2>&) const {
87 madd_helper<2>(a, b, c);
88 }
89 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
90 const FixedInt<3>&) const {
91 madd_helper<3>(a, b, c);
92 }
93
94 private:
95 template <int LaneID>
96 EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const {
97#if EIGEN_GNUC_STRICT_LESS_THAN(9, 0, 0)
98 // 1. workaround gcc issue https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89101
99 // vfmaq_laneq_f32 is implemented through a costly dup, which was fixed in gcc9
100 // 2. workaround the gcc register split problem on arm64-neon
101 if (LaneID == 0)
102 asm("fmla %0.4s, %1.4s, %2.s[0]\n" : "+w"(c) : "w"(a), "w"(b) :);
103 else if (LaneID == 1)
104 asm("fmla %0.4s, %1.4s, %2.s[1]\n" : "+w"(c) : "w"(a), "w"(b) :);
105 else if (LaneID == 2)
106 asm("fmla %0.4s, %1.4s, %2.s[2]\n" : "+w"(c) : "w"(a), "w"(b) :);
107 else if (LaneID == 3)
108 asm("fmla %0.4s, %1.4s, %2.s[3]\n" : "+w"(c) : "w"(a), "w"(b) :);
109#else
110 c = vfmaq_laneq_f32(c, a, b, LaneID);
111#endif
112 }
113};
114
115template <>
116struct gebp_traits<double, double, false, false, Architecture::NEON>
117 : gebp_traits<double, double, false, false, Architecture::Generic> {
118 using RhsPacket = double;
119 enum { nr = EIGEN_NEON_GEBP_NR };
120 struct RhsPacketx4 {
121 float64x2_t B_0, B_1;
122 };
123
124 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; }
125
126 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
127 dest.B_0 = vld1q_f64(b);
128 dest.B_1 = vld1q_f64(b + 2);
129 }
130
131 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b, dest); }
132
133 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
134
135 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b, dest); }
136
137 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/,
138 const FixedInt<0>&) const {
139 c = vfmaq_n_f64(c, a, b);
140 }
141
142 // NOTE: Template parameter inference failed when compiled with Android NDK:
143 // "candidate template ignored: could not match 'FixedInt<N>' against 'Eigen::internal::FixedInt<0>".
144
145 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
146 const FixedInt<0>&) const {
147 madd_helper<0>(a, b, c);
148 }
149 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
150 const FixedInt<1>&) const {
151 madd_helper<1>(a, b, c);
152 }
153 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
154 const FixedInt<2>&) const {
155 madd_helper<2>(a, b, c);
156 }
157 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
158 const FixedInt<3>&) const {
159 madd_helper<3>(a, b, c);
160 }
161
162 private:
163 template <int LaneID>
164 EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const {
165#if EIGEN_GNUC_STRICT_LESS_THAN(9, 0, 0)
166 // 1. workaround gcc issue https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89101
167 // vfmaq_laneq_f64 is implemented through a costly dup, which was fixed in gcc9
168 // 2. workaround the gcc register split problem on arm64-neon
169 if (LaneID == 0)
170 asm("fmla %0.2d, %1.2d, %2.d[0]\n" : "+w"(c) : "w"(a), "w"(b.B_0) :);
171 else if (LaneID == 1)
172 asm("fmla %0.2d, %1.2d, %2.d[1]\n" : "+w"(c) : "w"(a), "w"(b.B_0) :);
173 else if (LaneID == 2)
174 asm("fmla %0.2d, %1.2d, %2.d[0]\n" : "+w"(c) : "w"(a), "w"(b.B_1) :);
175 else if (LaneID == 3)
176 asm("fmla %0.2d, %1.2d, %2.d[1]\n" : "+w"(c) : "w"(a), "w"(b.B_1) :);
177#else
178 if (LaneID == 0)
179 c = vfmaq_laneq_f64(c, a, b.B_0, 0);
180 else if (LaneID == 1)
181 c = vfmaq_laneq_f64(c, a, b.B_0, 1);
182 else if (LaneID == 2)
183 c = vfmaq_laneq_f64(c, a, b.B_1, 0);
184 else if (LaneID == 3)
185 c = vfmaq_laneq_f64(c, a, b.B_1, 1);
186#endif
187 }
188};
189
190// The register at operand 3 of fmla for data type half must be v0~v15, the compiler may not
191// allocate a required register for the '%2' of inline asm 'fmla %0.8h, %1.8h, %2.h[id]',
192// so inline assembly can't be used here to avoid the bug that vfmaq_lane_f16 is implemented
193// through a costly dup in gcc compiler.
194#if EIGEN_HAS_ARM64_FP16 && EIGEN_COMP_CLANG
195
196template <>
197struct gebp_traits<half, half, false, false, Architecture::NEON>
198 : gebp_traits<half, half, false, false, Architecture::Generic> {
199 using RhsPacket = half;
200 using RhsPacketx4 = float16x4_t;
201 using PacketHalf = float16x4_t;
202 enum { nr = EIGEN_NEON_GEBP_NR };
203
204 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; }
205
206 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
207 dest = vld1_f16(reinterpret_cast<const float16_t*>(b));
208 }
209
210 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const { dest = *b; }
211
212 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
213
214 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar*, RhsPacket&) const {
215 // If LHS is a Packet8h, we cannot correctly mimic a ploadquad of the RHS
216 // using a single scalar value.
217 eigen_assert(false && "Cannot loadRhsQuad for a scalar RHS.");
218 }
219
220 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/,
221 const FixedInt<0>&) const {
222#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
223 c = vfmaq_n_f16(c, a, b);
224#else
225 c = vcombine_f16(vcvt_f16_f32(vfmaq_n_f32(vcvt_f32_f16(vget_low_f16(c)), vcvt_f32_f16(vget_low_f16(a)), b)),
226 vcvt_f16_f32(vfmaq_n_f32(vcvt_f32_f16(vget_high_f16(c)), vcvt_f32_f16(vget_high_f16(a)), b)));
227#endif
228 }
229 EIGEN_STRONG_INLINE void madd(const PacketHalf& a, const RhsPacket& b, PacketHalf& c, RhsPacket& /*tmp*/,
230 const FixedInt<0>&) const {
231#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
232 c = vfma_n_f16(c, a, b);
233#else
234 c = vcvt_f16_f32(vfmaq_n_f32(vcvt_f32_f16(c), vcvt_f32_f16(a), b));
235#endif
236 }
237
238 // NOTE: Template parameter inference failed when compiled with Android NDK:
239 // "candidate template ignored: could not match 'FixedInt<N>' against 'Eigen::internal::FixedInt<0>".
240 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
241 const FixedInt<0>&) const {
242 madd_helper<0>(a, b, c);
243 }
244 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
245 const FixedInt<1>&) const {
246 madd_helper<1>(a, b, c);
247 }
248 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
249 const FixedInt<2>&) const {
250 madd_helper<2>(a, b, c);
251 }
252 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
253 const FixedInt<3>&) const {
254 madd_helper<3>(a, b, c);
255 }
256
257 private:
258 template <int LaneID>
259 EIGEN_STRONG_INLINE void madd_helper(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c) const {
260#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
261 c = vfmaq_lane_f16(c, a, b, LaneID);
262#else
263 float32x2_t bf = get_half<LaneID>(vcvt_f32_f16(b));
264 c = vcombine_f16(
265 vcvt_f16_f32(vfmaq_lane_f32(vcvt_f32_f16(vget_low_f16(c)), vcvt_f32_f16(vget_low_f16(a)), bf, LaneID % 2)),
266 vcvt_f16_f32(vfmaq_lane_f32(vcvt_f32_f16(vget_high_f16(c)), vcvt_f32_f16(vget_high_f16(a)), bf, LaneID % 2)));
267#endif
268 }
269
270#if !EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
271 template <int LaneID>
272 EIGEN_STRONG_INLINE static std::enable_if_t<(LaneID <= 1), float32x2_t> get_half(float32x4_t vec) {
273 return vget_low_f32(vec);
274 }
275
276 template <int LaneID>
277 EIGEN_STRONG_INLINE static std::enable_if_t<(LaneID > 1), float32x2_t> get_half(float32x4_t vec) {
278 return vget_high_f32(vec);
279 }
280#endif
281};
282#endif // EIGEN_HAS_ARM64_FP16 && EIGEN_COMP_CLANG
283#endif // EIGEN_ARCH_ARM64
284
285} // namespace internal
286} // namespace Eigen