4#include "../../InternalHeaderCheck.h"
11struct gebp_traits<double, double, false, false, Architecture::NEON, GEBPPacketFull>
12 : gebp_traits<double, double, false, false, Architecture::Generic, GEBPPacketFull> {
14 static constexpr int mr = 3;
16 using RhsPacketx4 = RhsPacket;
20#if EIGEN_ARCH_ARM && EIGEN_COMP_CLANG
26struct gebp_traits<float, float, false, false, Architecture::NEON, GEBPPacketFull>
27 : gebp_traits<float, float, false, false, Architecture::Generic, GEBPPacketFull> {
28 EIGEN_STRONG_INLINE
void acc(
const AccPacket& c,
const ResPacket& alpha, ResPacket& r)
const {
31 asm volatile(
"vmla.f32 %q[r], %q[c], %q[alpha]" : [r]
"+w"(r) : [c]
"w"(c), [alpha]
"w"(alpha) :);
34 template <
typename LaneIdType>
35 EIGEN_STRONG_INLINE
void madd(
const Packet4f& a,
const Packet4f& b, Packet4f& c, Packet4f&,
const LaneIdType&)
const {
39 template <
typename LaneIdType>
40 EIGEN_STRONG_INLINE
void madd(
const Packet4f& a,
const QuadPacket<Packet4f>& b, Packet4f& c, Packet4f& tmp,
41 const LaneIdType& lane)
const {
42 madd(a, b.get(lane), c, tmp, lane);
50#ifndef EIGEN_NEON_GEBP_NR
51#define EIGEN_NEON_GEBP_NR 8
55struct gebp_traits<float, float, false, false, Architecture::NEON, GEBPPacketFull>
56 : gebp_traits<float, float, false, false, Architecture::Generic, GEBPPacketFull> {
57 using RhsPacket = float;
58 using RhsPacketx4 = float32x4_t;
59 enum { nr = EIGEN_NEON_GEBP_NR };
60 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacket& dest)
const { dest = *b; }
62 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const { dest = vld1q_f32(b); }
64 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, RhsPacket& dest)
const { dest = *b; }
66 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
68 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, RhsPacket& dest)
const { loadRhs(b, dest); }
70 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacket& b, AccPacket& c, RhsPacket& ,
71 const FixedInt<0>&)
const {
72 c = vfmaq_n_f32(c, a, b);
77 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
78 const FixedInt<0>&)
const {
79 madd_helper<0>(a, b, c);
81 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
82 const FixedInt<1>&)
const {
83 madd_helper<1>(a, b, c);
85 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
86 const FixedInt<2>&)
const {
87 madd_helper<2>(a, b, c);
89 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
90 const FixedInt<3>&)
const {
91 madd_helper<3>(a, b, c);
96 EIGEN_STRONG_INLINE
void madd_helper(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c)
const {
97#if EIGEN_GNUC_STRICT_LESS_THAN(9, 0, 0)
102 asm(
"fmla %0.4s, %1.4s, %2.s[0]\n" :
"+w"(c) :
"w"(a),
"w"(b) :);
103 else if (LaneID == 1)
104 asm(
"fmla %0.4s, %1.4s, %2.s[1]\n" :
"+w"(c) :
"w"(a),
"w"(b) :);
105 else if (LaneID == 2)
106 asm(
"fmla %0.4s, %1.4s, %2.s[2]\n" :
"+w"(c) :
"w"(a),
"w"(b) :);
107 else if (LaneID == 3)
108 asm(
"fmla %0.4s, %1.4s, %2.s[3]\n" :
"+w"(c) :
"w"(a),
"w"(b) :);
110 c = vfmaq_laneq_f32(c, a, b, LaneID);
116struct gebp_traits<double, double, false, false, Architecture::NEON>
117 : gebp_traits<double, double, false, false, Architecture::Generic> {
118 using RhsPacket = double;
119 enum { nr = EIGEN_NEON_GEBP_NR };
121 float64x2_t B_0, B_1;
124 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacket& dest)
const { dest = *b; }
126 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
127 dest.B_0 = vld1q_f64(b);
128 dest.B_1 = vld1q_f64(b + 2);
131 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, RhsPacket& dest)
const { loadRhs(b, dest); }
133 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
135 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, RhsPacket& dest)
const { loadRhs(b, dest); }
137 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacket& b, AccPacket& c, RhsPacket& ,
138 const FixedInt<0>&)
const {
139 c = vfmaq_n_f64(c, a, b);
145 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
146 const FixedInt<0>&)
const {
147 madd_helper<0>(a, b, c);
149 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
150 const FixedInt<1>&)
const {
151 madd_helper<1>(a, b, c);
153 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
154 const FixedInt<2>&)
const {
155 madd_helper<2>(a, b, c);
157 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
158 const FixedInt<3>&)
const {
159 madd_helper<3>(a, b, c);
163 template <
int LaneID>
164 EIGEN_STRONG_INLINE
void madd_helper(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c)
const {
165#if EIGEN_GNUC_STRICT_LESS_THAN(9, 0, 0)
170 asm(
"fmla %0.2d, %1.2d, %2.d[0]\n" :
"+w"(c) :
"w"(a),
"w"(b.B_0) :);
171 else if (LaneID == 1)
172 asm(
"fmla %0.2d, %1.2d, %2.d[1]\n" :
"+w"(c) :
"w"(a),
"w"(b.B_0) :);
173 else if (LaneID == 2)
174 asm(
"fmla %0.2d, %1.2d, %2.d[0]\n" :
"+w"(c) :
"w"(a),
"w"(b.B_1) :);
175 else if (LaneID == 3)
176 asm(
"fmla %0.2d, %1.2d, %2.d[1]\n" :
"+w"(c) :
"w"(a),
"w"(b.B_1) :);
179 c = vfmaq_laneq_f64(c, a, b.B_0, 0);
180 else if (LaneID == 1)
181 c = vfmaq_laneq_f64(c, a, b.B_0, 1);
182 else if (LaneID == 2)
183 c = vfmaq_laneq_f64(c, a, b.B_1, 0);
184 else if (LaneID == 3)
185 c = vfmaq_laneq_f64(c, a, b.B_1, 1);
194#if EIGEN_HAS_ARM64_FP16 && EIGEN_COMP_CLANG
197struct gebp_traits<half, half, false, false, Architecture::NEON>
198 : gebp_traits<half, half, false, false, Architecture::Generic> {
199 using RhsPacket = half;
200 using RhsPacketx4 = float16x4_t;
201 using PacketHalf = float16x4_t;
202 enum { nr = EIGEN_NEON_GEBP_NR };
204 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacket& dest)
const { dest = *b; }
206 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
207 dest = vld1_f16(
reinterpret_cast<const float16_t*
>(b));
210 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, RhsPacket& dest)
const { dest = *b; }
212 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
214 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar*, RhsPacket&)
const {
217 eigen_assert(
false &&
"Cannot loadRhsQuad for a scalar RHS.");
220 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacket& b, AccPacket& c, RhsPacket& ,
221 const FixedInt<0>&)
const {
222#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
223 c = vfmaq_n_f16(c, a, b);
225 c = vcombine_f16(vcvt_f16_f32(vfmaq_n_f32(vcvt_f32_f16(vget_low_f16(c)), vcvt_f32_f16(vget_low_f16(a)), b)),
226 vcvt_f16_f32(vfmaq_n_f32(vcvt_f32_f16(vget_high_f16(c)), vcvt_f32_f16(vget_high_f16(a)), b)));
229 EIGEN_STRONG_INLINE
void madd(
const PacketHalf& a,
const RhsPacket& b, PacketHalf& c, RhsPacket& ,
230 const FixedInt<0>&)
const {
231#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
232 c = vfma_n_f16(c, a, b);
234 c = vcvt_f16_f32(vfmaq_n_f32(vcvt_f32_f16(c), vcvt_f32_f16(a), b));
240 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
241 const FixedInt<0>&)
const {
242 madd_helper<0>(a, b, c);
244 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
245 const FixedInt<1>&)
const {
246 madd_helper<1>(a, b, c);
248 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
249 const FixedInt<2>&)
const {
250 madd_helper<2>(a, b, c);
252 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
253 const FixedInt<3>&)
const {
254 madd_helper<3>(a, b, c);
258 template <
int LaneID>
259 EIGEN_STRONG_INLINE
void madd_helper(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c)
const {
260#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
261 c = vfmaq_lane_f16(c, a, b, LaneID);
263 float32x2_t bf = get_half<LaneID>(vcvt_f32_f16(b));
265 vcvt_f16_f32(vfmaq_lane_f32(vcvt_f32_f16(vget_low_f16(c)), vcvt_f32_f16(vget_low_f16(a)), bf, LaneID % 2)),
266 vcvt_f16_f32(vfmaq_lane_f32(vcvt_f32_f16(vget_high_f16(c)), vcvt_f32_f16(vget_high_f16(a)), bf, LaneID % 2)));
270#if !EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
271 template <
int LaneID>
272 EIGEN_STRONG_INLINE
static std::enable_if_t<(LaneID <= 1), float32x2_t> get_half(float32x4_t vec) {
273 return vget_low_f32(vec);
276 template <
int LaneID>
277 EIGEN_STRONG_INLINE
static std::enable_if_t<(LaneID > 1), float32x2_t> get_half(float32x4_t vec) {
278 return vget_high_f32(vec);