11#ifndef EIGEN_GENERAL_BLOCK_PANEL_H
12#define EIGEN_GENERAL_BLOCK_PANEL_H
15#include "../InternalHeaderCheck.h"
21#pragma warning(disable : 4804)
28enum GEBPPacketSizeType { GEBPPacketFull = 0, GEBPPacketHalf, GEBPPacketQuarter };
30template <
typename LhsScalar_,
typename RhsScalar_,
bool ConjLhs_ =
false,
bool ConjRhs_ =
false,
31 int Arch = Architecture::Target,
int PacketSize_ = GEBPPacketFull>
35inline std::ptrdiff_t manage_caching_sizes_helper(std::ptrdiff_t a, std::ptrdiff_t b) {
return a <= 0 ? b : a; }
37#if defined(EIGEN_DEFAULT_L1_CACHE_SIZE)
38#define EIGEN_SET_DEFAULT_L1_CACHE_SIZE(val) EIGEN_DEFAULT_L1_CACHE_SIZE
40#define EIGEN_SET_DEFAULT_L1_CACHE_SIZE(val) val
43#if defined(EIGEN_DEFAULT_L2_CACHE_SIZE)
44#define EIGEN_SET_DEFAULT_L2_CACHE_SIZE(val) EIGEN_DEFAULT_L2_CACHE_SIZE
46#define EIGEN_SET_DEFAULT_L2_CACHE_SIZE(val) val
49#if defined(EIGEN_DEFAULT_L3_CACHE_SIZE)
50#define EIGEN_SET_DEFAULT_L3_CACHE_SIZE(val) EIGEN_DEFAULT_L3_CACHE_SIZE
52#define EIGEN_SET_DEFAULT_L3_CACHE_SIZE(val) val
55#if EIGEN_ARCH_i386_OR_x86_64
56const std::ptrdiff_t defaultL1CacheSize = EIGEN_SET_DEFAULT_L1_CACHE_SIZE(32 * 1024);
57const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(256 * 1024);
58const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(2 * 1024 * 1024);
60const std::ptrdiff_t defaultL1CacheSize = EIGEN_SET_DEFAULT_L1_CACHE_SIZE(64 * 1024);
62const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(2 * 1024 * 1024);
63const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(8 * 1024 * 1024);
65const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(512 * 1024);
66const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(4 * 1024 * 1024);
68#elif EIGEN_ARCH_ARM_OR_ARM64
69const std::ptrdiff_t defaultL1CacheSize = EIGEN_SET_DEFAULT_L1_CACHE_SIZE(64 * 1024);
70const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(1024 * 1024);
71const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(4 * 1024 * 1024);
73const std::ptrdiff_t defaultL1CacheSize = EIGEN_SET_DEFAULT_L1_CACHE_SIZE(16 * 1024);
74const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(512 * 1024);
75const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(512 * 1024);
78#undef EIGEN_SET_DEFAULT_L1_CACHE_SIZE
79#undef EIGEN_SET_DEFAULT_L2_CACHE_SIZE
80#undef EIGEN_SET_DEFAULT_L3_CACHE_SIZE
84 CacheSizes() : m_l1(-1), m_l2(-1), m_l3(-1), m_l3_per_cpu(0) {
85 std::ptrdiff_t l1CacheSize, l2CacheSize, l3CacheSize;
86 queryCacheSizes(l1CacheSize, l2CacheSize, l3CacheSize, m_l3_per_cpu);
87 m_l1 = manage_caching_sizes_helper(l1CacheSize, defaultL1CacheSize);
88 m_l2 = manage_caching_sizes_helper(l2CacheSize, defaultL2CacheSize);
89 m_l3 = manage_caching_sizes_helper(l3CacheSize, defaultL3CacheSize);
97 std::ptrdiff_t m_l3_per_cpu;
101inline void manage_caching_sizes(Action action, std::ptrdiff_t* l1, std::ptrdiff_t* l2, std::ptrdiff_t* l3,
102 std::ptrdiff_t* l3_per_cpu =
nullptr) {
103 static CacheSizes m_cacheSizes;
105 if (action == SetAction) {
107 eigen_internal_assert(l1 != 0 && l2 != 0);
108 m_cacheSizes.m_l1 = *l1;
109 m_cacheSizes.m_l2 = *l2;
110 m_cacheSizes.m_l3 = *l3;
111 m_cacheSizes.m_l3_per_cpu = l3_per_cpu !=
nullptr ? *l3_per_cpu : 0;
112 }
else if (action == GetAction) {
113 eigen_internal_assert(l1 != 0 && l2 != 0);
114 *l1 = m_cacheSizes.m_l1;
115 *l2 = m_cacheSizes.m_l2;
116 if (l3_per_cpu !=
nullptr) *l3_per_cpu = m_cacheSizes.m_l3_per_cpu;
117 *l3 = m_cacheSizes.m_l3;
119 eigen_internal_assert(
false);
137#if EIGEN_COMP_GNUC_STRICT
138#define EIGEN_GEMM_NO_CLONE __attribute__((noclone))
140#define EIGEN_GEMM_NO_CLONE
148struct gebp_overwrite_t {};
149template <
typename Gebp,
typename =
void>
150struct gebp_kernel_overwrites : std::false_type {};
151template <
typename Gebp>
152struct gebp_kernel_overwrites<Gebp, std::enable_if_t<Gebp::CanOverwrite>> : std::true_type {};
156template <
typename Gebp,
typename Index>
157EIGEN_ALWAYS_INLINE
bool gebp_overwrite_pays_impl(
int, Index rows, Index cols, Index depth,
158 decltype(Gebp::overwrite_pays(rows, cols, depth))* =
nullptr) {
159 return Gebp::overwrite_pays(rows, cols, depth);
161template <
typename Gebp,
typename Index>
162EIGEN_ALWAYS_INLINE
bool gebp_overwrite_pays_impl(
long, Index, Index, Index) {
165template <
typename Gebp,
typename Index>
166EIGEN_ALWAYS_INLINE
bool gebp_overwrite_pays(Index rows, Index cols, Index depth) {
167 return gebp_overwrite_pays_impl<Gebp>(0, rows, cols, depth);
170#ifdef EIGEN_VECTORIZE_SME
175template <
typename LhsScalar,
typename RhsScalar>
176struct sme_has_gebp_kernel : std::false_type {};
178struct sme_has_gebp_kernel<float, float> : std::true_type {};
179#ifdef EIGEN_VECTORIZE_SME_F64F64
181struct sme_has_gebp_kernel<double, double> : std::true_type {};
185template <
typename RealScalar>
186struct sme_has_gebp_kernel<std::complex<RealScalar>, std::complex<RealScalar>>
187 : sme_has_gebp_kernel<RealScalar, RealScalar> {};
192#ifndef EIGEN_SME_MAX_KC
193#define EIGEN_SME_MAX_KC 2048
195#ifndef EIGEN_SME_PACKED_RHS_BUDGET_BYTES
196#define EIGEN_SME_PACKED_RHS_BUDGET_BYTES (32 * 1024 * 1024)
198#ifndef EIGEN_SME_LHS_WORKING_SET_BUDGET_BYTES
199#define EIGEN_SME_LHS_WORKING_SET_BUDGET_BYTES (7 * 1024 * 1024)
201#ifndef EIGEN_SME_SINGLE_PASS_RHS_BUDGET_BYTES
202#define EIGEN_SME_SINGLE_PASS_RHS_BUDGET_BYTES (4 * 1024 * 1024)
205template <
typename LhsScalar,
typename RhsScalar,
typename Index>
206void evaluateProductBlockingSizesHeuristicForSme(Index& k, Index& m, Index& n) {
207 using Traits = gebp_traits<LhsScalar, RhsScalar>;
209 const Index mr =
static_cast<Index
>(Traits::mr);
210 const Index nr =
static_cast<Index
>(Traits::nr);
212#ifdef EIGEN_DEBUG_SMALL_PRODUCT_BLOCKS
217 constexpr Index sme_max_kc =
static_cast<Index
>(128);
218 constexpr Index sme_packed_rhs_budget_bytes =
static_cast<Index
>(128 * 1024);
219 constexpr Index sme_lhs_working_set_budget_bytes =
static_cast<Index
>(128 * 1024);
220 constexpr Index sme_single_pass_rhs_budget_bytes =
static_cast<Index
>(128 * 1024);
222 constexpr Index sme_max_kc =
static_cast<Index
>(EIGEN_SME_MAX_KC);
223 constexpr Index sme_packed_rhs_budget_bytes =
static_cast<Index
>(EIGEN_SME_PACKED_RHS_BUDGET_BYTES);
224 constexpr Index sme_lhs_working_set_budget_bytes =
static_cast<Index
>(EIGEN_SME_LHS_WORKING_SET_BUDGET_BYTES);
225 constexpr Index sme_single_pass_rhs_budget_bytes =
static_cast<Index
>(EIGEN_SME_SINGLE_PASS_RHS_BUDGET_BYTES);
232 const Index max_kc = (numext::maxi)(Index(1), sme_max_kc * Index(
sizeof(
float)) / Index(
sizeof(LhsScalar)));
233 k = (numext::mini)(k, max_kc);
235 const Index block_b_hot_bytes = k * nr * Index(
sizeof(RhsScalar));
236 const Index min_lhs_bytes = mr * k * Index(
sizeof(LhsScalar));
237 const Index block_a_bytes = sme_lhs_working_set_budget_bytes > block_b_hot_bytes
238 ? sme_lhs_working_set_budget_bytes - block_b_hot_bytes
240 Index mc = block_a_bytes / (k * Index(
sizeof(LhsScalar)));
245 const Index rhs_budget = m <= mc ? (numext::mini)(sme_packed_rhs_budget_bytes, sme_single_pass_rhs_budget_bytes)
246 : sme_packed_rhs_budget_bytes;
247 Index nc = rhs_budget / (numext::maxi)(Index(1), k * Index(
sizeof(RhsScalar)));
249 n = (numext::mini)(n, (numext::maxi)(nr, nc));
251 m = (numext::mini)(m, (numext::maxi)(mr, mc));
255template <
typename LhsScalar,
typename RhsScalar,
int KcFactor,
typename Index>
256void evaluateProductBlockingSizesHeuristic(Index& k, Index& m, Index& n, Index num_threads = 1) {
257 using Traits = gebp_traits<LhsScalar, RhsScalar>;
264 std::ptrdiff_t l1, l2, l3, l3_per_cpu;
265 manage_caching_sizes(GetAction, &l1, &l2, &l3, &l3_per_cpu);
266#ifdef EIGEN_VECTORIZE_AVX512
267 const std::ptrdiff_t phys_l1 = l1;
278 if (num_threads > 1) {
279 using ResScalar =
typename Traits::ResScalar;
281 kdiv = KcFactor * (Traits::mr *
sizeof(LhsScalar) + Traits::nr *
sizeof(RhsScalar)),
282 ksub = Traits::mr * (Traits::nr *
sizeof(ResScalar)),
292 const Index k_cache = numext::maxi<Index>(kr, (numext::mini<Index>)(
static_cast<Index
>((l1 - ksub) / kdiv), 320));
294 k = k_cache - (k_cache % kr);
295 eigen_internal_assert(k > 0);
298 const Index n_cache =
static_cast<Index
>((l2 - l1) / (nr *
sizeof(RhsScalar) * k));
299 const Index nr_index =
static_cast<Index
>(nr);
302 const Index n_cache_aligned = n_cache >= nr_index ? n_cache - (n_cache % nr_index) : nr_index;
303 const Index n_per_thread = numext::div_ceil(n, num_threads);
304 if (n_cache <= n_per_thread) {
307 n = (numext::mini<Index>)(n, n_cache_aligned);
308 eigen_internal_assert(n > 0);
310 n = (numext::mini<Index>)(n, (n_per_thread + nr - 1) - ((n_per_thread + nr - 1) % nr));
315 const Index m_cache =
static_cast<Index
>((l3 - l2) / (
sizeof(LhsScalar) * k * num_threads));
316 const Index m_per_thread = numext::div_ceil(m, num_threads);
317 if (m_cache < m_per_thread && m_cache >=
static_cast<Index
>(mr)) {
318 m = m_cache - (m_cache % mr);
319 eigen_internal_assert(m > 0);
321 m = (numext::mini<Index>)(m, (m_per_thread + mr - 1) - ((m_per_thread + mr - 1) % mr));
327#ifdef EIGEN_DEBUG_SMALL_PRODUCT_BLOCKS
339 if ((numext::maxi)(k, (numext::maxi)(m, n)) < 48)
return;
341#ifdef EIGEN_VECTORIZE_SME
344 EIGEN_IF_CONSTEXPR ((sme_has_gebp_kernel<LhsScalar, RhsScalar>::value)) {
345 evaluateProductBlockingSizesHeuristicForSme<LhsScalar, RhsScalar>(k, m, n);
350 using ResScalar =
typename Traits::ResScalar;
353 k_div = KcFactor * (Traits::mr *
sizeof(LhsScalar) + Traits::nr *
sizeof(RhsScalar)),
354 k_sub = Traits::mr * (Traits::nr *
sizeof(ResScalar))
364 const Index max_kc = numext::maxi<Index>(
static_cast<Index
>(((l1 - k_sub) / k_div) & (~(k_peeling - 1))), 1);
365 const Index old_k = k;
370 k = (k % max_kc) == 0 ? max_kc
371 : max_kc - k_peeling * ((max_kc - 1 - (k % max_kc)) / (k_peeling * (k / max_kc + 1)));
373 eigen_internal_assert(((old_k / k) == (old_k / max_kc)) &&
"the number of sweeps has to remain the same");
376#ifdef EIGEN_VECTORIZE_AVX512
381 const Index phys_l1_eff = convert_index<Index>(phys_l1 * 85 / 100);
382 const Index max_kc_phys = numext::maxi<Index>(((phys_l1_eff - k_sub) / k_div) & (~(k_peeling - 1)), k_peeling);
383 if (max_kc_phys < k) {
384 k = (old_k % max_kc_phys) == 0 ? max_kc_phys
385 : max_kc_phys - k_peeling * ((max_kc_phys - 1 - (old_k % max_kc_phys)) /
386 (k_peeling * (old_k / max_kc_phys + 1)));
397#ifdef EIGEN_DEBUG_SMALL_PRODUCT_BLOCKS
398 const Index actual_l2 =
static_cast<Index
>(l3);
400 const Index actual_l2 =
static_cast<Index
>(l2 * 3 / 2);
410 const Index rhs_panel_budget = numext::maxi<Index>(actual_l2,
static_cast<Index
>(l3_per_cpu));
419 const Index lhs_bytes = m * k *
sizeof(LhsScalar);
420 const Index remaining_l1 =
static_cast<Index
>(l1 - k_sub - lhs_bytes);
421 if (remaining_l1 >= Index(Traits::nr *
sizeof(RhsScalar)) * k) {
423 max_nc = remaining_l1 / (k *
sizeof(RhsScalar));
427 max_nc = (3 * rhs_panel_budget) / (2 * 2 * k *
sizeof(RhsScalar));
430 Index nc = numext::mini<Index>(rhs_panel_budget / (2 * k *
sizeof(RhsScalar)), max_nc) & (~(Traits::nr - 1));
433 nc = numext::maxi<Index>(nc, Traits::nr);
439 n = (n % nc) == 0 ? nc : (nc - Traits::nr * ((nc - (n % nc)) / (Traits::nr * (n / nc + 1))));
440 }
else if (old_k == k) {
450 Index problem_size = k * n *
sizeof(LhsScalar);
451 Index actual_lm = actual_l2;
453 if (problem_size <= 1024) {
456 actual_lm =
static_cast<Index
>(l1);
457 }
else if (l3 != 0 && problem_size <= l1) {
462 actual_lm =
static_cast<Index
>(l2);
463 max_mc = (numext::mini<Index>)(576, max_mc);
465 Index mc = (numext::mini<Index>)(actual_lm / (3 * k *
sizeof(LhsScalar)), max_mc);
467 mc -= mc % Traits::mr;
470 m = (m % mc) == 0 ? mc : (mc - Traits::mr * ((mc - (m % mc)) / (Traits::mr * (m / mc + 1))));
475template <
typename Index>
476inline bool useSpecificBlockingSizes(Index& k, Index& m, Index& n) {
477#ifdef EIGEN_TEST_SPECIFIC_BLOCKING_SIZES
478 if (EIGEN_TEST_SPECIFIC_BLOCKING_SIZES) {
479 k = numext::mini<Index>(k, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_K);
480 m = numext::mini<Index>(m, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_M);
481 n = numext::mini<Index>(n, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_N);
485 EIGEN_UNUSED_VARIABLE(k);
486 EIGEN_UNUSED_VARIABLE(m);
487 EIGEN_UNUSED_VARIABLE(n);
510template <
typename LhsScalar,
typename RhsScalar,
int KcFactor,
typename Index>
511void computeProductBlockingSizes(Index& k, Index& m, Index& n, Index num_threads = 1) {
512 if (!useSpecificBlockingSizes(k, m, n)) {
513 evaluateProductBlockingSizesHeuristic<LhsScalar, RhsScalar, KcFactor, Index>(k, m, n, num_threads);
517template <
typename LhsScalar,
typename RhsScalar,
typename Index>
518inline void computeProductBlockingSizes(Index& k, Index& m, Index& n, Index num_threads = 1) {
519 computeProductBlockingSizes<LhsScalar, RhsScalar, 1, Index>(k, m, n, num_threads);
523template <
typename RhsPacket,
typename RhsPacketx4,
int registers_taken,
int rhs_registers = 1>
524struct RhsPanelHelper {
526 static constexpr int remaining_registers =
527 (std::max)(
int(EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS) - registers_taken, 0);
530 using type = std::conditional_t<remaining_registers >= 4 * rhs_registers, RhsPacketx4, RhsPacket>;
533template <
typename Packet>
535 Packet B_0, B1, B2, B3;
536 const Packet& get(
const FixedInt<0>&)
const {
return B_0; }
537 const Packet& get(
const FixedInt<1>&)
const {
return B1; }
538 const Packet& get(
const FixedInt<2>&)
const {
return B2; }
539 const Packet& get(
const FixedInt<3>&)
const {
return B3; }
542template <
int N,
typename T1,
typename T2,
typename T3>
543struct packet_conditional {
547template <
typename T1,
typename T2,
typename T3>
548struct packet_conditional<GEBPPacketFull, T1, T2, T3> {
552template <
typename T1,
typename T2,
typename T3>
553struct packet_conditional<GEBPPacketHalf, T1, T2, T3> {
557#define PACKET_DECL_COND_POSTFIX(postfix, name, packet_size) \
558 using name##Packet##postfix = \
559 typename packet_conditional<packet_size, typename packet_traits<name##Scalar>::type, \
560 typename packet_traits<name##Scalar>::half, \
561 typename unpacket_traits<typename packet_traits<name##Scalar>::half>::half>::type
563#define PACKET_DECL_COND(name, packet_size) \
564 using name##Packet = \
565 typename packet_conditional<packet_size, typename packet_traits<name##Scalar>::type, \
566 typename packet_traits<name##Scalar>::half, \
567 typename unpacket_traits<typename packet_traits<name##Scalar>::half>::half>::type
569#define PACKET_DECL_COND_SCALAR_POSTFIX(postfix, packet_size) \
570 using ScalarPacket##postfix = \
571 typename packet_conditional<packet_size, typename packet_traits<Scalar>::type, \
572 typename packet_traits<Scalar>::half, \
573 typename unpacket_traits<typename packet_traits<Scalar>::half>::half>::type
575#define PACKET_DECL_COND_SCALAR(packet_size) \
576 using ScalarPacket = \
577 typename packet_conditional<packet_size, typename packet_traits<Scalar>::type, \
578 typename packet_traits<Scalar>::half, \
579 typename unpacket_traits<typename packet_traits<Scalar>::half>::half>::type
591template <
typename LhsScalar_,
typename RhsScalar_,
bool ConjLhs_,
bool ConjRhs_,
int Arch,
int PacketSize_>
594 using LhsScalar = LhsScalar_;
595 using RhsScalar = RhsScalar_;
596 using ResScalar =
typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
598 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
599 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
600 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
605 Vectorizable = unpacket_traits<LhsPacket_>::vectorizable && unpacket_traits<RhsPacket_>::vectorizable,
606 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
607 RhsPacketSize = Vectorizable ? unpacket_traits<RhsPacket_>::size : 1,
608 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
610 NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS,
616 default_mr = (plain_enum_min(16, NumberOfRegisters) / 2 / nr) * LhsPacketSize,
617#
if defined(EIGEN_HAS_SINGLE_INSTRUCTION_MADD) && !defined(EIGEN_VECTORIZE_ALTIVEC) && \
618 !defined(EIGEN_VECTORIZE_VSX) && ((!EIGEN_COMP_MSVC) || (EIGEN_COMP_MSVC >= 1914))
623 mr = Vectorizable ? 3 * LhsPacketSize : default_mr,
628 LhsProgress = LhsPacketSize,
632 using LhsPacket = std::conditional_t<Vectorizable, LhsPacket_, LhsScalar>;
633 using RhsPacket = std::conditional_t<Vectorizable, RhsPacket_, RhsScalar>;
634 using ResPacket = std::conditional_t<Vectorizable, ResPacket_, ResScalar>;
635 using LhsPacket4Packing = LhsPacket;
637 using RhsPacketx4 = QuadPacket<RhsPacket>;
638 using AccPacket = ResPacket;
640 EIGEN_STRONG_INLINE
void initAcc(AccPacket& p)
const { p = pset1<ResPacket>(ResScalar(0)); }
642 template <
typename RhsPacketType>
643 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketType& dest)
const {
644 dest = pset1<RhsPacketType>(*b);
647 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
648 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
651 template <
typename RhsPacketType>
652 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, RhsPacketType& dest)
const {
656 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
658 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, RhsPacket& dest)
const { dest = ploadquad<RhsPacket>(b); }
660 template <
typename LhsPacketType>
661 EIGEN_STRONG_INLINE
void loadLhs(
const LhsScalar* a, LhsPacketType& dest)
const {
662 dest = pload<LhsPacketType>(a);
665 template <
typename LhsPacketType>
666 EIGEN_STRONG_INLINE
void loadLhsUnaligned(
const LhsScalar* a, LhsPacketType& dest)
const {
667 dest = ploadu<LhsPacketType>(a);
670 template <
typename LhsPacketType,
typename RhsPacketType,
typename AccPacketType,
typename LaneIdType>
671 EIGEN_STRONG_INLINE
void madd(
const LhsPacketType& a,
const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp,
672 const LaneIdType&)
const {
673 conj_helper<LhsPacketType, RhsPacketType, ConjLhs, ConjRhs> cj;
678#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
679 EIGEN_UNUSED_VARIABLE(tmp);
680 c = cj.pmadd(a, b, c);
682 madd_unfused(a, b, c, tmp, cj, bool_constant<std::is_same<RhsPacketType,
decltype(cj.pmul(a, b))>::value>());
686 template <
typename LhsPacketType,
typename RhsPacketType,
typename AccPacketType,
typename Conjugator>
687 EIGEN_STRONG_INLINE
void madd_unfused(
const LhsPacketType& a,
const RhsPacketType& b, AccPacketType& c,
688 RhsPacketType& tmp,
const Conjugator& cj, std::true_type)
const {
690 tmp = cj.pmul(a, tmp);
694 template <
typename LhsPacketType,
typename RhsPacketType,
typename AccPacketType,
typename Conjugator>
695 EIGEN_STRONG_INLINE
void madd_unfused(
const LhsPacketType& a,
const RhsPacketType& b, AccPacketType& c,
696 RhsPacketType& tmp,
const Conjugator& cj, std::false_type)
const {
697 EIGEN_UNUSED_VARIABLE(tmp);
698 c = padd(c, cj.pmul(a, b));
701 template <
typename LhsPacketType,
typename AccPacketType,
typename LaneIdType>
702 EIGEN_STRONG_INLINE
void madd(
const LhsPacketType& a,
const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
703 const LaneIdType& lane)
const {
704 madd(a, b.get(lane), c, tmp, lane);
707 EIGEN_STRONG_INLINE
void acc(
const AccPacket& c,
const ResPacket& alpha, ResPacket& r)
const {
708 r = pmadd(c, alpha, r);
711 template <
typename ResPacketHalf>
712 EIGEN_STRONG_INLINE
void acc(
const ResPacketHalf& c,
const ResPacketHalf& alpha, ResPacketHalf& r)
const {
713 r = pmadd(c, alpha, r);
717template <
typename RealScalar,
bool ConjLhs_,
int Arch,
int PacketSize_>
718class gebp_traits<std::complex<RealScalar>, RealScalar, ConjLhs_, false, Arch, PacketSize_> {
720 using LhsScalar = std::complex<RealScalar>;
721 using RhsScalar = RealScalar;
722 using ResScalar =
typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
724 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
725 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
726 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
731 Vectorizable = unpacket_traits<LhsPacket_>::vectorizable && unpacket_traits<RhsPacket_>::vectorizable,
732 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
733 RhsPacketSize = Vectorizable ? unpacket_traits<RhsPacket_>::size : 1,
734 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
736 NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS,
738#if defined(EIGEN_HAS_SINGLE_INSTRUCTION_MADD) && !defined(EIGEN_VECTORIZE_ALTIVEC) && !defined(EIGEN_VECTORIZE_VSX)
740 mr = 3 * LhsPacketSize,
742 mr = (plain_enum_min(16, NumberOfRegisters) / 2 / nr) * LhsPacketSize,
745 LhsProgress = LhsPacketSize,
749 using LhsPacket = std::conditional_t<Vectorizable, LhsPacket_, LhsScalar>;
750 using RhsPacket = std::conditional_t<Vectorizable, RhsPacket_, RhsScalar>;
751 using ResPacket = std::conditional_t<Vectorizable, ResPacket_, ResScalar>;
752 using LhsPacket4Packing = LhsPacket;
754 using RhsPacketx4 = QuadPacket<RhsPacket>;
756 using AccPacket = ResPacket;
758 EIGEN_STRONG_INLINE
void initAcc(AccPacket& p)
const { p = pset1<ResPacket>(ResScalar(0)); }
760 template <
typename RhsPacketType>
761 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketType& dest)
const {
762 dest = pset1<RhsPacketType>(*b);
765 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
766 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
769 template <
typename RhsPacketType>
770 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, RhsPacketType& dest)
const {
774 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
776 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, RhsPacket& dest)
const {
777 loadRhsQuad_impl(b, dest, bool_constant<RhsPacketSize == 16>());
780 EIGEN_STRONG_INLINE
void loadRhsQuad_impl(
const RhsScalar* b, RhsPacket& dest,
const std::true_type&)
const {
782 RhsScalar tmp[4] = {b[0], b[0], b[1], b[1]};
783 dest = ploadquad<RhsPacket>(tmp);
786 EIGEN_STRONG_INLINE
void loadRhsQuad_impl(
const RhsScalar* b, RhsPacket& dest,
const std::false_type&)
const {
787 eigen_internal_assert(RhsPacketSize <= 8);
788 dest = pset1<RhsPacket>(*b);
791 EIGEN_STRONG_INLINE
void loadLhs(
const LhsScalar* a, LhsPacket& dest)
const { dest = pload<LhsPacket>(a); }
793 template <
typename LhsPacketType>
794 EIGEN_STRONG_INLINE
void loadLhsUnaligned(
const LhsScalar* a, LhsPacketType& dest)
const {
795 dest = ploadu<LhsPacketType>(a);
798 template <
typename LhsPacketType,
typename RhsPacketType,
typename AccPacketType,
typename LaneIdType>
799 EIGEN_STRONG_INLINE
void madd(
const LhsPacketType& a,
const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp,
800 const LaneIdType&)
const {
801 madd_impl(a, b, c, tmp, bool_constant<Vectorizable>());
804 template <
typename LhsPacketType,
typename RhsPacketType,
typename AccPacketType>
805 EIGEN_STRONG_INLINE
void madd_impl(
const LhsPacketType& a,
const RhsPacketType& b, AccPacketType& c,
806 RhsPacketType& tmp,
const std::true_type&)
const {
807#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
808 EIGEN_UNUSED_VARIABLE(tmp);
810 conj_helper<LhsPacketType, RhsPacketType, false, false> cj;
811 c = cj.pmadd(a, b, c);
814 tmp = pmul(a.v, tmp);
815 c.v = padd(c.v, tmp);
819 EIGEN_STRONG_INLINE
void madd_impl(
const LhsScalar& a,
const RhsScalar& b, ResScalar& c, RhsScalar& ,
820 const std::false_type&)
const {
824 template <
typename LhsPacketType,
typename AccPacketType,
typename LaneIdType>
825 EIGEN_STRONG_INLINE
void madd(
const LhsPacketType& a,
const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
826 const LaneIdType& lane)
const {
827 madd(a, b.get(lane), c, tmp, lane);
830 template <
typename ResPacketType,
typename AccPacketType>
831 EIGEN_STRONG_INLINE
void acc(
const AccPacketType& c,
const ResPacketType& alpha, ResPacketType& r)
const {
832 conj_helper<ResPacketType, ResPacketType, ConjLhs, false> cj;
833 r = cj.pmadd(c, alpha, r);
837template <
typename Packet>
843template <
typename Packet>
844struct is_double_packet : std::false_type {};
845template <
typename Packet>
846struct is_double_packet<DoublePacket<Packet>> : std::true_type {};
848template <
typename Packet>
849DoublePacket<Packet> padd(
const DoublePacket<Packet>& a,
const DoublePacket<Packet>& b) {
850 DoublePacket<Packet> res;
851 res.first = padd(a.first, b.first);
852 res.second = padd(a.second, b.second);
856template <typename Packet, std::enable_if_t<unpacket_traits<Packet>::size <= 8,
int> = 0>
857const DoublePacket<Packet>& predux_half(
const DoublePacket<Packet>& a) {
861template <typename Packet, std::enable_if_t<unpacket_traits<Packet>::size >= 16 &&
862 !NumTraits<typename unpacket_traits<Packet>::type>::IsComplex,
864DoublePacket<typename unpacket_traits<Packet>::half> predux_half(
const DoublePacket<Packet>& a) {
866 DoublePacket<typename unpacket_traits<Packet>::half> res;
867 using Cplx = std::complex<typename unpacket_traits<Packet>::type>;
868 using CplxPacket =
typename packet_traits<Cplx>::type;
869 res.first = predux_half(CplxPacket(a.first)).v;
870 res.second = predux_half(CplxPacket(a.second)).v;
875template <typename Scalar, typename RealPacket, std::enable_if_t<unpacket_traits<RealPacket>::size <= 8,
int> = 0>
876void loadQuadToDoublePacket(
const Scalar* b, DoublePacket<RealPacket>& dest) {
877 dest.first = pset1<RealPacket>(numext::real(*b));
878 dest.second = pset1<RealPacket>(numext::imag(*b));
885template <
typename Scalar,
typename RealPacket, std::enable_if_t<(unpacket_traits<RealPacket>::size > 8),
int> = 0>
886void loadQuadToDoublePacket(
const Scalar* b, DoublePacket<RealPacket>& dest) {
887 using RealScalar =
typename NumTraits<Scalar>::Real;
888 constexpr int kQuads = unpacket_traits<RealPacket>::size / 4;
889 RealScalar r[kQuads], i[kQuads];
890 for (
int j = 0; j < kQuads; ++j) {
891 r[j] = numext::real(b[j / 2]);
892 i[j] = numext::imag(b[j / 2]);
894 dest.first = ploadquad<RealPacket>(r);
895 dest.second = ploadquad<RealPacket>(i);
898template <
typename Packet>
899struct unpacket_traits<DoublePacket<Packet>> {
900 using half = DoublePacket<typename unpacket_traits<Packet>::half>;
901 enum { size = 2 * unpacket_traits<Packet>::size };
904template <
typename RealScalar,
bool ConjLhs_,
bool ConjRhs_,
int Arch,
int PacketSize_>
905class gebp_traits<std::complex<RealScalar>, std::complex<RealScalar>, ConjLhs_, ConjRhs_, Arch, PacketSize_> {
907 using Scalar = std::complex<RealScalar>;
908 using LhsScalar = std::complex<RealScalar>;
909 using RhsScalar = std::complex<RealScalar>;
910 using ResScalar = std::complex<RealScalar>;
912 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
913 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
914 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
915 PACKET_DECL_COND(Real, PacketSize_);
916 PACKET_DECL_COND_SCALAR(PacketSize_);
921 Vectorizable = unpacket_traits<RealPacket>::vectorizable && unpacket_traits<ScalarPacket>::vectorizable,
922 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
923 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
924 RhsPacketSize = Vectorizable ? unpacket_traits<RhsScalar>::size : 1,
925 RealPacketSize = Vectorizable ? unpacket_traits<RealPacket>::size : 1,
926 NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS,
929#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
931 mr = plain_enum_max(1, NumberOfRegisters / 4 / nr) * ResPacketSize,
933 mr = (plain_enum_min(16, NumberOfRegisters) / 2 / nr) * ResPacketSize,
936 LhsProgress = ResPacketSize,
940 using DoublePacketType = DoublePacket<RealPacket>;
942 using LhsPacket4Packing = std::conditional_t<Vectorizable, ScalarPacket, Scalar>;
943 using LhsPacket = std::conditional_t<Vectorizable, RealPacket, Scalar>;
944 using RhsPacket = std::conditional_t<Vectorizable, DoublePacketType, Scalar>;
945 using ResPacket = std::conditional_t<Vectorizable, ScalarPacket, Scalar>;
946 using AccPacket = std::conditional_t<Vectorizable, DoublePacketType, Scalar>;
949 using RhsPacketx4 = QuadPacket<RhsPacket>;
951 EIGEN_STRONG_INLINE
void initAcc(Scalar& p)
const { p = Scalar(0); }
953 EIGEN_STRONG_INLINE
void initAcc(DoublePacketType& p)
const {
954 p.first = pset1<RealPacket>(RealScalar(0));
955 p.second = pset1<RealPacket>(RealScalar(0));
959 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, ScalarPacket& dest)
const { dest = pset1<ScalarPacket>(*b); }
962 template <
typename RealPacketType>
963 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, DoublePacket<RealPacketType>& dest)
const {
964 dest.first = pset1<RealPacketType>(numext::real(*b));
965 dest.second = pset1<RealPacketType>(numext::imag(*b));
968 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
969 loadRhs(b, dest.B_0);
970 loadRhs(b + 1, dest.B1);
971 loadRhs(b + 2, dest.B2);
972 loadRhs(b + 3, dest.B3);
976 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, ScalarPacket& dest)
const { loadRhs(b, dest); }
979 template <
typename RealPacketType>
980 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, DoublePacket<RealPacketType>& dest)
const {
984 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
986 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, ResPacket& dest)
const { loadRhs(b, dest); }
987 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, DoublePacketType& dest)
const {
988 loadQuadToDoublePacket(b, dest);
992 EIGEN_STRONG_INLINE
void loadLhs(
const LhsScalar* a, LhsPacket& dest)
const {
993 dest = pload<LhsPacket>((
const typename unpacket_traits<LhsPacket>::type*)(a));
996 template <
typename LhsPacketType>
997 EIGEN_STRONG_INLINE
void loadLhsUnaligned(
const LhsScalar* a, LhsPacketType& dest)
const {
998 dest = ploadu<LhsPacketType>((
const typename unpacket_traits<LhsPacketType>::type*)(a));
1001 template <
typename LhsPacketType,
typename RhsPacketType,
typename ResPacketType,
typename TmpType,
1002 typename LaneIdType>
1003 EIGEN_STRONG_INLINE std::enable_if_t<!std::is_same<RhsPacketType, RhsPacketx4>::value> madd(
1004 const LhsPacketType& a,
const RhsPacketType& b, DoublePacket<ResPacketType>& c, TmpType& ,
1005 const LaneIdType&)
const {
1006 c.first = pmadd(a, b.first, c.first);
1007 c.second = pmadd(a, b.second, c.second);
1010 template <
typename LaneIdType>
1011 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacket& b, ResPacket& c, RhsPacket& ,
1012 const LaneIdType&)
const {
1013 c = cj.pmadd(a, b, c);
1016 template <
typename LhsPacketType,
typename AccPacketType,
typename LaneIdType>
1017 EIGEN_STRONG_INLINE
void madd(
const LhsPacketType& a,
const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
1018 const LaneIdType& lane)
const {
1019 madd(a, b.get(lane), c, tmp, lane);
1022 EIGEN_STRONG_INLINE
void acc(
const Scalar& c,
const Scalar& alpha, Scalar& r)
const { r += alpha * c; }
1024 template <
typename RealPacketType,
typename ResPacketType>
1025 EIGEN_STRONG_INLINE
void acc(
const DoublePacket<RealPacketType>& c,
const ResPacketType& alpha,
1026 ResPacketType& r)
const {
1029 EIGEN_IF_CONSTEXPR ((!ConjLhs) && (!ConjRhs)) {
1030 tmp = pcplxflip(pconj(ResPacketType(c.second)));
1031 tmp = padd(ResPacketType(c.first), tmp);
1032 }
else EIGEN_IF_CONSTEXPR ((!ConjLhs) && (ConjRhs)) {
1033 tmp = pconj(pcplxflip(ResPacketType(c.second)));
1034 tmp = padd(ResPacketType(c.first), tmp);
1035 }
else EIGEN_IF_CONSTEXPR ((ConjLhs) && (!ConjRhs)) {
1036 tmp = pcplxflip(ResPacketType(c.second));
1037 tmp = padd(pconj(ResPacketType(c.first)), tmp);
1039 tmp = pcplxflip(ResPacketType(c.second));
1040 tmp = psub(pconj(ResPacketType(c.first)), tmp);
1043 r = pmadd(tmp, alpha, r);
1047 conj_helper<LhsScalar, RhsScalar, ConjLhs, ConjRhs> cj;
1050template <
typename RealScalar,
bool ConjRhs_,
int Arch,
int PacketSize_>
1051class gebp_traits<RealScalar, std::complex<RealScalar>, false, ConjRhs_, Arch, PacketSize_> {
1053 using Scalar = std::complex<RealScalar>;
1054 using LhsScalar = RealScalar;
1055 using RhsScalar = Scalar;
1056 using ResScalar = Scalar;
1058 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
1059 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
1060 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
1061 PACKET_DECL_COND_POSTFIX(_, Real, PacketSize_);
1062 PACKET_DECL_COND_SCALAR_POSTFIX(_, PacketSize_);
1064#undef PACKET_DECL_COND_SCALAR_POSTFIX
1065#undef PACKET_DECL_COND_POSTFIX
1066#undef PACKET_DECL_COND_SCALAR
1067#undef PACKET_DECL_COND
1072 Vectorizable = unpacket_traits<RealPacket_>::vectorizable && unpacket_traits<ScalarPacket_>::vectorizable,
1073 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
1074 RhsPacketSize = Vectorizable ? unpacket_traits<RhsPacket_>::size : 1,
1075 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
1077 NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS,
1080 mr = (plain_enum_min(16, NumberOfRegisters) / 2 / nr) * ResPacketSize,
1082 LhsProgress = ResPacketSize,
1086 using LhsPacket = std::conditional_t<Vectorizable, LhsPacket_, LhsScalar>;
1087 using RhsPacket = std::conditional_t<Vectorizable, RhsPacket_, RhsScalar>;
1088 using ResPacket = std::conditional_t<Vectorizable, ResPacket_, ResScalar>;
1089 using LhsPacket4Packing = LhsPacket;
1090 using RhsPacketx4 = QuadPacket<RhsPacket>;
1091 using AccPacket = ResPacket;
1093 EIGEN_STRONG_INLINE
void initAcc(AccPacket& p)
const { p = pset1<ResPacket>(ResScalar(0)); }
1095 template <
typename RhsPacketType>
1096 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketType& dest)
const {
1097 dest = pset1<RhsPacketType>(*b);
1100 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
1101 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
1104 template <
typename RhsPacketType>
1105 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, RhsPacketType& dest)
const {
1109 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
1111 EIGEN_STRONG_INLINE
void loadLhs(
const LhsScalar* a, LhsPacket& dest)
const { dest = ploaddup<LhsPacket>(a); }
1113 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, RhsPacket& dest)
const { dest = ploadquad<RhsPacket>(b); }
1115 template <
typename LhsPacketType>
1116 EIGEN_STRONG_INLINE
void loadLhsUnaligned(
const LhsScalar* a, LhsPacketType& dest)
const {
1117 dest = ploaddup<LhsPacketType>(a);
1120 template <
typename LhsPacketType,
typename RhsPacketType,
typename AccPacketType,
typename LaneIdType>
1121 EIGEN_STRONG_INLINE
void madd(
const LhsPacketType& a,
const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp,
1122 const LaneIdType&)
const {
1123 madd_impl(a, b, c, tmp, bool_constant<Vectorizable>());
1126 template <
typename LhsPacketType,
typename RhsPacketType,
typename AccPacketType>
1127 EIGEN_STRONG_INLINE
void madd_impl(
const LhsPacketType& a,
const RhsPacketType& b, AccPacketType& c,
1128 RhsPacketType& tmp,
const std::true_type&)
const {
1129#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
1130 EIGEN_UNUSED_VARIABLE(tmp);
1132 conj_helper<LhsPacketType, RhsPacketType, false, false> cj;
1133 c = cj.pmadd(a, b, c);
1136 tmp.v = pmul(a, tmp.v);
1141 EIGEN_STRONG_INLINE
void madd_impl(
const LhsScalar& a,
const RhsScalar& b, ResScalar& c, RhsScalar& ,
1142 const std::false_type&)
const {
1146 template <
typename LhsPacketType,
typename AccPacketType,
typename LaneIdType>
1147 EIGEN_STRONG_INLINE
void madd(
const LhsPacketType& a,
const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
1148 const LaneIdType& lane)
const {
1149 madd(a, b.get(lane), c, tmp, lane);
1152 template <
typename ResPacketType,
typename AccPacketType>
1153 EIGEN_STRONG_INLINE
void acc(
const AccPacketType& c,
const ResPacketType& alpha, ResPacketType& r)
const {
1154 conj_helper<ResPacketType, ResPacketType, false, ConjRhs> cj;
1155 r = cj.pmadd(alpha, c, r);
1168template <
typename Scalar>
1169struct gebp_overwritable_scalar : bool_constant<std::is_arithmetic<Scalar>::value> {};
1170template <
typename Real>
1171struct gebp_overwritable_scalar<std::complex<Real>> : bool_constant<std::is_arithmetic<Real>::value> {};
1173template <
typename LhsScalar,
typename RhsScalar,
typename Index,
typename DataMapper,
int mr,
int nr,
1174 bool ConjugateLhs,
bool ConjugateRhs>
1176 using Traits = gebp_traits<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs, Architecture::Target>;
1178 gebp_traits<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs, Architecture::Target, GEBPPacketHalf>;
1179 using QuarterTraits =
1180 gebp_traits<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs, Architecture::Target, GEBPPacketQuarter>;
1182 using ResScalar =
typename Traits::ResScalar;
1183 using LhsPacket =
typename Traits::LhsPacket;
1184 using RhsPacket =
typename Traits::RhsPacket;
1185 using ResPacket =
typename Traits::ResPacket;
1186 using AccPacket =
typename Traits::AccPacket;
1187 using RhsPacketx4 =
typename Traits::RhsPacketx4;
1189 using SwappedTraits = gebp_traits<RhsScalar, LhsScalar, ConjugateRhs, ConjugateLhs, Architecture::Target>;
1191 using SLhsPacket =
typename SwappedTraits::LhsPacket;
1192 using SRhsPacket =
typename SwappedTraits::RhsPacket;
1193 using SResPacket =
typename SwappedTraits::ResPacket;
1194 using SAccPacket =
typename SwappedTraits::AccPacket;
1196 using LhsPacketHalf =
typename HalfTraits::LhsPacket;
1197 using RhsPacketHalf =
typename HalfTraits::RhsPacket;
1198 using ResPacketHalf =
typename HalfTraits::ResPacket;
1199 using AccPacketHalf =
typename HalfTraits::AccPacket;
1201 using LhsPacketQuarter =
typename QuarterTraits::LhsPacket;
1202 using RhsPacketQuarter =
typename QuarterTraits::RhsPacket;
1203 using ResPacketQuarter =
typename QuarterTraits::ResPacket;
1204 using AccPacketQuarter =
typename QuarterTraits::AccPacket;
1206 using LinearMapper =
typename DataMapper::LinearMapper;
1209 Vectorizable = Traits::Vectorizable,
1210 LhsProgress = Traits::LhsProgress,
1211 LhsProgressHalf = HalfTraits::LhsProgress,
1212 LhsProgressQuarter = QuarterTraits::LhsProgress,
1213 RhsProgress = Traits::RhsProgress,
1214 RhsProgressHalf = HalfTraits::RhsProgress,
1215 RhsProgressQuarter = QuarterTraits::RhsProgress,
1216 ResPacketSize = Traits::ResPacketSize
1221 static constexpr bool CanOverwrite = gebp_overwritable_scalar<ResScalar>::value;
1223 EIGEN_STRONG_INLINE
void operator()(
const DataMapper& res,
const LhsScalar* blockA,
const RhsScalar* blockB,
1224 Index rows, Index depth, Index cols, ResScalar alpha, Index strideA = -1,
1225 Index strideB = -1, Index offsetA = 0, Index offsetB = 0)
const {
1226 run(
false, res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
1229 EIGEN_STRONG_INLINE
void operator()(gebp_overwrite_t,
const DataMapper& res,
const LhsScalar* blockA,
1230 const RhsScalar* blockB, Index rows, Index depth, Index cols, ResScalar alpha,
1231 Index strideA = -1, Index strideB = -1, Index offsetA = 0,
1232 Index offsetB = 0)
const {
1233 run(
true, res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
1238 EIGEN_DONT_INLINE EIGEN_GEMM_NO_CLONE
void run(
bool overwrite,
const DataMapper& res,
const LhsScalar* blockA,
1239 const RhsScalar* blockB, Index rows, Index depth, Index cols,
1240 ResScalar alpha, Index strideA, Index strideB, Index offsetA,
1241 Index offsetB)
const;
1247template <
typename ResScalar,
typename Alpha,
typename Acc>
1248EIGEN_ALWAYS_INLINE
void gebp_store_scalar_impl(std::false_type,
bool, ResScalar& dst,
const Alpha& alpha,
1252template <
typename ResScalar,
typename Alpha,
typename Acc>
1253EIGEN_ALWAYS_INLINE
void gebp_store_scalar_impl(std::true_type,
bool overwrite, ResScalar& dst,
const Alpha& alpha,
1256 dst = ResScalar(0) + alpha * c;
1260template <
bool CanOverwrite,
typename ResScalar,
typename Alpha,
typename Acc>
1261EIGEN_ALWAYS_INLINE
void gebp_store_scalar(
bool overwrite, ResScalar& dst,
const Alpha& alpha,
const Acc& c) {
1262 gebp_store_scalar_impl(bool_constant<CanOverwrite>(), overwrite, dst, alpha, c);
1265template <
typename LhsScalar,
typename RhsScalar,
typename Index,
typename DataMapper,
int mr,
int nr,
1266 bool ConjugateLhs,
bool ConjugateRhs,
1267 int SwappedLhsProgress =
1268 gebp_traits<RhsScalar, LhsScalar, ConjugateRhs, ConjugateLhs, Architecture::Target>::LhsProgress>
1269struct last_row_process_16_packets {
1270 using Traits = gebp_traits<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs, Architecture::Target>;
1271 using SwappedTraits = gebp_traits<RhsScalar, LhsScalar, ConjugateRhs, ConjugateLhs, Architecture::Target>;
1273 using ResScalar =
typename Traits::ResScalar;
1274 using SLhsPacket =
typename SwappedTraits::LhsPacket;
1275 using SRhsPacket =
typename SwappedTraits::RhsPacket;
1276 using SResPacket =
typename SwappedTraits::ResPacket;
1277 using SAccPacket =
typename SwappedTraits::AccPacket;
1279 EIGEN_STRONG_INLINE
void operator()(
const DataMapper& res, SwappedTraits& straits,
const LhsScalar* blA,
1280 const RhsScalar* blB, Index depth,
const Index endk, Index i, Index j2,
1281 ResScalar alpha, SAccPacket& C0,
bool overwrite =
false)
const {
1282 EIGEN_UNUSED_VARIABLE(overwrite);
1283 EIGEN_UNUSED_VARIABLE(res);
1284 EIGEN_UNUSED_VARIABLE(straits);
1285 EIGEN_UNUSED_VARIABLE(blA);
1286 EIGEN_UNUSED_VARIABLE(blB);
1287 EIGEN_UNUSED_VARIABLE(depth);
1288 EIGEN_UNUSED_VARIABLE(endk);
1289 EIGEN_UNUSED_VARIABLE(i);
1290 EIGEN_UNUSED_VARIABLE(j2);
1291 EIGEN_UNUSED_VARIABLE(alpha);
1292 EIGEN_UNUSED_VARIABLE(C0);
1296template <
typename LhsScalar,
typename RhsScalar,
typename Index,
typename DataMapper,
int mr,
int nr,
1297 bool ConjugateLhs,
bool ConjugateRhs>
1298struct last_row_process_16_packets<LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs, 16> {
1299 using Traits = gebp_traits<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs, Architecture::Target>;
1300 using SwappedTraits = gebp_traits<RhsScalar, LhsScalar, ConjugateRhs, ConjugateLhs, Architecture::Target>;
1302 using ResScalar =
typename Traits::ResScalar;
1303 using SLhsPacket =
typename SwappedTraits::LhsPacket;
1304 using SRhsPacket =
typename SwappedTraits::RhsPacket;
1305 using SResPacket =
typename SwappedTraits::ResPacket;
1306 using SAccPacket =
typename SwappedTraits::AccPacket;
1308 EIGEN_STRONG_INLINE
void operator()(
const DataMapper& res, SwappedTraits& straits,
const LhsScalar* blA,
1309 const RhsScalar* blB, Index depth,
const Index endk, Index i, Index j2,
1310 ResScalar alpha, SAccPacket& C0,
bool overwrite =
false)
const {
1311 using SResPacketQuarter =
typename unpacket_traits<typename unpacket_traits<SResPacket>::half>::half;
1312 using SLhsPacketQuarter =
typename unpacket_traits<typename unpacket_traits<SLhsPacket>::half>::half;
1313 using SRhsPacketQuarter =
typename unpacket_traits<typename unpacket_traits<SRhsPacket>::half>::half;
1314 using SAccPacketQuarter =
typename unpacket_traits<typename unpacket_traits<SAccPacket>::half>::half;
1316 SResPacketQuarter R =
1317 overwrite ? pset1<SResPacketQuarter>(ResScalar(0)) : res.template gatherPacket<SResPacketQuarter>(i, j2);
1318 SResPacketQuarter alphav = pset1<SResPacketQuarter>(alpha);
1320 if (depth - endk > 0) {
1323 SAccPacketQuarter c0 = predux_half(predux_half(C0));
1325 for (Index kk = endk; kk < depth; kk++) {
1326 SLhsPacketQuarter a0;
1327 SRhsPacketQuarter b0;
1328 straits.loadLhsUnaligned(blB, a0);
1329 straits.loadRhs(blA, b0);
1330 straits.madd(a0, b0, c0, b0, fix<0>);
1331 blB += SwappedTraits::LhsProgress / 4;
1334 straits.acc(c0, alphav, R);
1336 straits.acc(predux_half(predux_half(C0)), alphav, R);
1338 res.scatterPacket(i, j2, R);
1345template <
int J,
int MrPackets,
int NrCols,
bool Continue = (J < NrCols)>
1346struct gebp_rhs_cols;
1349template <
int J,
int MrPackets,
int NrCols>
1350struct gebp_rhs_cols<J, MrPackets, NrCols, false> {
1351 template <
typename GEBPTraits,
typename LhsArray,
typename RhsPanelType,
typename RhsPacketType,
typename AccArray,
1353 static EIGEN_ALWAYS_INLINE
void run(GEBPTraits&,
const RhsScalar*, Index, LhsArray&, RhsPanelType&, RhsPacketType&,
1358template <
int J,
int MrPackets,
int NrCols>
1359struct gebp_rhs_cols<J, MrPackets, NrCols, true> {
1360 template <
typename GEBPTraits,
typename LhsArray,
typename RhsPanelType,
typename RhsPacketType,
typename AccArray,
1362 static EIGEN_ALWAYS_INLINE
void run(GEBPTraits& traits,
const RhsScalar* blB, Index rhs_offset, LhsArray& A,
1363 RhsPanelType& rhs_panel, RhsPacketType& T0, AccArray& C) {
1364 constexpr int lane = J % 4;
1365 EIGEN_IF_CONSTEXPR (lane == 0)
1366 traits.loadRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel);
1368 traits.updateRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel);
1370 EIGEN_IF_CONSTEXPR (MrPackets >= 1) traits.madd(A[0], rhs_panel, C[J + 0 * NrCols], T0, fix<lane>);
1371 EIGEN_IF_CONSTEXPR (MrPackets >= 2) traits.madd(A[1], rhs_panel, C[J + 1 * NrCols], T0, fix<lane>);
1372 EIGEN_IF_CONSTEXPR (MrPackets >= 3) traits.madd(A[2], rhs_panel, C[J + 2 * NrCols], T0, fix<lane>);
1374 gebp_rhs_cols<J + 1, MrPackets, NrCols>::run(traits, blB, rhs_offset, A, rhs_panel, T0, C);
1382#if EIGEN_ARCH_ARM64 && defined(EIGEN_VECTORIZE_NEON) && EIGEN_GNUC_STRICT_AT_LEAST(9, 0, 0)
1383#define EIGEN_GEBP_NEON_GCC_WORKAROUNDS
1384#define EIGEN_GEBP_UNROLL_ACCUMULATORS _Pragma("GCC unroll 24")
1386#define EIGEN_GEBP_UNROLL_ACCUMULATORS
1391template <
int K,
int MrPackets,
int NrCols>
1392struct gebp_micro_step {
1393 template <
typename GEBPTraits,
typename LhsScalar_,
typename RhsScalar_,
typename LhsArray,
typename RhsPanelType,
1394 typename RhsPacketType,
typename AccArray>
1395 static EIGEN_ALWAYS_INLINE
void run(GEBPTraits& traits,
const LhsScalar_* blA,
const RhsScalar_* blB, LhsArray& A,
1396 RhsPanelType& rhs_panel, RhsPacketType& T0, AccArray& C) {
1397 constexpr int LhsProg = GEBPTraits::LhsProgress;
1399 EIGEN_IF_CONSTEXPR (MrPackets >= 1) traits.loadLhs(&blA[(0 + MrPackets * K) * LhsProg], A[0]);
1400 EIGEN_IF_CONSTEXPR (MrPackets >= 2) traits.loadLhs(&blA[(1 + MrPackets * K) * LhsProg], A[1]);
1401 EIGEN_IF_CONSTEXPR (MrPackets >= 3) traits.loadLhs(&blA[(2 + MrPackets * K) * LhsProg], A[2]);
1403 gebp_rhs_cols<0, MrPackets, NrCols>::run(traits, blB, Index(NrCols * K), A, rhs_panel, T0, C);
1404#if defined(EIGEN_GEBP_NEON_GCC_WORKAROUNDS) && !defined(EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING)
1405 using AccPacket =
typename GEBPTraits::AccPacket;
1406 constexpr bool kRealPacket = std::is_same<AccPacket, Packet4f>::value || std::is_same<AccPacket, Packet2d>::value;
1407 EIGEN_IF_CONSTEXPR (MrPackets == 3 && NrCols == 8 && kRealPacket) {
1409 __asm__
volatile(
"" :::
"memory");
1410#if EIGEN_GNUC_STRICT_AT_LEAST(16, 0, 0)
1412 EIGEN_IF_CONSTEXPR (std::is_same<AccPacket, Packet2d>::value) {
1413 EIGEN_GEBP_UNROLL_ACCUMULATORS
1414 for (
int n = 0; n < MrPackets * NrCols; ++n) {
1415 EIGEN_OPTIMIZATION_BARRIER(C[n]);
1431template <
int MrPackets,
typename GEBPTraits_,
typename FullLhsPacket_,
typename LhsArray_>
1432EIGEN_ALWAYS_INLINE
void gebp_neon_3p_workaround(LhsArray_& A) {
1433#if EIGEN_ARCH_ARM64 && defined(EIGEN_VECTORIZE_NEON) && EIGEN_GNUC_STRICT_LESS_THAN(9, 0, 0)
1434 using LhsElement = std::remove_all_extents_t<std::remove_reference_t<LhsArray_>>;
1435 constexpr bool apply = GEBPTraits_::Vectorizable && MrPackets == 3 && std::is_same<LhsElement, FullLhsPacket_>::value;
1436 EIGEN_IF_CONSTEXPR (apply) {
1437 __asm__(
"" :
"+w,m"(A[0]),
"+w,m"(A[1]),
"+w,m"(A[2]));
1440 EIGEN_UNUSED_VARIABLE(A);
1447template <
int MrPackets,
int NrCols,
typename GEBPTraits_,
typename FullLhsPacket_,
typename LhsArray_,
1449EIGEN_ALWAYS_INLINE
void gebp_sse_spilling_workaround(LhsArray_& A, AccArray_& ACC) {
1450 EIGEN_UNUSED_VARIABLE(A);
1451 EIGEN_UNUSED_VARIABLE(ACC);
1452#if EIGEN_GNUC_STRICT_AT_LEAST(6, 0, 0) && defined(EIGEN_VECTORIZE_SSE)
1453 using LhsElement = std::remove_all_extents_t<std::remove_reference_t<LhsArray_>>;
1454 constexpr bool apply =
1455 GEBPTraits_::Vectorizable && MrPackets <= 2 && NrCols >= 4 && std::is_same<LhsElement, FullLhsPacket_>::value;
1456 EIGEN_IF_CONSTEXPR (apply) {
1457#ifdef EIGEN_HAS_CXX17_IFCONSTEXPR
1458 using AccElement = std::decay_t<
decltype(ACC[0])>;
1459 constexpr bool pin_acc = std::is_same<AccElement, FullLhsPacket_>::value && MrPackets == 2 && NrCols == 4;
1460 if constexpr (pin_acc) {
1462 :
"+x"(ACC[0]),
"+x"(ACC[1]),
"+x"(ACC[2]),
"+x"(ACC[3]),
"+x"(ACC[4]),
"+x"(ACC[5]),
"+x"(ACC[6]),
1466 EIGEN_IF_CONSTEXPR (MrPackets == 2) {
1467 __asm__(
"" :
"+x,m"(A[0]),
"+x,m"(A[1]));
1480#if EIGEN_COMP_CLANG && EIGEN_ARCH_x86_64 && defined(EIGEN_VECTORIZE_AVX) && !defined(EIGEN_VECTORIZE_AVX512)
1481#define EIGEN_GEBP_AVX_CLANG_PIN_ACCUMULATORS
1482template <
typename Packet>
1483EIGEN_ALWAYS_INLINE
void gebp_avx_clang_pin_accumulator(Packet& x) {
1486EIGEN_ALWAYS_INLINE
void gebp_avx_clang_pin_accumulator(Packet4cf& x) { pbarrier(x.v); }
1487EIGEN_ALWAYS_INLINE
void gebp_avx_clang_pin_accumulator(Packet2cd& x) { pbarrier(x.v); }
1489template <
int MrPackets,
int NrCols,
typename GEBPTraits_,
typename FullLhsPacket_,
typename AccArray_>
1490EIGEN_ALWAYS_INLINE
void gebp_avx_clang_spilling_workaround(AccArray_& ACC) {
1491 EIGEN_UNUSED_VARIABLE(ACC);
1492#ifdef EIGEN_GEBP_AVX_CLANG_PIN_ACCUMULATORS
1493 using AccElement = std::decay_t<
decltype(ACC[0])>;
1494 constexpr bool kFloatingPoint =
1495 std::is_same<AccElement, Packet8f>::value || std::is_same<AccElement, Packet4d>::value ||
1496 std::is_same<AccElement, Packet4cf>::value || std::is_same<AccElement, Packet2cd>::value;
1497 constexpr bool apply = GEBPTraits_::Vectorizable && MrPackets == 3 && NrCols == 4 && kFloatingPoint &&
1498 std::is_same<AccElement, FullLhsPacket_>::value;
1499 EIGEN_IF_CONSTEXPR (apply) {
1500 for (
int n = 0; n < MrPackets * NrCols; ++n) gebp_avx_clang_pin_accumulator(ACC[n]);
1507template <
typename AccPacket,
int MrPackets,
int NrCols>
1508struct gebp_use_double_accum {
1509 static constexpr bool value = MrPackets == 1 && NrCols == 4
1510#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
1511 && !(is_double_packet<AccPacket>::value && EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS < 32)
1518template <
int MrPackets,
int NrCols>
1519struct gebp_peeled_loop {
1520 template <
typename GEBPTraits,
typename LhsScalar_,
typename RhsScalar_,
typename LhsArray,
typename RhsPanelType,
1521 typename RhsPacketType,
typename AccArray,
typename AccArrayD,
typename FullLhsPacket>
1522 static EIGEN_ALWAYS_INLINE
void run(GEBPTraits& traits,
const LhsScalar_* blA,
const RhsScalar_* blB, LhsArray& A,
1523 RhsPanelType& rhs_panel, RhsPacketType& T0, AccArray& C, AccArrayD& D) {
1524 constexpr bool use_double_accum = gebp_use_double_accum<typename GEBPTraits::AccPacket, MrPackets, NrCols>::value;
1527 EIGEN_IF_CONSTEXPR (NrCols == 4) {
1528 internal::prefetch(blB + (48 + 0));
1532#define EIGEN_GEBP_DO_STEP(KVAL, ACC) \
1534 gebp_micro_step<KVAL, MrPackets, NrCols>::run(traits, blA, blB, A, rhs_panel, T0, ACC); \
1535 gebp_neon_3p_workaround<MrPackets, GEBPTraits, FullLhsPacket>(A); \
1536 gebp_sse_spilling_workaround<MrPackets, NrCols, GEBPTraits, FullLhsPacket>(A, ACC); \
1537 gebp_avx_clang_spilling_workaround<MrPackets, NrCols, GEBPTraits, FullLhsPacket>(ACC); \
1539 EIGEN_IF_CONSTEXPR ((MrPackets == 2 || MrPackets == 3) && NrCols == 4) { \
1540 internal::prefetch(blA + (MrPackets * KVAL + 16) * GEBPTraits::LhsProgress); \
1541 if (EIGEN_ARCH_ARM || EIGEN_ARCH_MIPS) { \
1542 internal::prefetch(blB + (NrCols * KVAL + 16) * GEBPTraits::RhsProgress); \
1547 EIGEN_IF_CONSTEXPR (use_double_accum) {
1548 EIGEN_GEBP_DO_STEP(0, C);
1549 EIGEN_GEBP_DO_STEP(1, D);
1550 EIGEN_GEBP_DO_STEP(2, C);
1551 EIGEN_GEBP_DO_STEP(3, D);
1552 EIGEN_IF_CONSTEXPR (NrCols == 4) {
1553 internal::prefetch(blB + (48 + 16));
1555 EIGEN_GEBP_DO_STEP(4, C);
1556 EIGEN_GEBP_DO_STEP(5, D);
1557 EIGEN_GEBP_DO_STEP(6, C);
1558 EIGEN_GEBP_DO_STEP(7, D);
1560 EIGEN_GEBP_DO_STEP(0, C);
1561 EIGEN_GEBP_DO_STEP(1, C);
1562 EIGEN_GEBP_DO_STEP(2, C);
1563 EIGEN_GEBP_DO_STEP(3, C);
1564 EIGEN_IF_CONSTEXPR (NrCols == 4 && MrPackets == 2) {
1565 internal::prefetch(blB + (48 + 16));
1567 EIGEN_GEBP_DO_STEP(4, C);
1568 EIGEN_GEBP_DO_STEP(5, C);
1569 EIGEN_GEBP_DO_STEP(6, C);
1570 EIGEN_GEBP_DO_STEP(7, C);
1573#undef EIGEN_GEBP_DO_STEP
1580template <
int MrPackets,
int NrCols,
typename GEBPTraits,
typename LhsScalar_,
typename RhsScalar_,
typename ResScalar_,
1581 typename Index_,
typename DataMapper_,
typename LinearMapper_,
typename FullLhsPacket>
1582EIGEN_ALWAYS_INLINE
void gebp_micro_panel_impl(GEBPTraits& traits,
const DataMapper_& res,
const LhsScalar_* blockA,
1583 const RhsScalar_* blockB, ResScalar_ alpha, Index_ i, Index_ j2,
1584 Index_ depth, Index_ strideA, Index_ strideB, Index_ offsetA,
1585 Index_ offsetB,
int prefetch_res_offset, Index_ peeled_kc,
int pk,
1587 using LhsPacketLocal =
typename GEBPTraits::LhsPacket;
1588 using RhsPacketLocal =
typename GEBPTraits::RhsPacket;
1589 using ResPacketLocal =
typename GEBPTraits::ResPacket;
1590 using AccPacketLocal =
typename GEBPTraits::AccPacket;
1591 using RhsPacketx4Local =
typename GEBPTraits::RhsPacketx4;
1592 constexpr int LhsProg = GEBPTraits::LhsProgress;
1593 constexpr int RhsProg = GEBPTraits::RhsProgress;
1594 constexpr int ResPacketSz = GEBPTraits::ResPacketSize;
1597 constexpr bool use_double_accum = gebp_use_double_accum<AccPacketLocal, MrPackets, NrCols>::value;
1598 constexpr int AccRegisters = is_double_packet<AccPacketLocal>::value ? 2 : 1;
1599 constexpr int RhsRegisters = is_double_packet<RhsPacketLocal>::value ? 2 : 1;
1600 constexpr int RegistersTaken = AccRegisters * (MrPackets + use_double_accum) * NrCols + MrPackets;
1601 using RhsPanelType =
1602 std::conditional_t<NrCols == 1, RhsPacketLocal,
1603 typename RhsPanelHelper<RhsPacketLocal, RhsPacketx4Local, RegistersTaken, RhsRegisters>::type>;
1605 const LhsScalar_* blA = &blockA[i * strideA + offsetA * (MrPackets * LhsProg)];
1611#ifdef EIGEN_HAS_CXX17_IFCONSTEXPR
1612 constexpr int CSize = MrPackets * NrCols;
1614 constexpr int CSize = 3 * NrCols > MrPackets * NrCols ? 3 * NrCols : MrPackets * NrCols;
1616 alignas(AccPacketLocal) AccPacketLocal C[CSize];
1617 EIGEN_GEBP_UNROLL_ACCUMULATORS
1618 for (
int n = 0; n < MrPackets * NrCols; ++n) traits.initAcc(C[n]);
1620#ifdef EIGEN_HAS_CXX17_IFCONSTEXPR
1621 alignas(AccPacketLocal) AccPacketLocal D[use_double_accum ? NrCols : 1];
1625 alignas(AccPacketLocal) AccPacketLocal D[CSize];
1627 EIGEN_IF_CONSTEXPR (use_double_accum) {
1628 for (
int n = 0; n < NrCols; ++n) traits.initAcc(D[n]);
1632 for (
int j = 0; j < NrCols; ++j) res.getLinearMapper(i, j2 + j).prefetch(NrCols > 1 ? prefetch_res_offset : 0);
1635 const RhsScalar_* blB = &blockB[j2 * strideB + offsetB * NrCols];
1639#ifdef EIGEN_HAS_CXX17_IFCONSTEXPR
1640 alignas(LhsPacketLocal) LhsPacketLocal A[MrPackets];
1642 alignas(LhsPacketLocal) LhsPacketLocal A[3];
1646#if defined(EIGEN_VECTORIZE_RVV10) && EIGEN_GNUC_STRICT_AT_LEAST(15, 0, 0) && EIGEN_GNUC_STRICT_LESS_THAN(17, 0, 0)
1650 for (Index_ k = 0; k < peeled_kc; k += pk) {
1651 alignas(RhsPanelType) RhsPanelType rhs_panel;
1652 alignas(RhsPacketLocal) RhsPacketLocal T0;
1654 gebp_peeled_loop<MrPackets, NrCols>::template run<GEBPTraits, LhsScalar_, RhsScalar_,
decltype(A), RhsPanelType,
1655 RhsPacketLocal,
decltype(C),
decltype(D), FullLhsPacket>(
1656 traits, blA, blB, A, rhs_panel, T0, C, D);
1658 blB += pk * NrCols * RhsProg;
1659 blA += pk * MrPackets * LhsProg;
1663 EIGEN_IF_CONSTEXPR (use_double_accum) {
1664 EIGEN_GEBP_UNROLL_ACCUMULATORS
1665 for (
int n = 0; n < NrCols; ++n) C[n] = padd(C[n], D[n]);
1669 for (Index_ k = peeled_kc; k < depth; k++) {
1670 alignas(RhsPanelType) RhsPanelType rhs_panel;
1671 alignas(RhsPacketLocal) RhsPacketLocal T0;
1673 gebp_micro_step<0, MrPackets, NrCols>::run(traits, blA, blB, A, rhs_panel, T0, C);
1675 blB += NrCols * RhsProg;
1676 blA += MrPackets * LhsProg;
1680 alignas(ResPacketLocal) ResPacketLocal alphav = pset1<ResPacketLocal>(alpha);
1681 EIGEN_GEBP_UNROLL_ACCUMULATORS
1682 for (
int j = 0; j < NrCols; ++j) {
1683 LinearMapper_ r = res.getLinearMapper(i, j2 + j);
1684 EIGEN_GEBP_UNROLL_ACCUMULATORS
1685 for (
int p = 0; p < MrPackets; ++p) {
1686 alignas(ResPacketLocal) ResPacketLocal R =
1687 overwrite ? pset1<ResPacketLocal>(ResScalar_(0)) : r.template loadPacket<ResPacketLocal>(p * ResPacketSz);
1688 traits.acc(C[j + p * NrCols], alphav, R);
1689 r.storePacket(p * ResPacketSz, R);
1693#undef EIGEN_GEBP_UNROLL_ACCUMULATORS
1694#undef EIGEN_GEBP_NEON_GCC_WORKAROUNDS
1695#undef EIGEN_GEBP_AVX_CLANG_PIN_ACCUMULATORS
1705#if EIGEN_COMP_GNUC_STRICT && EIGEN_ARCH_ARM64 && !defined(EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING)
1706#pragma GCC push_options
1707#pragma GCC optimize("no-schedule-insns")
1708#define EIGEN_GEBP_DISABLED_INSN_SCHEDULING
1710template <
typename LhsScalar,
typename RhsScalar,
typename Index,
typename DataMapper,
int mr,
int nr,
1711 bool ConjugateLhs,
bool ConjugateRhs>
1712EIGEN_DONT_INLINE EIGEN_GEMM_NO_CLONE
void
1713gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::run(
1714 bool overwrite_,
const DataMapper& res,
const LhsScalar* blockA,
const RhsScalar* blockB, Index rows, Index depth,
1715 Index cols, ResScalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
const {
1717 const bool overwrite = CanOverwrite && overwrite_;
1719 SwappedTraits straits;
1721 if (strideA == -1) strideA = depth;
1722 if (strideB == -1) strideB = depth;
1723 conj_helper<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs> cj;
1724 Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0;
1725 Index packet_cols8 = nr >= 8 ? (cols / 8) * 8 : 0;
1726 const Index peeled_mc3 = mr >= 3 * Traits::LhsProgress ? (rows / (3 * LhsProgress)) * (3 * LhsProgress) : 0;
1727 const Index peeled_mc2 =
1728 mr >= 2 * Traits::LhsProgress ? peeled_mc3 + ((rows - peeled_mc3) / (2 * LhsProgress)) * (2 * LhsProgress) : 0;
1729 const Index peeled_mc1 =
1730 mr >= 1 * Traits::LhsProgress ? peeled_mc2 + ((rows - peeled_mc2) / (1 * LhsProgress)) * (1 * LhsProgress) : 0;
1731 const Index peeled_mc_half =
1732 mr >= LhsProgressHalf ? peeled_mc1 + ((rows - peeled_mc1) / (LhsProgressHalf)) * (LhsProgressHalf) : 0;
1733 const Index peeled_mc_quarter =
1734 mr >= LhsProgressQuarter
1735 ? peeled_mc_half + ((rows - peeled_mc_half) / (LhsProgressQuarter)) * (LhsProgressQuarter)
1738 const Index peeled_kc = depth & ~(pk - 1);
1739 const int prefetch_res_offset = 32 /
sizeof(ResScalar);
1746 auto micro_panel = [&](
auto mrp_tag,
auto nrc_tag,
auto& local_traits, Index i, Index j2) EIGEN_LAMBDA_ALWAYS_INLINE {
1747 constexpr int MrP =
decltype(mrp_tag)::value;
1748 constexpr int NrC =
decltype(nrc_tag)::value;
1749 using LTraits = std::remove_reference_t<
decltype(local_traits)>;
1750 gebp_micro_panel_impl<MrP, NrC, LTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1751 LhsPacket>(local_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB, offsetA,
1752 offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1765 std::ptrdiff_t l1, l2, l3;
1766 manage_caching_sizes(GetAction, &l1, &l2, &l3);
1767#if EIGEN_ARCH_i386_OR_x86_64
1768 lhs_budget =
static_cast<Index
>(l2 / 2);
1770 lhs_budget =
static_cast<Index
>(l1);
1775 EIGEN_IF_CONSTEXPR (mr >= 3 * Traits::LhsProgress) {
1776 const Index rhs_block =
sizeof(ResScalar) * mr * nr + depth * nr *
sizeof(RhsScalar);
1777 const Index lhs_strip = depth *
sizeof(LhsScalar) * 3 * LhsProgress;
1778 const Index lhs_avail = (lhs_budget > rhs_block) ? (lhs_budget - rhs_block) : 0;
1779 const Index actual_panel_rows = (lhs_avail >= peeled_mc3 * depth *
static_cast<Index
>(
sizeof(LhsScalar)))
1781 : (3 * LhsProgress) * std::max<Index>(1, lhs_avail / lhs_strip);
1782 for (Index i1 = 0; i1 < peeled_mc3; i1 += actual_panel_rows) {
1783 const Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc3);
1784 EIGEN_IF_CONSTEXPR (nr >= 8) {
1785 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1786 for (Index i = i1; i < actual_panel_end; i += 3 * LhsProgress) {
1787 micro_panel(fix<3>, fix<8>, traits, i, j2);
1791 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1792 for (Index i = i1; i < actual_panel_end; i += 3 * LhsProgress) {
1793 micro_panel(fix<3>, fix<4>, traits, i, j2);
1796 for (Index j2 = packet_cols4; j2 < cols; j2++) {
1797 for (Index i = i1; i < actual_panel_end; i += 3 * LhsProgress) {
1798 micro_panel(fix<3>, fix<1>, traits, i, j2);
1805 EIGEN_IF_CONSTEXPR (mr >= 2 * Traits::LhsProgress) {
1806 const Index rhs_block2 =
sizeof(ResScalar) * mr * nr + depth * nr *
sizeof(RhsScalar);
1807 const Index lhs_strip2 = depth *
sizeof(LhsScalar) * 2 * LhsProgress;
1808 const Index lhs_avail2 = (lhs_budget > rhs_block2) ? (lhs_budget - rhs_block2) : 0;
1809 const Index mc2_range = peeled_mc2 - peeled_mc3;
1810 Index actual_panel_rows = (lhs_avail2 >= mc2_range * depth *
static_cast<Index
>(
sizeof(LhsScalar)))
1812 : (2 * LhsProgress) * std::max<Index>(1, lhs_avail2 / lhs_strip2);
1813 for (Index i1 = peeled_mc3; i1 < peeled_mc2; i1 += actual_panel_rows) {
1814 Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc2);
1815 EIGEN_IF_CONSTEXPR (nr >= 8) {
1816 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1817 for (Index i = i1; i < actual_panel_end; i += 2 * LhsProgress) {
1818 micro_panel(fix<2>, fix<8>, traits, i, j2);
1822 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1823 for (Index i = i1; i < actual_panel_end; i += 2 * LhsProgress) {
1824 micro_panel(fix<2>, fix<4>, traits, i, j2);
1827 for (Index j2 = packet_cols4; j2 < cols; j2++) {
1828 for (Index i = i1; i < actual_panel_end; i += 2 * LhsProgress) {
1829 micro_panel(fix<2>, fix<1>, traits, i, j2);
1837 EIGEN_IF_CONSTEXPR (mr > 1 * Traits::LhsProgress) {
1838 for (Index i = peeled_mc2; i < peeled_mc1; i += LhsProgress) {
1839 EIGEN_IF_CONSTEXPR (nr >= 8) {
1840 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1841 micro_panel(fix<1>, fix<8>, traits, i, j2);
1844 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1845 micro_panel(fix<1>, fix<4>, traits, i, j2);
1847 for (Index j2 = packet_cols4; j2 < cols; j2++) {
1848 micro_panel(fix<1>, fix<1>, traits, i, j2);
1851 }
else EIGEN_IF_CONSTEXPR (mr == 1 * Traits::LhsProgress) {
1854 const Index rhs_block1 =
sizeof(ResScalar) * mr * nr + depth * nr *
sizeof(RhsScalar);
1855 const Index lhs_strip1 = depth *
sizeof(LhsScalar) * LhsProgress;
1856 const Index lhs_avail1 = (lhs_budget > rhs_block1) ? (lhs_budget - rhs_block1) : 0;
1857 const Index mc1_range = peeled_mc1 - peeled_mc2;
1858 const Index actual_panel_rows = (lhs_avail1 >= mc1_range * depth *
static_cast<Index
>(
sizeof(LhsScalar)))
1860 : LhsProgress * std::max<Index>(1, lhs_avail1 / lhs_strip1);
1861 for (Index i1 = peeled_mc2; i1 < peeled_mc1; i1 += actual_panel_rows) {
1862 const Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc1);
1863 EIGEN_IF_CONSTEXPR (nr >= 8) {
1864 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1865 for (Index i = i1; i < actual_panel_end; i += LhsProgress) {
1866 micro_panel(fix<1>, fix<8>, traits, i, j2);
1870 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1871 for (Index i = i1; i < actual_panel_end; i += LhsProgress) {
1872 micro_panel(fix<1>, fix<4>, traits, i, j2);
1875 for (Index j2 = packet_cols4; j2 < cols; j2++) {
1876 for (Index i = i1; i < actual_panel_end; i += LhsProgress) {
1877 micro_panel(fix<1>, fix<1>, traits, i, j2);
1884 EIGEN_IF_CONSTEXPR ((LhsProgressHalf < LhsProgress) && mr >= LhsProgressHalf) {
1885 HalfTraits half_traits;
1886 for (Index i = peeled_mc1; i < peeled_mc_half; i += LhsProgressHalf) {
1887 EIGEN_IF_CONSTEXPR (nr >= 8) {
1888 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1889 gebp_micro_panel_impl<1, 8, HalfTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1890 LhsPacket>(half_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
1891 offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1894 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1895 gebp_micro_panel_impl<1, 4, HalfTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1896 LhsPacket>(half_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
1897 offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1899 for (Index j2 = packet_cols4; j2 < cols; j2++) {
1900 gebp_micro_panel_impl<1, 1, HalfTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1901 LhsPacket>(half_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
1902 offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1908 EIGEN_IF_CONSTEXPR ((LhsProgressQuarter < LhsProgressHalf) && mr >= LhsProgressQuarter) {
1909 QuarterTraits quarter_traits;
1910 for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += LhsProgressQuarter) {
1911 EIGEN_IF_CONSTEXPR (nr >= 8) {
1912 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1913 gebp_micro_panel_impl<1, 8, QuarterTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1914 LhsPacket>(quarter_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
1915 offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1918 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1919 gebp_micro_panel_impl<1, 4, QuarterTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1920 LhsPacket>(quarter_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
1921 offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1923 for (Index j2 = packet_cols4; j2 < cols; j2++) {
1924 gebp_micro_panel_impl<1, 1, QuarterTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1925 LhsPacket>(quarter_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
1926 offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1932 if (peeled_mc_quarter < rows) {
1933 EIGEN_IF_CONSTEXPR (nr >= 8) {
1935 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1937 for (Index i = peeled_mc_quarter; i < rows; i += 1) {
1938 const LhsScalar* blA = &blockA[i * strideA + offsetA];
1941 ResScalar C0(0), C1(0), C2(0), C3(0), C4(0), C5(0), C6(0), C7(0);
1942 const RhsScalar* blB = &blockB[j2 * strideB + offsetB * 8];
1943 for (Index k = 0; k < depth; k++) {
1944 LhsScalar A0 = blA[k];
1948 C0 = cj.pmadd(A0, B_0, C0);
1951 C1 = cj.pmadd(A0, B_0, C1);
1954 C2 = cj.pmadd(A0, B_0, C2);
1957 C3 = cj.pmadd(A0, B_0, C3);
1960 C4 = cj.pmadd(A0, B_0, C4);
1963 C5 = cj.pmadd(A0, B_0, C5);
1966 C6 = cj.pmadd(A0, B_0, C6);
1969 C7 = cj.pmadd(A0, B_0, C7);
1973 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 0), alpha, C0);
1974 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 1), alpha, C1);
1975 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 2), alpha, C2);
1976 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 3), alpha, C3);
1977 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 4), alpha, C4);
1978 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 5), alpha, C5);
1979 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 6), alpha, C6);
1980 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 7), alpha, C7);
1985 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1987 for (Index i = peeled_mc_quarter; i < rows; i += 1) {
1988 const LhsScalar* blA = &blockA[i * strideA + offsetA];
1990 const RhsScalar* blB = &blockB[j2 * strideB + offsetB * 4];
1995 constexpr int SResPacketHalfSize = unpacket_traits<typename unpacket_traits<SResPacket>::half>::size;
1996 constexpr int SResPacketQuarterSize =
1997 unpacket_traits<typename unpacket_traits<typename unpacket_traits<SResPacket>::half>::half>::size;
2002 constexpr bool kCanLoadSRhsQuad =
2003 (unpacket_traits<SLhsPacket>::size < 4) ||
2004 (unpacket_traits<SRhsPacket>::size % ((std::max<int>)(unpacket_traits<SLhsPacket>::size, 4) / 4)) == 0;
2005 EIGEN_IF_CONSTEXPR (kCanLoadSRhsQuad && (SwappedTraits::LhsProgress % 4) == 0 &&
2006 (SwappedTraits::LhsProgress <= 16) &&
2007 (SwappedTraits::LhsProgress != 8 || SResPacketHalfSize == 4) &&
2008 (SwappedTraits::LhsProgress != 16 || SResPacketQuarterSize == 4)) {
2009 SAccPacket C0, C1, C2, C3;
2010 straits.initAcc(C0);
2011 straits.initAcc(C1);
2012 straits.initAcc(C2);
2013 straits.initAcc(C3);
2015 const Index spk = (std::max)(1, SwappedTraits::LhsProgress / 4);
2016 const Index endk = (depth / spk) * spk;
2017 const Index endk4 = (depth / (spk * 4)) * (spk * 4);
2020 for (; k < endk4; k += 4 * spk) {
2022 SRhsPacket B_0, B_1;
2024 straits.loadLhsUnaligned(blB + 0 * SwappedTraits::LhsProgress, A0);
2025 straits.loadLhsUnaligned(blB + 1 * SwappedTraits::LhsProgress, A1);
2027 straits.loadRhsQuad(blA + 0 * spk, B_0);
2028 straits.loadRhsQuad(blA + 1 * spk, B_1);
2029 straits.madd(A0, B_0, C0, B_0, fix<0>);
2030 straits.madd(A1, B_1, C1, B_1, fix<0>);
2032 straits.loadLhsUnaligned(blB + 2 * SwappedTraits::LhsProgress, A0);
2033 straits.loadLhsUnaligned(blB + 3 * SwappedTraits::LhsProgress, A1);
2034 straits.loadRhsQuad(blA + 2 * spk, B_0);
2035 straits.loadRhsQuad(blA + 3 * spk, B_1);
2036 straits.madd(A0, B_0, C2, B_0, fix<0>);
2037 straits.madd(A1, B_1, C3, B_1, fix<0>);
2039 blB += 4 * SwappedTraits::LhsProgress;
2042 C0 = padd(padd(C0, C1), padd(C2, C3));
2043 for (; k < endk; k += spk) {
2047 straits.loadLhsUnaligned(blB, A0);
2048 straits.loadRhsQuad(blA, B_0);
2049 straits.madd(A0, B_0, C0, B_0, fix<0>);
2051 blB += SwappedTraits::LhsProgress;
2054 if (SwappedTraits::LhsProgress == 8) {
2056 using SResPacketHalf = std::conditional_t<SwappedTraits::LhsProgress >= 8,
2057 typename unpacket_traits<SResPacket>::half, SResPacket>;
2058 using SLhsPacketHalf = std::conditional_t<SwappedTraits::LhsProgress >= 8,
2059 typename unpacket_traits<SLhsPacket>::half, SLhsPacket>;
2060 using SRhsPacketHalf = std::conditional_t<SwappedTraits::LhsProgress >= 8,
2061 typename unpacket_traits<SRhsPacket>::half, SRhsPacket>;
2062 using SAccPacketHalf = std::conditional_t<SwappedTraits::LhsProgress >= 8,
2063 typename unpacket_traits<SAccPacket>::half, SAccPacket>;
2066 overwrite ? pset1<SResPacketHalf>(ResScalar(0)) : res.template gatherPacket<SResPacketHalf>(i, j2);
2067 SResPacketHalf alphav = pset1<SResPacketHalf>(alpha);
2069 if (depth - endk > 0) {
2073 straits.loadLhsUnaligned(blB, a0);
2074 straits.loadRhs(blA, b0);
2075 SAccPacketHalf c0 = predux_half(C0);
2076 straits.madd(a0, b0, c0, b0, fix<0>);
2077 straits.acc(c0, alphav, R);
2079 straits.acc(predux_half(C0), alphav, R);
2081 res.scatterPacket(i, j2, R);
2082 }
else if (SwappedTraits::LhsProgress == 16) {
2087 last_row_process_16_packets<LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> p;
2088 p(res, straits, blA, blB, depth, endk, i, j2, alpha, C0, overwrite);
2090 SResPacket R = overwrite ? pset1<SResPacket>(ResScalar(0)) : res.template gatherPacket<SResPacket>(i, j2);
2091 SResPacket alphav = pset1<SResPacket>(alpha);
2092 straits.acc(C0, alphav, R);
2093 res.scatterPacket(i, j2, R);
2098 ResScalar C0(0), C1(0), C2(0), C3(0);
2100 for (Index k = 0; k < depth; k++) {
2108 C0 = cj.pmadd(A0, B_0, C0);
2109 C1 = cj.pmadd(A0, B_1, C1);
2113 C2 = cj.pmadd(A0, B_0, C2);
2114 C3 = cj.pmadd(A0, B_1, C3);
2118 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 0), alpha, C0);
2119 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 1), alpha, C1);
2120 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 2), alpha, C2);
2121 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 3), alpha, C3);
2126 for (Index j2 = packet_cols4; j2 < cols; j2++) {
2128 for (Index i = peeled_mc_quarter; i < rows; i += 1) {
2129 const LhsScalar* blA = &blockA[i * strideA + offsetA];
2133 const RhsScalar* blB = &blockB[j2 * strideB + offsetB];
2134 for (Index k = 0; k < depth; k++) {
2135 LhsScalar A0 = blA[k];
2136 RhsScalar B_0 = blB[k];
2137 C0 = cj.pmadd(A0, B_0, C0);
2139 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2), alpha, C0);
2144#ifdef EIGEN_GEBP_DISABLED_INSN_SCHEDULING
2145#pragma GCC pop_options
2146#undef EIGEN_GEBP_DISABLED_INSN_SCHEDULING
2163template <
typename Scalar,
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
2165struct gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate, PanelMode> {
2166 using LinearMapper =
typename DataMapper::LinearMapper;
2167 EIGEN_DONT_INLINE
void operator()(Scalar* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
2168 Index offset = 0)
const;
2171template <
typename Scalar,
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
2173EIGEN_DONT_INLINE
void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate,
2174 PanelMode>::operator()(Scalar* blockA,
const DataMapper& lhs, Index depth,
2175 Index rows, Index stride, Index offset)
const {
2176 using HalfPacket =
typename unpacket_traits<Packet>::half;
2177 using QuarterPacket =
typename unpacket_traits<typename unpacket_traits<Packet>::half>::half;
2179 PacketSize = unpacket_traits<Packet>::size,
2180 HalfPacketSize = unpacket_traits<HalfPacket>::size,
2181 QuarterPacketSize = unpacket_traits<QuarterPacket>::size,
2182 HasHalf = (int)HalfPacketSize < (
int)PacketSize,
2183 HasQuarter = (int)QuarterPacketSize < (
int)HalfPacketSize
2186 EIGEN_ASM_COMMENT(
"EIGEN PRODUCT PACK LHS");
2187 EIGEN_UNUSED_VARIABLE(stride);
2188 EIGEN_UNUSED_VARIABLE(offset);
2189 eigen_assert(((!PanelMode) && stride == 0 && offset == 0) || (PanelMode && stride >= depth && offset <= stride));
2190 eigen_assert(((Pack1 % PacketSize) == 0 && Pack1 <= 4 * PacketSize) || (Pack1 <= 4) || (Pack1 < PacketSize));
2191 conj_if<NumTraits<Scalar>::IsComplex && Conjugate> cj;
2194 const Index peeled_mc3 = Pack1 >= 3 * PacketSize ? (rows / (3 * PacketSize)) * (3 * PacketSize) : 0;
2195 const Index peeled_mc2 =
2196 Pack1 >= 2 * PacketSize ? peeled_mc3 + ((rows - peeled_mc3) / (2 * PacketSize)) * (2 * PacketSize) : 0;
2197 const Index peeled_mc1 =
2198 Pack1 >= 1 * PacketSize ? peeled_mc2 + ((rows - peeled_mc2) / (1 * PacketSize)) * (1 * PacketSize) : 0;
2199 const Index peeled_mc_half =
2200 Pack1 >= HalfPacketSize ? peeled_mc1 + ((rows - peeled_mc1) / (HalfPacketSize)) * (HalfPacketSize) : 0;
2201 const Index peeled_mc_quarter = Pack1 >= QuarterPacketSize ? (rows / (QuarterPacketSize)) * (QuarterPacketSize) : 0;
2202 const Index last_lhs_progress = rows > peeled_mc_quarter ? (rows - peeled_mc_quarter) & ~1 : 0;
2203 const Index peeled_mc0 = Pack2 >= PacketSize ? peeled_mc_quarter
2204 : Pack2 > 1 && last_lhs_progress ? (rows / last_lhs_progress) * last_lhs_progress
2210 EIGEN_IF_CONSTEXPR (Pack1 >= 3 * PacketSize) {
2211 for (; i < peeled_mc3; i += 3 * PacketSize) {
2212 EIGEN_IF_CONSTEXPR (PanelMode) count += (3 * PacketSize) * offset;
2214 for (Index k = 0; k < depth; k++) {
2216 A = lhs.template loadPacket<Packet>(i + 0 * PacketSize, k);
2217 B = lhs.template loadPacket<Packet>(i + 1 * PacketSize, k);
2218 C = lhs.template loadPacket<Packet>(i + 2 * PacketSize, k);
2219 pstore(blockA + count, cj.pconj(A));
2220 count += PacketSize;
2221 pstore(blockA + count, cj.pconj(B));
2222 count += PacketSize;
2223 pstore(blockA + count, cj.pconj(C));
2224 count += PacketSize;
2226 EIGEN_IF_CONSTEXPR (PanelMode) count += (3 * PacketSize) * (stride - offset - depth);
2230 EIGEN_IF_CONSTEXPR (Pack1 >= 2 * PacketSize) {
2231 for (; i < peeled_mc2; i += 2 * PacketSize) {
2232 EIGEN_IF_CONSTEXPR (PanelMode) count += (2 * PacketSize) * offset;
2234 for (Index k = 0; k < depth; k++) {
2236 A = lhs.template loadPacket<Packet>(i + 0 * PacketSize, k);
2237 B = lhs.template loadPacket<Packet>(i + 1 * PacketSize, k);
2238 pstore(blockA + count, cj.pconj(A));
2239 count += PacketSize;
2240 pstore(blockA + count, cj.pconj(B));
2241 count += PacketSize;
2243 EIGEN_IF_CONSTEXPR (PanelMode) count += (2 * PacketSize) * (stride - offset - depth);
2247 EIGEN_IF_CONSTEXPR (Pack1 >= 1 * PacketSize) {
2248 for (; i < peeled_mc1; i += 1 * PacketSize) {
2249 EIGEN_IF_CONSTEXPR (PanelMode) count += (1 * PacketSize) * offset;
2251 for (Index k = 0; k < depth; k++) {
2253 A = lhs.template loadPacket<Packet>(i + 0 * PacketSize, k);
2254 pstore(blockA + count, cj.pconj(A));
2255 count += PacketSize;
2257 EIGEN_IF_CONSTEXPR (PanelMode) count += (1 * PacketSize) * (stride - offset - depth);
2261 EIGEN_IF_CONSTEXPR (HasHalf && Pack1 >= HalfPacketSize) {
2262 for (; i < peeled_mc_half; i += HalfPacketSize) {
2263 EIGEN_IF_CONSTEXPR (PanelMode) count += (HalfPacketSize)*offset;
2265 for (Index k = 0; k < depth; k++) {
2267 A = lhs.template loadPacket<HalfPacket>(i + 0 * (HalfPacketSize), k);
2268 pstoreu(blockA + count, cj.pconj(A));
2269 count += HalfPacketSize;
2271 EIGEN_IF_CONSTEXPR (PanelMode) count += (HalfPacketSize) * (stride - offset - depth);
2275 EIGEN_IF_CONSTEXPR (HasQuarter && Pack1 >= QuarterPacketSize) {
2276 for (; i < peeled_mc_quarter; i += QuarterPacketSize) {
2277 EIGEN_IF_CONSTEXPR (PanelMode) count += (QuarterPacketSize)*offset;
2279 for (Index k = 0; k < depth; k++) {
2281 A = lhs.template loadPacket<QuarterPacket>(i + 0 * (QuarterPacketSize), k);
2282 pstoreu(blockA + count, cj.pconj(A));
2283 count += QuarterPacketSize;
2285 EIGEN_IF_CONSTEXPR (PanelMode) count += (QuarterPacketSize) * (stride - offset - depth);
2301 EIGEN_IF_CONSTEXPR (Pack2 < PacketSize && Pack2 > 1) {
2302 const Index pack2_progress = (HasHalf || HasQuarter) ? last_lhs_progress : Pack2;
2303 const Index peeled = (HasHalf || HasQuarter) ? peeled_mc0 : (rows / Pack2) * Pack2;
2304 for (; i < peeled; i += pack2_progress) {
2305 EIGEN_IF_CONSTEXPR (PanelMode) count += pack2_progress * offset;
2307 for (Index k = 0; k < depth; k++)
2308 for (Index w = 0; w < pack2_progress; w++) blockA[count++] = cj(lhs(i + w, k));
2310 EIGEN_IF_CONSTEXPR (PanelMode) count += pack2_progress * (stride - offset - depth);
2314 for (; i < rows; i++) {
2315 EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
2316 for (Index k = 0; k < depth; k++) blockA[count++] = cj(lhs(i, k));
2317 EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
2321template <
typename Scalar,
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
2323struct gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate, PanelMode> {
2324 using LinearMapper =
typename DataMapper::LinearMapper;
2325 EIGEN_DONT_INLINE
void operator()(Scalar* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
2326 Index offset = 0)
const;
2329template <
typename Scalar,
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
2331EIGEN_DONT_INLINE
void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate,
2332 PanelMode>::operator()(Scalar* blockA,
const DataMapper& lhs, Index depth,
2333 Index rows, Index stride, Index offset)
const {
2334 using HalfPacket =
typename unpacket_traits<Packet>::half;
2335 using QuarterPacket =
typename unpacket_traits<typename unpacket_traits<Packet>::half>::half;
2337 PacketSize = unpacket_traits<Packet>::size,
2338 HalfPacketSize = unpacket_traits<HalfPacket>::size,
2339 QuarterPacketSize = unpacket_traits<QuarterPacket>::size,
2340 HasHalf = (int)HalfPacketSize < (
int)PacketSize,
2341 HasQuarter = (int)QuarterPacketSize < (
int)HalfPacketSize
2344 EIGEN_ASM_COMMENT(
"EIGEN PRODUCT PACK LHS");
2345 EIGEN_UNUSED_VARIABLE(stride);
2346 EIGEN_UNUSED_VARIABLE(offset);
2347 eigen_assert(((!PanelMode) && stride == 0 && offset == 0) || (PanelMode && stride >= depth && offset <= stride));
2348 conj_if<NumTraits<Scalar>::IsComplex && Conjugate> cj;
2350 bool gone_half =
false, gone_quarter =
false, gone_last =
false;
2354 Index psize = PacketSize;
2356 Index remaining_rows = rows - i;
2357 Index peeled_mc = gone_last ? Pack2 > 1 ? (rows / pack) * pack : 0 : i + (remaining_rows / pack) * pack;
2358 Index starting_pos = i;
2359 for (; i < peeled_mc; i += pack) {
2360 EIGEN_IF_CONSTEXPR (PanelMode) count += pack * offset;
2363 if (pack >= psize && psize >= QuarterPacketSize) {
2364 const Index peeled_k = (depth / psize) * psize;
2365 for (; k < peeled_k; k += psize) {
2366 for (Index m = 0; m < pack; m += psize) {
2367 if (psize == PacketSize) {
2368 PacketBlock<Packet> kernel;
2369 for (Index p = 0; p < psize; ++p) kernel.packet[p] = lhs.template loadPacket<Packet>(i + p + m, k);
2371 for (Index p = 0; p < psize; ++p) pstore(blockA + count + m + (pack)*p, cj.pconj(kernel.packet[p]));
2372 }
else if (HasHalf && psize == HalfPacketSize) {
2374 PacketBlock<HalfPacket> kernel_half;
2375 for (Index p = 0; p < psize; ++p)
2376 kernel_half.packet[p] = lhs.template loadPacket<HalfPacket>(i + p + m, k);
2377 ptranspose(kernel_half);
2378 for (Index p = 0; p < psize; ++p) pstore(blockA + count + m + (pack)*p, cj.pconj(kernel_half.packet[p]));
2379 }
else if (HasQuarter && psize == QuarterPacketSize) {
2380 gone_quarter =
true;
2381 PacketBlock<QuarterPacket> kernel_quarter;
2382 for (Index p = 0; p < psize; ++p)
2383 kernel_quarter.packet[p] = lhs.template loadPacket<QuarterPacket>(i + p + m, k);
2384 ptranspose(kernel_quarter);
2385 for (Index p = 0; p < psize; ++p)
2386 pstore(blockA + count + m + (pack)*p, cj.pconj(kernel_quarter.packet[p]));
2389 count += psize * pack;
2393 for (; k < depth; k++) {
2395 for (; w < pack - 3; w += 4) {
2396 Scalar a(cj(lhs(i + w + 0, k))), b(cj(lhs(i + w + 1, k))), c(cj(lhs(i + w + 2, k))), d(cj(lhs(i + w + 3, k)));
2397 blockA[count++] = a;
2398 blockA[count++] = b;
2399 blockA[count++] = c;
2400 blockA[count++] = d;
2403 for (; w < pack; ++w) blockA[count++] = cj(lhs(i + w, k));
2406 EIGEN_IF_CONSTEXPR (PanelMode) count += pack * (stride - offset - depth);
2410 Index left = rows - i;
2412 if (!gone_last && (starting_pos == i || left >= psize / 2 || left >= psize / 4) &&
2413 ((psize / 2 == HalfPacketSize && HasHalf && !gone_half) ||
2414 (psize / 2 == QuarterPacketSize && HasQuarter && !gone_quarter))) {
2432 EIGEN_IF_CONSTEXPR (Pack2 < PacketSize) {
2435 psize = pack = (HasHalf || HasQuarter) ? (left & ~1) : Pack2;
2441 for (; i < rows; i++) {
2442 EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
2443 for (Index k = 0; k < depth; k++) blockA[count++] = cj(lhs(i, k));
2444 EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
2455template <
typename Scalar,
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2456struct gemm_pack_rhs<Scalar, Index, DataMapper, nr,
ColMajor, Conjugate, PanelMode> {
2457 using Packet =
typename packet_traits<Scalar>::type;
2458 using LinearMapper =
typename DataMapper::LinearMapper;
2459 enum { PacketSize = packet_traits<Scalar>::size };
2460 EIGEN_DONT_INLINE
void operator()(Scalar* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
2461 Index offset = 0)
const;
2464template <
typename Scalar,
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2465EIGEN_DONT_INLINE
void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>::operator()(
2466 Scalar* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset)
const {
2467 EIGEN_ASM_COMMENT(
"EIGEN PRODUCT PACK RHS COLMAJOR");
2468 EIGEN_UNUSED_VARIABLE(stride);
2469 EIGEN_UNUSED_VARIABLE(offset);
2470 eigen_assert(((!PanelMode) && stride == 0 && offset == 0) || (PanelMode && stride >= depth && offset <= stride));
2471 conj_if<NumTraits<Scalar>::IsComplex && Conjugate> cj;
2472 Index packet_cols8 = nr >= 8 ? (cols / 8) * 8 : 0;
2473 Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0;
2475 const Index peeled_k = (depth / PacketSize) * PacketSize;
2477 EIGEN_IF_CONSTEXPR (nr >= 8) {
2478 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
2480 EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset;
2481 const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
2482 const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
2483 const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
2484 const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3);
2485 const LinearMapper dm4 = rhs.getLinearMapper(0, j2 + 4);
2486 const LinearMapper dm5 = rhs.getLinearMapper(0, j2 + 5);
2487 const LinearMapper dm6 = rhs.getLinearMapper(0, j2 + 6);
2488 const LinearMapper dm7 = rhs.getLinearMapper(0, j2 + 7);
2490 EIGEN_IF_CONSTEXPR (PacketSize % 2 == 0 && PacketSize <= 8)
2492 for (; k < peeled_k; k += PacketSize) {
2493 EIGEN_IF_CONSTEXPR (PacketSize == 2) {
2494 PacketBlock<Packet, PacketSize == 2 ? 2 : PacketSize> kernel0, kernel1, kernel2, kernel3;
2495 kernel0.packet[0 % PacketSize] = dm0.template loadPacket<Packet>(k);
2496 kernel0.packet[1 % PacketSize] = dm1.template loadPacket<Packet>(k);
2497 kernel1.packet[0 % PacketSize] = dm2.template loadPacket<Packet>(k);
2498 kernel1.packet[1 % PacketSize] = dm3.template loadPacket<Packet>(k);
2499 kernel2.packet[0 % PacketSize] = dm4.template loadPacket<Packet>(k);
2500 kernel2.packet[1 % PacketSize] = dm5.template loadPacket<Packet>(k);
2501 kernel3.packet[0 % PacketSize] = dm6.template loadPacket<Packet>(k);
2502 kernel3.packet[1 % PacketSize] = dm7.template loadPacket<Packet>(k);
2503 ptranspose(kernel0);
2504 ptranspose(kernel1);
2505 ptranspose(kernel2);
2506 ptranspose(kernel3);
2508 pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel0.packet[0 % PacketSize]));
2509 pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel1.packet[0 % PacketSize]));
2510 pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel2.packet[0 % PacketSize]));
2511 pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel3.packet[0 % PacketSize]));
2513 pstoreu(blockB + count + 4 * PacketSize, cj.pconj(kernel0.packet[1 % PacketSize]));
2514 pstoreu(blockB + count + 5 * PacketSize, cj.pconj(kernel1.packet[1 % PacketSize]));
2515 pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel2.packet[1 % PacketSize]));
2516 pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel3.packet[1 % PacketSize]));
2517 count += 8 * PacketSize;
2518 }
else EIGEN_IF_CONSTEXPR (PacketSize == 4) {
2519 PacketBlock<Packet, PacketSize == 4 ? 4 : PacketSize> kernel0, kernel1;
2521 kernel0.packet[0 % PacketSize] = dm0.template loadPacket<Packet>(k);
2522 kernel0.packet[1 % PacketSize] = dm1.template loadPacket<Packet>(k);
2523 kernel0.packet[2 % PacketSize] = dm2.template loadPacket<Packet>(k);
2524 kernel0.packet[3 % PacketSize] = dm3.template loadPacket<Packet>(k);
2525 kernel1.packet[0 % PacketSize] = dm4.template loadPacket<Packet>(k);
2526 kernel1.packet[1 % PacketSize] = dm5.template loadPacket<Packet>(k);
2527 kernel1.packet[2 % PacketSize] = dm6.template loadPacket<Packet>(k);
2528 kernel1.packet[3 % PacketSize] = dm7.template loadPacket<Packet>(k);
2529 ptranspose(kernel0);
2530 ptranspose(kernel1);
2532 pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel0.packet[0 % PacketSize]));
2533 pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel1.packet[0 % PacketSize]));
2534 pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel0.packet[1 % PacketSize]));
2535 pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel1.packet[1 % PacketSize]));
2536 pstoreu(blockB + count + 4 * PacketSize, cj.pconj(kernel0.packet[2 % PacketSize]));
2537 pstoreu(blockB + count + 5 * PacketSize, cj.pconj(kernel1.packet[2 % PacketSize]));
2538 pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel0.packet[3 % PacketSize]));
2539 pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel1.packet[3 % PacketSize]));
2540 count += 8 * PacketSize;
2541 }
else EIGEN_IF_CONSTEXPR (PacketSize == 8) {
2542 PacketBlock<Packet, PacketSize == 8 ? 8 : PacketSize> kernel0;
2544 kernel0.packet[0 % PacketSize] = dm0.template loadPacket<Packet>(k);
2545 kernel0.packet[1 % PacketSize] = dm1.template loadPacket<Packet>(k);
2546 kernel0.packet[2 % PacketSize] = dm2.template loadPacket<Packet>(k);
2547 kernel0.packet[3 % PacketSize] = dm3.template loadPacket<Packet>(k);
2548 kernel0.packet[4 % PacketSize] = dm4.template loadPacket<Packet>(k);
2549 kernel0.packet[5 % PacketSize] = dm5.template loadPacket<Packet>(k);
2550 kernel0.packet[6 % PacketSize] = dm6.template loadPacket<Packet>(k);
2551 kernel0.packet[7 % PacketSize] = dm7.template loadPacket<Packet>(k);
2552 ptranspose(kernel0);
2554 pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel0.packet[0 % PacketSize]));
2555 pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel0.packet[1 % PacketSize]));
2556 pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel0.packet[2 % PacketSize]));
2557 pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel0.packet[3 % PacketSize]));
2558 pstoreu(blockB + count + 4 * PacketSize, cj.pconj(kernel0.packet[4 % PacketSize]));
2559 pstoreu(blockB + count + 5 * PacketSize, cj.pconj(kernel0.packet[5 % PacketSize]));
2560 pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel0.packet[6 % PacketSize]));
2561 pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel0.packet[7 % PacketSize]));
2562 count += 8 * PacketSize;
2567 for (; k < depth; k++) {
2568 blockB[count + 0] = cj(dm0(k));
2569 blockB[count + 1] = cj(dm1(k));
2570 blockB[count + 2] = cj(dm2(k));
2571 blockB[count + 3] = cj(dm3(k));
2572 blockB[count + 4] = cj(dm4(k));
2573 blockB[count + 5] = cj(dm5(k));
2574 blockB[count + 6] = cj(dm6(k));
2575 blockB[count + 7] = cj(dm7(k));
2579 EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * (stride - offset - depth);
2583 EIGEN_IF_CONSTEXPR (nr >= 4) {
2584 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
2586 EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset;
2587 const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
2588 const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
2589 const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
2590 const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3);
2593 EIGEN_IF_CONSTEXPR ((PacketSize % 4) == 0 || PacketSize == 2) {
2594 for (; k < peeled_k; k += PacketSize) {
2595 PacketBlock<Packet, 4> kernel;
2596 kernel.packet[0] = dm0.template loadPacket<Packet>(k);
2597 kernel.packet[1] = dm1.template loadPacket<Packet>(k);
2598 kernel.packet[2] = dm2.template loadPacket<Packet>(k);
2599 kernel.packet[3] = dm3.template loadPacket<Packet>(k);
2600 EIGEN_IF_CONSTEXPR (PacketSize == 2) {
2604 PacketBlock<Packet, 2> tmp01;
2605 tmp01.packet[0] = kernel.packet[0];
2606 tmp01.packet[1] = kernel.packet[1];
2608 PacketBlock<Packet, 2> tmp23;
2609 tmp23.packet[0] = kernel.packet[2];
2610 tmp23.packet[1] = kernel.packet[3];
2612 kernel.packet[0] = tmp01.packet[0];
2613 kernel.packet[1] = tmp23.packet[0];
2614 kernel.packet[2] = tmp01.packet[1];
2615 kernel.packet[3] = tmp23.packet[1];
2619 pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel.packet[0]));
2620 pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel.packet[1]));
2621 pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel.packet[2]));
2622 pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel.packet[3]));
2623 count += 4 * PacketSize;
2626 for (; k < depth; k++) {
2627 blockB[count + 0] = cj(dm0(k));
2628 blockB[count + 1] = cj(dm1(k));
2629 blockB[count + 2] = cj(dm2(k));
2630 blockB[count + 3] = cj(dm3(k));
2634 EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * (stride - offset - depth);
2639 for (Index j2 = packet_cols4; j2 < cols; ++j2) {
2640 EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
2641 const LinearMapper dm0 = rhs.getLinearMapper(0, j2);
2642 for (Index k = 0; k < depth; k++) {
2643 blockB[count] = cj(dm0(k));
2646 EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
2651template <
typename Scalar,
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2652struct gemm_pack_rhs<Scalar, Index, DataMapper, nr,
RowMajor, Conjugate, PanelMode> {
2653 using Packet =
typename packet_traits<Scalar>::type;
2654 using HalfPacket =
typename unpacket_traits<Packet>::half;
2655 using QuarterPacket =
typename unpacket_traits<typename unpacket_traits<Packet>::half>::half;
2656 using LinearMapper =
typename DataMapper::LinearMapper;
2658 PacketSize = packet_traits<Scalar>::size,
2659 HalfPacketSize = unpacket_traits<HalfPacket>::size,
2660 QuarterPacketSize = unpacket_traits<QuarterPacket>::size
2662 EIGEN_DONT_INLINE
void operator()(Scalar* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
2663 Index offset = 0)
const {
2664 EIGEN_ASM_COMMENT(
"EIGEN PRODUCT PACK RHS ROWMAJOR");
2665 EIGEN_UNUSED_VARIABLE(stride);
2666 EIGEN_UNUSED_VARIABLE(offset);
2667 eigen_assert(((!PanelMode) && stride == 0 && offset == 0) || (PanelMode && stride >= depth && offset <= stride));
2668 constexpr bool HasHalf = (int)HalfPacketSize < (
int)PacketSize;
2669 constexpr bool HasQuarter = (int)QuarterPacketSize < (
int)HalfPacketSize;
2670 conj_if<NumTraits<Scalar>::IsComplex && Conjugate> cj;
2671 Index packet_cols8 = nr >= 8 ? (cols / 8) * 8 : 0;
2672 Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0;
2675 EIGEN_IF_CONSTEXPR (nr >= 8) {
2676 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
2678 EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset;
2679 for (Index k = 0; k < depth; k++) {
2680 EIGEN_IF_CONSTEXPR (PacketSize == 8) {
2681 Packet A = rhs.template loadPacket<Packet>(k, j2);
2682 pstoreu(blockB + count, cj.pconj(A));
2683 count += PacketSize;
2684 }
else EIGEN_IF_CONSTEXPR (PacketSize == 4) {
2685 Packet A = rhs.template loadPacket<Packet>(k, j2);
2686 Packet B = rhs.template loadPacket<Packet>(k, j2 + 4);
2687 pstoreu(blockB + count, cj.pconj(A));
2688 pstoreu(blockB + count + PacketSize, cj.pconj(B));
2689 count += 2 * PacketSize;
2691 const LinearMapper dm0 = rhs.getLinearMapper(k, j2);
2692 blockB[count + 0] = cj(dm0(0));
2693 blockB[count + 1] = cj(dm0(1));
2694 blockB[count + 2] = cj(dm0(2));
2695 blockB[count + 3] = cj(dm0(3));
2696 blockB[count + 4] = cj(dm0(4));
2697 blockB[count + 5] = cj(dm0(5));
2698 blockB[count + 6] = cj(dm0(6));
2699 blockB[count + 7] = cj(dm0(7));
2704 EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * (stride - offset - depth);
2708 EIGEN_IF_CONSTEXPR (nr >= 4) {
2709 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
2711 EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset;
2712 for (Index k = 0; k < depth; k++) {
2713 EIGEN_IF_CONSTEXPR (PacketSize == 4) {
2714 Packet A = rhs.template loadPacket<Packet>(k, j2);
2715 pstoreu(blockB + count, cj.pconj(A));
2716 count += PacketSize;
2717 }
else EIGEN_IF_CONSTEXPR (HasHalf && HalfPacketSize == 4) {
2718 HalfPacket A = rhs.template loadPacket<HalfPacket>(k, j2);
2719 pstoreu(blockB + count, cj.pconj(A));
2720 count += HalfPacketSize;
2721 }
else EIGEN_IF_CONSTEXPR (HasQuarter && QuarterPacketSize == 4) {
2722 QuarterPacket A = rhs.template loadPacket<QuarterPacket>(k, j2);
2723 pstoreu(blockB + count, cj.pconj(A));
2724 count += QuarterPacketSize;
2726 const LinearMapper dm0 = rhs.getLinearMapper(k, j2);
2727 blockB[count + 0] = cj(dm0(0));
2728 blockB[count + 1] = cj(dm0(1));
2729 blockB[count + 2] = cj(dm0(2));
2730 blockB[count + 3] = cj(dm0(3));
2735 EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * (stride - offset - depth);
2739 for (Index j2 = packet_cols4; j2 < cols; ++j2) {
2740 EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
2741 for (Index k = 0; k < depth; k++) {
2742 blockB[count] = cj(rhs(k, j2));
2745 EIGEN_IF_CONSTEXPR (PanelMode) count += stride - offset - depth;
2754inline std::ptrdiff_t l1CacheSize() {
2755 std::ptrdiff_t l1, l2, l3;
2756 internal::manage_caching_sizes(GetAction, &l1, &l2, &l3);
2762inline std::ptrdiff_t l2CacheSize() {
2763 std::ptrdiff_t l1, l2, l3;
2764 internal::manage_caching_sizes(GetAction, &l1, &l2, &l3);
2770inline std::ptrdiff_t l3CacheSize() {
2771 std::ptrdiff_t l1, l2, l3;
2772 internal::manage_caching_sizes(GetAction, &l1, &l2, &l3);
2782inline void setCpuCacheSizes(std::ptrdiff_t l1, std::ptrdiff_t l2, std::ptrdiff_t l3) {
2783 internal::manage_caching_sizes(SetAction, &l1, &l2, &l3);
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321