11#ifndef EIGEN_PACKET_MATH_CLANG_H
12#define EIGEN_PACKET_MATH_CLANG_H
15#include "../../InternalHeaderCheck.h"
24template <
typename ScalarT,
int n>
25using VectorType = ScalarT __attribute__((ext_vector_type(n), aligned(n *
sizeof(ScalarT))));
44static_assert(EIGEN_GENERIC_VECTOR_SIZE_BYTES == 16 || EIGEN_GENERIC_VECTOR_SIZE_BYTES == 32 ||
45 EIGEN_GENERIC_VECTOR_SIZE_BYTES == 64,
46 "EIGEN_GENERIC_VECTOR_SIZE_BYTES must be 16, 32, or 64");
48constexpr int kFloatPacketSize = EIGEN_GENERIC_VECTOR_SIZE_BYTES /
sizeof(float);
49constexpr int kDoublePacketSize = EIGEN_GENERIC_VECTOR_SIZE_BYTES /
sizeof(double);
50using PacketXf = detail::VectorType<float, kFloatPacketSize>;
51using PacketXd = detail::VectorType<double, kDoublePacketSize>;
52using PacketXi = detail::VectorType<int32_t, kFloatPacketSize>;
53using PacketXl = detail::VectorType<int64_t, kDoublePacketSize>;
56struct generic_float_packet_traits : default_packet_traits {
103struct packet_traits<float> : generic_float_packet_traits {
104 using type = PacketXf;
105 using half = PacketXf;
107 size = kFloatPacketSize,
112struct packet_traits<double> : generic_float_packet_traits {
113 using type = PacketXd;
114 using half = PacketXd;
117 enum { size = kDoublePacketSize, HasACos = 0, HasASin = 0 };
120struct generic_integer_packet_traits : default_packet_traits {
152struct packet_traits<int32_t> : generic_integer_packet_traits {
153 using type = PacketXi;
154 using half = PacketXi;
156 size = kFloatPacketSize,
161struct packet_traits<int64_t> : generic_integer_packet_traits {
162 using type = PacketXl;
163 using half = PacketXl;
165 size = kDoublePacketSize,
170struct generic_unpacket_traits : default_unpacket_traits {
172 alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES,
178struct unpacket_traits<PacketXf> : generic_unpacket_traits {
180 using half = PacketXf;
181 using integer_packet = PacketXi;
183 size = kFloatPacketSize,
187struct unpacket_traits<PacketXd> : generic_unpacket_traits {
189 using half = PacketXd;
190 using integer_packet = PacketXl;
192 size = kDoublePacketSize,
196struct unpacket_traits<PacketXi> : generic_unpacket_traits {
197 using type = int32_t;
198 using half = PacketXi;
200 size = kFloatPacketSize,
204struct unpacket_traits<PacketXl> : generic_unpacket_traits {
205 using type = int64_t;
206 using half = PacketXl;
208 size = kDoublePacketSize,
214template <
typename VectorT>
215struct ScalarTypeOfVector {
216 using type = std::remove_all_extents_t<std::remove_reference_t<
decltype(VectorT()[0])>>;
219template <
typename VectorT>
220using scalar_type_of_vector_t =
typename ScalarTypeOfVector<VectorT>::type;
222template <
typename VectorType>
223struct UnsignedVectorHelper {
225 static constexpr int n = __builtin_vectorelements(v);
226 using UnsignedScalar = std::make_unsigned_t<scalar_type_of_vector_t<VectorType>>;
227 using type = UnsignedScalar __attribute__((ext_vector_type(n), aligned(n *
sizeof(UnsignedScalar))));
230template <
typename VectorT>
231using unsigned_vector_t =
typename UnsignedVectorHelper<VectorT>::type;
233template <
typename VectorT>
234constexpr int vector_elements() {
235 return static_cast<int>(
sizeof(VectorT) /
sizeof(scalar_type_of_vector_t<VectorT>));
240template <
typename VectorT>
241struct SignedVectorHelper {
242 using SignedScalar = std::conditional_t<
sizeof(scalar_type_of_vector_t<VectorT>) == 4, int32_t, int64_t>;
243 using type = VectorType<SignedScalar, vector_elements<VectorT>()>;
246template <
typename VectorT>
247using signed_vector_t =
typename SignedVectorHelper<VectorT>::type;
249template <
typename VectorT>
250using half_vector_t = VectorType<scalar_type_of_vector_t<VectorT>, vector_elements<VectorT>() / 2>;
252template <
typename VectorT>
253using quarter_vector_t = VectorType<scalar_type_of_vector_t<VectorT>, vector_elements<VectorT>() / 4>;
255template <
typename VectorT>
256using scalar_pair_t = std::pair<scalar_type_of_vector_t<VectorT>, scalar_type_of_vector_t<VectorT>>;
261template <
typename VectorT>
262using vector_indices = std::make_index_sequence<vector_elements<VectorT>()>;
265template <
typename VectorT>
266EIGEN_STRONG_INLINE VectorT load_vector_unaligned(
const scalar_type_of_vector_t<VectorT>* from) {
268 __builtin_memcpy(&to, from,
sizeof(VectorT));
272template <
typename VectorT>
273EIGEN_STRONG_INLINE VectorT load_vector_aligned(
const scalar_type_of_vector_t<VectorT>* from) {
274 eigen_assert((std::uintptr_t(from) %
alignof(VectorT) == 0) &&
"load_vector_aligned");
275 return *
reinterpret_cast<const VectorT*
>(assume_aligned<alignof(VectorT)>(from));
278template <
typename VectorT>
279EIGEN_STRONG_INLINE
void store_vector_unaligned(scalar_type_of_vector_t<VectorT>* to,
const VectorT& from) {
280 __builtin_memcpy(to, &from,
sizeof(VectorT));
283template <
typename VectorT>
284EIGEN_STRONG_INLINE
void store_vector_aligned(scalar_type_of_vector_t<VectorT>* to,
const VectorT& from) {
285 eigen_assert((std::uintptr_t(to) %
alignof(VectorT) == 0) &&
"store_vector_aligned");
286 *
reinterpret_cast<VectorT*
>(assume_aligned<alignof(VectorT)>(to)) = from;
294#define EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PACKET_TYPE) \
296 EIGEN_STRONG_INLINE PACKET_TYPE ploadu<PACKET_TYPE>(const detail::scalar_type_of_vector_t<PACKET_TYPE>* from) { \
297 return detail::load_vector_unaligned<PACKET_TYPE>(from); \
300 EIGEN_STRONG_INLINE PACKET_TYPE pload<PACKET_TYPE>(const detail::scalar_type_of_vector_t<PACKET_TYPE>* from) { \
301 return detail::load_vector_aligned<PACKET_TYPE>(from); \
304 EIGEN_STRONG_INLINE void pstoreu<detail::scalar_type_of_vector_t<PACKET_TYPE>, PACKET_TYPE>( \
305 detail::scalar_type_of_vector_t<PACKET_TYPE> * to, const PACKET_TYPE& from) { \
306 detail::store_vector_unaligned<PACKET_TYPE>(to, from); \
309 EIGEN_STRONG_INLINE void pstore<detail::scalar_type_of_vector_t<PACKET_TYPE>, PACKET_TYPE>( \
310 detail::scalar_type_of_vector_t<PACKET_TYPE> * to, const PACKET_TYPE& from) { \
311 detail::store_vector_aligned<PACKET_TYPE>(to, from); \
314EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PacketXf)
315EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PacketXd)
316EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PacketXi)
317EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PacketXl)
318#undef EIGEN_CLANG_PACKET_LOAD_STORE_PACKET
322EIGEN_STRONG_INLINE PacketXf pset1frombits<PacketXf>(uint32_t from) {
323 return PacketXf(numext::bit_cast<float>(from));
327EIGEN_STRONG_INLINE PacketXd pset1frombits<PacketXd>(uint64_t from) {
328 return PacketXd(numext::bit_cast<double>(from));
331#define EIGEN_CLANG_PACKET_SET1(PACKET_TYPE) \
333 EIGEN_STRONG_INLINE PACKET_TYPE pset1<PACKET_TYPE>(const unpacket_traits<PACKET_TYPE>::type& from) { \
334 return PACKET_TYPE(from); \
337 EIGEN_STRONG_INLINE unpacket_traits<PACKET_TYPE>::type pfirst<PACKET_TYPE>(const PACKET_TYPE& from) { \
341EIGEN_CLANG_PACKET_SET1(PacketXf)
342EIGEN_CLANG_PACKET_SET1(PacketXd)
343EIGEN_CLANG_PACKET_SET1(PacketXi)
344EIGEN_CLANG_PACKET_SET1(PacketXl)
345#undef EIGEN_CLANG_PACKET_SET1
348#define EIGEN_CLANG_PACKET_ARITHMETIC(PACKET_TYPE) \
350 EIGEN_STRONG_INLINE PACKET_TYPE pisnan<PACKET_TYPE>(const PACKET_TYPE& a) { \
351 return reinterpret_cast<PACKET_TYPE>(a != a); \
354 EIGEN_STRONG_INLINE PACKET_TYPE pnegate<PACKET_TYPE>(const PACKET_TYPE& a) { \
358EIGEN_CLANG_PACKET_ARITHMETIC(PacketXf)
359EIGEN_CLANG_PACKET_ARITHMETIC(PacketXd)
360EIGEN_CLANG_PACKET_ARITHMETIC(PacketXi)
361EIGEN_CLANG_PACKET_ARITHMETIC(PacketXl)
362#undef EIGEN_CLANG_PACKET_ARITHMETIC
370EIGEN_STRONG_INLINE PacketXi preinterpret_float_to_int(
const PacketXf& a) {
return reinterpret_cast<PacketXi
>(a); }
371EIGEN_STRONG_INLINE PacketXf preinterpret_int_to_float(
const PacketXi& a) {
return reinterpret_cast<PacketXf
>(a); }
372EIGEN_STRONG_INLINE PacketXl preinterpret_double_to_long(
const PacketXd& a) {
return reinterpret_cast<PacketXl
>(a); }
373EIGEN_STRONG_INLINE PacketXd preinterpret_long_to_double(
const PacketXl& a) {
return reinterpret_cast<PacketXd
>(a); }
378#define EIGEN_CLANG_PACKET_BITWISE_INT(PACKET_TYPE) \
380 constexpr EIGEN_STRONG_INLINE PACKET_TYPE pzero<PACKET_TYPE>(const PACKET_TYPE& ) { \
381 return PACKET_TYPE(0); \
384 constexpr EIGEN_STRONG_INLINE PACKET_TYPE ptrue<PACKET_TYPE>(const PACKET_TYPE& ) { \
385 return numext::bit_cast<PACKET_TYPE>(PACKET_TYPE(0) == PACKET_TYPE(0)); \
388 EIGEN_STRONG_INLINE PACKET_TYPE pand<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
392 EIGEN_STRONG_INLINE PACKET_TYPE por<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
396 EIGEN_STRONG_INLINE PACKET_TYPE pxor<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
400 EIGEN_STRONG_INLINE PACKET_TYPE pandnot<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
404 EIGEN_STRONG_INLINE PACKET_TYPE parithmetic_shift_right(const PACKET_TYPE& a) { \
408 EIGEN_STRONG_INLINE PACKET_TYPE plogical_shift_right(const PACKET_TYPE& a) { \
409 using UnsignedT = detail::unsigned_vector_t<PACKET_TYPE>; \
410 return reinterpret_cast<PACKET_TYPE>(reinterpret_cast<UnsignedT>(a) >> N); \
413 EIGEN_STRONG_INLINE PACKET_TYPE plogical_shift_left(const PACKET_TYPE& a) { \
417EIGEN_CLANG_PACKET_BITWISE_INT(PacketXi)
418EIGEN_CLANG_PACKET_BITWISE_INT(PacketXl)
419#undef EIGEN_CLANG_PACKET_BITWISE_INT
422#define EIGEN_CLANG_PACKET_BITWISE_FLOAT(PACKET_TYPE, CAST_TO_INT, CAST_FROM_INT) \
424 constexpr EIGEN_STRONG_INLINE PACKET_TYPE pzero<PACKET_TYPE>(const PACKET_TYPE& ) { \
425 using Scalar = detail::scalar_type_of_vector_t<PACKET_TYPE>; \
426 return PACKET_TYPE(Scalar(0)); \
429 EIGEN_STRONG_INLINE PACKET_TYPE ptrue<PACKET_TYPE>(const PACKET_TYPE& ) { \
430 using Scalar = detail::scalar_type_of_vector_t<PACKET_TYPE>; \
431 PACKET_TYPE r = numext::bit_cast<PACKET_TYPE>(PACKET_TYPE(Scalar(0)) == PACKET_TYPE(Scalar(0))); \
432 EIGEN_FAST_MATH_CONSTANT_BARRIER(r); \
436 EIGEN_STRONG_INLINE PACKET_TYPE pand<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
437 return CAST_FROM_INT(CAST_TO_INT(a) & CAST_TO_INT(b)); \
440 EIGEN_STRONG_INLINE PACKET_TYPE por<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
441 return CAST_FROM_INT(CAST_TO_INT(a) | CAST_TO_INT(b)); \
444 EIGEN_STRONG_INLINE PACKET_TYPE pxor<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
445 return CAST_FROM_INT(CAST_TO_INT(a) ^ CAST_TO_INT(b)); \
448 EIGEN_STRONG_INLINE PACKET_TYPE pandnot<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
449 return CAST_FROM_INT(CAST_TO_INT(a) & ~CAST_TO_INT(b)); \
452EIGEN_CLANG_PACKET_BITWISE_FLOAT(PacketXf, detail::preinterpret_float_to_int, detail::preinterpret_int_to_float)
453EIGEN_CLANG_PACKET_BITWISE_FLOAT(PacketXd, detail::preinterpret_double_to_long, detail::preinterpret_long_to_double)
454#undef EIGEN_CLANG_PACKET_BITWISE_FLOAT
461#define EIGEN_CLANG_PACKET_CMP(PACKET_TYPE, INT_PACKET_TYPE) \
463 EIGEN_STRONG_INLINE PACKET_TYPE pcmp_eq<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
464 return numext::bit_cast<PACKET_TYPE>(INT_PACKET_TYPE(a == b)); \
467 EIGEN_STRONG_INLINE PACKET_TYPE pcmp_lt<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
468 return numext::bit_cast<PACKET_TYPE>(INT_PACKET_TYPE(a < b)); \
471 EIGEN_STRONG_INLINE PACKET_TYPE pcmp_le<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
472 return numext::bit_cast<PACKET_TYPE>(INT_PACKET_TYPE(a <= b)); \
475 EIGEN_STRONG_INLINE PACKET_TYPE pcmp_lt_or_nan<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
476 return numext::bit_cast<PACKET_TYPE>(INT_PACKET_TYPE(!(a >= b))); \
479EIGEN_CLANG_PACKET_CMP(PacketXf, PacketXi)
480EIGEN_CLANG_PACKET_CMP(PacketXd, PacketXl)
481#undef EIGEN_CLANG_PACKET_CMP
492 template <
typename VectorT>
493 EIGEN_STRONG_INLINE VectorT operator()(
const VectorT& a,
const VectorT& b)
const {
494 return b < a ? b : a;
498 template <
typename VectorT>
499 EIGEN_STRONG_INLINE VectorT operator()(
const VectorT& a,
const VectorT& b)
const {
500 return b > a ? b : a;
512 template <
typename VectorT>
513 EIGEN_STRONG_INLINE VectorT operator()(
const VectorT& a,
const VectorT& b)
const {
514#if EIGEN_HAS_BUILTIN(__builtin_elementwise_minnum)
515 return __builtin_elementwise_minnum(a, b);
517 return __builtin_elementwise_min(a, b);
522 template <
typename VectorT>
523 EIGEN_STRONG_INLINE VectorT operator()(
const VectorT& a,
const VectorT& b)
const {
524#if EIGEN_HAS_BUILTIN(__builtin_elementwise_maxnum)
525 return __builtin_elementwise_maxnum(a, b);
527 return __builtin_elementwise_max(a, b);
535 template <
typename VectorT>
536 EIGEN_STRONG_INLINE VectorT operator()(
const VectorT& a,
const VectorT& b)
const {
537 return b != b ? b : pmin_op()(a, b);
541 template <
typename VectorT>
542 EIGEN_STRONG_INLINE VectorT operator()(
const VectorT& a,
const VectorT& b)
const {
543 return b != b ? b : pmax_op()(a, b);
549#define EIGEN_CLANG_PACKET_MINMAX_SELECT(PACKET_TYPE) \
551 EIGEN_STRONG_INLINE PACKET_TYPE pmin<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
552 return detail::pmin_op()(a, b); \
555 EIGEN_STRONG_INLINE PACKET_TYPE pmax<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
556 return detail::pmax_op()(a, b); \
559 EIGEN_STRONG_INLINE PACKET_TYPE pselect<PACKET_TYPE>(const PACKET_TYPE& mask, const PACKET_TYPE& a, \
560 const PACKET_TYPE& b) { \
565 return reinterpret_cast<detail::signed_vector_t<PACKET_TYPE>>(mask) < 0 ? a : b; \
568EIGEN_CLANG_PACKET_MINMAX_SELECT(PacketXf)
569EIGEN_CLANG_PACKET_MINMAX_SELECT(PacketXd)
570EIGEN_CLANG_PACKET_MINMAX_SELECT(PacketXi)
571EIGEN_CLANG_PACKET_MINMAX_SELECT(PacketXl)
572#undef EIGEN_CLANG_PACKET_MINMAX_SELECT
575#define EIGEN_CLANG_PACKET_MINMAX_FLOAT(PACKET_TYPE) \
577 EIGEN_STRONG_INLINE PACKET_TYPE pmin<PropagateNumbers, PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
578 return detail::pmin_num_op()(a, b); \
581 EIGEN_STRONG_INLINE PACKET_TYPE pmax<PropagateNumbers, PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
582 return detail::pmax_num_op()(a, b); \
585 EIGEN_STRONG_INLINE PACKET_TYPE pmin<PropagateNaN, PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
586 return detail::pmin_nan_op()(a, b); \
589 EIGEN_STRONG_INLINE PACKET_TYPE pmax<PropagateNaN, PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
590 return detail::pmax_nan_op()(a, b); \
593EIGEN_CLANG_PACKET_MINMAX_FLOAT(PacketXf)
594EIGEN_CLANG_PACKET_MINMAX_FLOAT(PacketXd)
595#undef EIGEN_CLANG_PACKET_MINMAX_FLOAT
597#if EIGEN_HAS_BUILTIN(__builtin_elementwise_abs)
598#define EIGEN_CLANG_PACKET_ABS(PACKET_TYPE) \
600 EIGEN_STRONG_INLINE PACKET_TYPE pabs<PACKET_TYPE>(const PACKET_TYPE& a) { \
601 return __builtin_elementwise_abs(a); \
604EIGEN_CLANG_PACKET_ABS(PacketXf)
605EIGEN_CLANG_PACKET_ABS(PacketXd)
606EIGEN_CLANG_PACKET_ABS(PacketXi)
607EIGEN_CLANG_PACKET_ABS(PacketXl)
608#undef EIGEN_CLANG_PACKET_ABS
614EIGEN_STRONG_INLINE PacketXf psignbit(
const PacketXf& a) {
615 return reinterpret_cast<PacketXf
>(
reinterpret_cast<PacketXi
>(a) < 0);
618EIGEN_STRONG_INLINE PacketXd psignbit(
const PacketXd& a) {
619 return reinterpret_cast<PacketXd
>(
reinterpret_cast<PacketXl
>(a) < 0);
624#if EIGEN_HAS_BUILTIN(__builtin_elementwise_floor) && EIGEN_HAS_BUILTIN(__builtin_elementwise_ceil) && \
625 EIGEN_HAS_BUILTIN(__builtin_elementwise_round) && EIGEN_HAS_BUILTIN(__builtin_elementwise_roundeven) && \
626 EIGEN_HAS_BUILTIN(__builtin_elementwise_trunc) && EIGEN_HAS_BUILTIN(__builtin_elementwise_sqrt)
627#define EIGEN_CLANG_PACKET_MATH_FLOAT(PACKET_TYPE) \
629 EIGEN_STRONG_INLINE PACKET_TYPE pfloor<PACKET_TYPE>(const PACKET_TYPE& a) { \
630 return __builtin_elementwise_floor(a); \
633 EIGEN_STRONG_INLINE PACKET_TYPE pceil<PACKET_TYPE>(const PACKET_TYPE& a) { \
634 return __builtin_elementwise_ceil(a); \
637 EIGEN_STRONG_INLINE PACKET_TYPE pround<PACKET_TYPE>(const PACKET_TYPE& a) { \
638 return __builtin_elementwise_round(a); \
641 EIGEN_STRONG_INLINE PACKET_TYPE print<PACKET_TYPE>(const PACKET_TYPE& a) { \
642 return __builtin_elementwise_roundeven(a); \
645 EIGEN_STRONG_INLINE PACKET_TYPE ptrunc<PACKET_TYPE>(const PACKET_TYPE& a) { \
646 return __builtin_elementwise_trunc(a); \
649 EIGEN_STRONG_INLINE PACKET_TYPE psqrt<PACKET_TYPE>(const PACKET_TYPE& a) { \
650 return __builtin_elementwise_sqrt(a); \
653EIGEN_CLANG_PACKET_MATH_FLOAT(PacketXf)
654EIGEN_CLANG_PACKET_MATH_FLOAT(PacketXd)
655#undef EIGEN_CLANG_PACKET_MATH_FLOAT
659#if (defined(EIGEN_VECTORIZE_FMA) || defined(__FMA__)) && EIGEN_HAS_BUILTIN(__builtin_elementwise_fma)
660#define EIGEN_CLANG_PACKET_MADD(PACKET_TYPE) \
662 struct is_fused_madd<PACKET_TYPE> : true_type {}; \
664 EIGEN_STRONG_INLINE PACKET_TYPE pmadd(const PACKET_TYPE& a, const PACKET_TYPE& b, const PACKET_TYPE& c) { \
665 return __builtin_elementwise_fma(a, b, c); \
668EIGEN_CLANG_PACKET_MADD(PacketXf)
669EIGEN_CLANG_PACKET_MADD(PacketXd)
670#undef EIGEN_CLANG_PACKET_MADD
673#define EIGEN_CLANG_PACKET_SCATTER_GATHER(PACKET_TYPE) \
675 EIGEN_STRONG_INLINE void pscatter(unpacket_traits<PACKET_TYPE>::type* to, const PACKET_TYPE& from, Index stride) { \
676 constexpr int size = unpacket_traits<PACKET_TYPE>::size; \
677 for (int i = 0; i < size; ++i) { \
678 to[i * stride] = from[i]; \
682 EIGEN_STRONG_INLINE PACKET_TYPE pgather<typename unpacket_traits<PACKET_TYPE>::type, PACKET_TYPE>( \
683 const unpacket_traits<PACKET_TYPE>::type* from, Index stride) { \
684 constexpr int size = unpacket_traits<PACKET_TYPE>::size; \
685 PACKET_TYPE result; \
686 for (int i = 0; i < size; ++i) { \
687 result[i] = from[i * stride]; \
692EIGEN_CLANG_PACKET_SCATTER_GATHER(PacketXf)
693EIGEN_CLANG_PACKET_SCATTER_GATHER(PacketXd)
694EIGEN_CLANG_PACKET_SCATTER_GATHER(PacketXi)
695EIGEN_CLANG_PACKET_SCATTER_GATHER(PacketXl)
697#undef EIGEN_CLANG_PACKET_SCATTER_GATHER
700#if EIGEN_HAS_BUILTIN(__builtin_shufflevector)
704template <
typename VectorT, std::size_t... Is>
705EIGEN_STRONG_INLINE half_vector_t<VectorT> lower_half_impl(
const VectorT& a, std::index_sequence<Is...>) {
706 return __builtin_shufflevector(a, a, Is...);
709template <
typename VectorT, std::size_t... Is>
710EIGEN_STRONG_INLINE half_vector_t<VectorT> upper_half_impl(
const VectorT& a, std::index_sequence<Is...>) {
711 return __builtin_shufflevector(a, a, (
sizeof...(Is) + Is)...);
714template <
typename VectorT,
typename HalfT, std::size_t... Is>
715EIGEN_STRONG_INLINE VectorT concat_halves_impl(
const HalfT& lo,
const HalfT& hi, std::index_sequence<Is...>) {
716 return __builtin_shufflevector(lo, hi, Is...);
719template <
typename VectorT>
720EIGEN_STRONG_INLINE half_vector_t<VectorT> lower_half(
const VectorT& a) {
721 return lower_half_impl(a, vector_indices<half_vector_t<VectorT>>{});
724template <
typename VectorT>
725EIGEN_STRONG_INLINE half_vector_t<VectorT> upper_half(
const VectorT& a) {
726 return upper_half_impl(a, vector_indices<half_vector_t<VectorT>>{});
729template <
typename VectorT,
typename HalfT>
730EIGEN_STRONG_INLINE VectorT concat_halves(
const HalfT& lo,
const HalfT& hi) {
731 return concat_halves_impl<VectorT>(lo, hi, vector_indices<VectorT>{});
735template <
typename Packet, std::size_t... Is>
736EIGEN_STRONG_INLINE Packet preverse_impl(
const Packet& a, std::index_sequence<Is...>) {
737 return __builtin_shufflevector(a, a, (
sizeof...(Is) - 1 - Is)...);
741template <
typename Packet, std::size_t... Is>
742EIGEN_STRONG_INLINE Packet ploaddup_impl(
const typename unpacket_traits<Packet>::type* from,
743 std::index_sequence<Is...>) {
744 static_assert((unpacket_traits<Packet>::size) % 2 == 0,
"Packet size must be a multiple of 2");
745 using HalfT = half_vector_t<Packet>;
746 const HalfT a = load_vector_unaligned<HalfT>(from);
747 return __builtin_shufflevector(a, a, (Is / 2)...);
751template <
typename Packet, std::size_t... Is>
752EIGEN_STRONG_INLINE Packet ploadquad_impl(
const typename unpacket_traits<Packet>::type* from,
753 std::index_sequence<Is...>) {
754 static_assert((unpacket_traits<Packet>::size) % 4 == 0,
"Packet size must be a multiple of 4");
755 using QuarterT = quarter_vector_t<Packet>;
756 const QuarterT a = load_vector_unaligned<QuarterT>(from);
757 return __builtin_shufflevector(a, a, (Is / 4)...);
760template <
typename Packet, std::size_t... Is>
761EIGEN_STRONG_INLINE Packet plset_impl(
const typename unpacket_traits<Packet>::type& a, std::index_sequence<Is...>) {
762 using Scalar =
typename unpacket_traits<Packet>::type;
763 return Packet{(a + Scalar(Is))...};
768template <
typename Packet, std::size_t... Is>
769EIGEN_STRONG_INLINE
typename unpacket_traits<Packet>::integer_packet peven_mask_impl(std::index_sequence<Is...>) {
770 using IntegerPacket =
typename unpacket_traits<Packet>::integer_packet;
771 using Bits = scalar_type_of_vector_t<IntegerPacket>;
772 return IntegerPacket{(Is % 2 == 0 ? Bits(-1) : Bits(0))...};
777#define EIGEN_CLANG_PACKET_PREVERSE(PACKET_TYPE) \
779 EIGEN_STRONG_INLINE PACKET_TYPE preverse<PACKET_TYPE>(const PACKET_TYPE& a) { \
780 return detail::preverse_impl(a, detail::vector_indices<PACKET_TYPE>{}); \
783EIGEN_CLANG_PACKET_PREVERSE(PacketXf)
784EIGEN_CLANG_PACKET_PREVERSE(PacketXd)
785EIGEN_CLANG_PACKET_PREVERSE(PacketXi)
786EIGEN_CLANG_PACKET_PREVERSE(PacketXl)
787#undef EIGEN_CLANG_PACKET_PREVERSE
789#define EIGEN_CLANG_PACKET_LOADDUP(PACKET_TYPE) \
791 EIGEN_STRONG_INLINE PACKET_TYPE ploaddup<PACKET_TYPE>(const unpacket_traits<PACKET_TYPE>::type* from) { \
792 return detail::ploaddup_impl<PACKET_TYPE>(from, detail::vector_indices<PACKET_TYPE>{}); \
795EIGEN_CLANG_PACKET_LOADDUP(PacketXf)
796EIGEN_CLANG_PACKET_LOADDUP(PacketXd)
797EIGEN_CLANG_PACKET_LOADDUP(PacketXi)
798EIGEN_CLANG_PACKET_LOADDUP(PacketXl)
799#undef EIGEN_CLANG_PACKET_LOADDUP
801#define EIGEN_CLANG_PACKET_LOADQUAD(PACKET_TYPE) \
803 EIGEN_STRONG_INLINE PACKET_TYPE ploadquad<PACKET_TYPE>(const unpacket_traits<PACKET_TYPE>::type* from) { \
804 return detail::ploadquad_impl<PACKET_TYPE>(from, detail::vector_indices<PACKET_TYPE>{}); \
807EIGEN_CLANG_PACKET_LOADQUAD(PacketXf)
808EIGEN_CLANG_PACKET_LOADQUAD(PacketXi)
809#if EIGEN_GENERIC_VECTOR_SIZE_BYTES >= 32
811EIGEN_CLANG_PACKET_LOADQUAD(PacketXd)
812EIGEN_CLANG_PACKET_LOADQUAD(PacketXl)
814#undef EIGEN_CLANG_PACKET_LOADQUAD
816#define EIGEN_CLANG_PACKET_PLSET(PACKET_TYPE) \
818 EIGEN_STRONG_INLINE PACKET_TYPE plset<PACKET_TYPE>(const unpacket_traits<PACKET_TYPE>::type& a) { \
819 return detail::plset_impl<PACKET_TYPE>(a, detail::vector_indices<PACKET_TYPE>{}); \
822EIGEN_CLANG_PACKET_PLSET(PacketXf)
823EIGEN_CLANG_PACKET_PLSET(PacketXd)
824EIGEN_CLANG_PACKET_PLSET(PacketXi)
825EIGEN_CLANG_PACKET_PLSET(PacketXl)
826#undef EIGEN_CLANG_PACKET_PLSET
830EIGEN_STRONG_INLINE PacketXf peven_mask(
const PacketXf& ) {
831 PacketXf r = numext::bit_cast<PacketXf>(detail::peven_mask_impl<PacketXf>(detail::vector_indices<PacketXf>{}));
832 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
836EIGEN_STRONG_INLINE PacketXd peven_mask(
const PacketXd& ) {
837 PacketXd r = numext::bit_cast<PacketXd>(detail::peven_mask_impl<PacketXd>(detail::vector_indices<PacketXd>{}));
838 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
850template <std::
size_t Group, std::
size_t Size>
851constexpr std::size_t zip_index(std::size_t i, std::size_t first_group) {
852 return Size * ((i / Group) % 2) + Group * (first_group + i / Group / 2) + i % Group;
857template <std::size_t Group,
typename VectorT, std::size_t... Is>
858EIGEN_ALWAYS_INLINE
void zip_in_place_impl(VectorT& p1, VectorT& p2, std::index_sequence<Is...>) {
859 constexpr std::size_t kSize =
sizeof...(Is);
862 static_assert(kSize >= 2 * Group,
"zip_in_place needs at least two lane groups per vector");
863 const VectorT tmp = __builtin_shufflevector(p1, p2, zip_index<Group, kSize>(Is, 0)...);
864 p2 = __builtin_shufflevector(p1, p2, zip_index<Group, kSize>(Is, kSize / (2 * Group))...);
870template <
typename Packet>
871EIGEN_ALWAYS_INLINE
void zip_in_place(Packet& p1, Packet& p2) {
872 zip_in_place_impl<1>(p1, p2, vector_indices<Packet>{});
875template <
typename Packet>
876EIGEN_ALWAYS_INLINE
void ptranspose_impl(PacketBlock<Packet, 2>& kernel) {
877 zip_in_place(kernel.packet[0], kernel.packet[1]);
880template <
typename Packet>
881EIGEN_ALWAYS_INLINE
void ptranspose_impl(PacketBlock<Packet, 4>& kernel) {
882 zip_in_place(kernel.packet[0], kernel.packet[2]);
883 zip_in_place(kernel.packet[1], kernel.packet[3]);
884 zip_in_place(kernel.packet[0], kernel.packet[1]);
885 zip_in_place(kernel.packet[2], kernel.packet[3]);
888template <
typename Packet>
889EIGEN_ALWAYS_INLINE
void ptranspose_impl(PacketBlock<Packet, 8>& kernel) {
890 zip_in_place(kernel.packet[0], kernel.packet[4]);
891 zip_in_place(kernel.packet[1], kernel.packet[5]);
892 zip_in_place(kernel.packet[2], kernel.packet[6]);
893 zip_in_place(kernel.packet[3], kernel.packet[7]);
895 zip_in_place(kernel.packet[0], kernel.packet[2]);
896 zip_in_place(kernel.packet[1], kernel.packet[3]);
897 zip_in_place(kernel.packet[4], kernel.packet[6]);
898 zip_in_place(kernel.packet[5], kernel.packet[7]);
900 zip_in_place(kernel.packet[0], kernel.packet[1]);
901 zip_in_place(kernel.packet[2], kernel.packet[3]);
902 zip_in_place(kernel.packet[4], kernel.packet[5]);
903 zip_in_place(kernel.packet[6], kernel.packet[7]);
906template <
typename Packet>
907EIGEN_ALWAYS_INLINE
void ptranspose_impl(PacketBlock<Packet, 16>& kernel) {
909 for (
int i = 0; i < 4; ++i) {
910 const int m = (1 << i);
912 for (
int j = 0; j < m; ++j) {
913 const int n = (1 << (3 - i));
915 for (
int k = 0; k < n; ++k) {
916 const int idx = 2 * j * n + k;
917 zip_in_place(kernel.packet[idx], kernel.packet[idx + n]);
931EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXf, 4>& kernel) {
932 detail::ptranspose_impl(kernel);
934EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXf, 2>& kernel) {
935 detail::ptranspose_impl(kernel);
939EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXd, 2>& kernel) {
940 detail::ptranspose_impl(kernel);
944EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXi, 4>& kernel) {
945 detail::ptranspose_impl(kernel);
947EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXi, 2>& kernel) {
948 detail::ptranspose_impl(kernel);
952EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXl, 2>& kernel) {
953 detail::ptranspose_impl(kernel);
956#if EIGEN_GENERIC_VECTOR_SIZE_BYTES >= 32
958EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXf, 8>& kernel) {
959 detail::ptranspose_impl(kernel);
961EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXd, 4>& kernel) {
962 detail::ptranspose_impl(kernel);
964EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXi, 8>& kernel) {
965 detail::ptranspose_impl(kernel);
967EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXl, 4>& kernel) {
968 detail::ptranspose_impl(kernel);
972#if EIGEN_GENERIC_VECTOR_SIZE_BYTES >= 64
974EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXf, 16>& kernel) {
975 detail::ptranspose_impl(kernel);
977EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXd, 8>& kernel) {
978 detail::ptranspose_impl(kernel);
980EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXi, 16>& kernel) {
981 detail::ptranspose_impl(kernel);
983EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<PacketXl, 8>& kernel) {
984 detail::ptranspose_impl(kernel);