23#ifndef EIGEN_MALLOC_ALREADY_ALIGNED
34#if defined(__GLIBC__) && ((__GLIBC__ >= 2 && __GLIBC_MINOR__ >= 8) || __GLIBC__ > 2) && defined(__LP64__) && \
35 !defined(__SANITIZE_ADDRESS__) && (EIGEN_DEFAULT_ALIGN_BYTES == 16)
36#define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 1
38#define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 0
45#if defined(__FreeBSD__) && !(EIGEN_ARCH_ARM || EIGEN_ARCH_MIPS) && (EIGEN_DEFAULT_ALIGN_BYTES == 16)
46#define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 1
48#define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 0
51#if (EIGEN_OS_MAC && (EIGEN_DEFAULT_ALIGN_BYTES == 16)) || (EIGEN_OS_WIN64 && (EIGEN_DEFAULT_ALIGN_BYTES == 16)) || \
52 EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED || EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED
53#define EIGEN_MALLOC_ALREADY_ALIGNED 1
55#define EIGEN_MALLOC_ALREADY_ALIGNED 0
60#ifndef EIGEN_MALLOC_CHECK_THREAD_LOCAL
65#ifndef EIGEN_AVOID_THREAD_LOCAL
67#if !defined(EIGEN_GPU_COMPILE_PHASE)
68#define EIGEN_MALLOC_CHECK_THREAD_LOCAL thread_local
70#define EIGEN_MALLOC_CHECK_THREAD_LOCAL
74#define EIGEN_MALLOC_CHECK_THREAD_LOCAL
80#include "../InternalHeaderCheck.h"
91EIGEN_DEVICE_FUNC
inline void check_that_malloc_is_allowed() {
92 eigen_assert(
false &&
"heap allocation is forbidden (EIGEN_NO_MALLOC is defined)");
94EIGEN_DEVICE_FUNC
inline void check_that_free_is_allowed() {
95 eigen_assert(
false &&
"heap deallocation is forbidden (EIGEN_NO_MALLOC is defined)");
97#elif defined EIGEN_RUNTIME_NO_MALLOC
98EIGEN_DEVICE_FUNC
inline bool is_malloc_allowed_impl(
bool update,
bool new_value =
false) {
99 EIGEN_MALLOC_CHECK_THREAD_LOCAL
static bool value =
true;
100 if (update == 1) value = new_value;
103EIGEN_DEVICE_FUNC
inline bool is_malloc_allowed() {
return is_malloc_allowed_impl(
false); }
104EIGEN_DEVICE_FUNC
inline bool set_is_malloc_allowed(
bool new_value) {
return is_malloc_allowed_impl(
true, new_value); }
105EIGEN_DEVICE_FUNC
inline void check_that_malloc_is_allowed() {
106 eigen_assert(is_malloc_allowed() &&
107 "heap allocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and set_is_malloc_allowed is false)");
109EIGEN_DEVICE_FUNC
inline bool is_free_allowed_impl(
bool update,
bool new_value =
false) {
110 EIGEN_MALLOC_CHECK_THREAD_LOCAL
static bool value =
true;
111 if (update == 1) value = new_value;
114EIGEN_DEVICE_FUNC
inline bool is_free_allowed() {
return is_free_allowed_impl(
false); }
115EIGEN_DEVICE_FUNC
inline bool set_is_free_allowed(
bool new_value) {
return is_free_allowed_impl(
true, new_value); }
116EIGEN_DEVICE_FUNC
inline void check_that_free_is_allowed() {
117 eigen_assert(is_free_allowed() &&
118 "heap deallocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and set_is_free_allowed is false)");
121EIGEN_DEVICE_FUNC
inline void check_that_malloc_is_allowed() {}
122EIGEN_DEVICE_FUNC
inline void check_that_free_is_allowed() {}
125EIGEN_DEVICE_FUNC
inline void throw_std_bad_alloc() {
126#ifdef EIGEN_EXCEPTIONS
127 throw std::bad_alloc();
129 std::size_t huge =
static_cast<std::size_t
>(-1);
130#if defined(EIGEN_HIPCC)
142 void* unused = ::operator
new(huge);
143 EIGEN_UNUSED_VARIABLE(unused);
157EIGEN_DEVICE_FUNC
inline void* handmade_aligned_malloc(std::size_t size,
158 std::size_t alignment = EIGEN_DEFAULT_ALIGN_BYTES) {
159 eigen_assert(alignment >=
sizeof(
void*) && alignment <= 256 && (alignment & (alignment - 1)) == 0 &&
160 "Alignment must be at least sizeof(void*), less than or equal to 256, and a power of 2");
162 check_that_malloc_is_allowed();
164 if (size >
static_cast<std::size_t
>((std::numeric_limits<std::ptrdiff_t>::max)()) - alignment)
return nullptr;
165 EIGEN_USING_STD(malloc)
166 void* original = malloc(size + alignment);
167 if (original ==
nullptr)
return nullptr;
168 std::size_t offset = alignment - (
reinterpret_cast<std::size_t
>(original) & (alignment - 1));
169 void* aligned =
static_cast<void*
>(
static_cast<uint8_t*
>(original) + offset);
171 *(
static_cast<uint8_t*
>(aligned) - 1) =
static_cast<uint8_t
>(offset - 1);
176EIGEN_DEVICE_FUNC
inline void handmade_aligned_free(
void* ptr) {
177 if (ptr !=
nullptr) {
178 std::size_t offset =
static_cast<std::size_t
>(*(
static_cast<uint8_t*
>(ptr) - 1)) + 1;
179 void* original =
static_cast<void*
>(
static_cast<uint8_t*
>(ptr) - offset);
181 check_that_free_is_allowed();
182 EIGEN_USING_STD(free)
192EIGEN_DEVICE_FUNC
inline void* handmade_aligned_realloc(
void* ptr, std::size_t new_size, std::size_t old_size,
193 std::size_t alignment = EIGEN_DEFAULT_ALIGN_BYTES) {
194 if (ptr ==
nullptr)
return handmade_aligned_malloc(new_size, alignment);
195 std::size_t old_offset =
static_cast<std::size_t
>(*(
static_cast<uint8_t*
>(ptr) - 1)) + 1;
196 void* old_original =
static_cast<uint8_t*
>(ptr) - old_offset;
198 check_that_malloc_is_allowed();
200 if (new_size >
static_cast<std::size_t
>((std::numeric_limits<std::ptrdiff_t>::max)()) - alignment)
return nullptr;
201 EIGEN_USING_STD(realloc)
202 void* original = realloc(old_original, new_size + alignment);
203 if (original ==
nullptr)
return nullptr;
204 if (original == old_original)
return ptr;
205 std::size_t offset = alignment - (
reinterpret_cast<std::size_t
>(original) & (alignment - 1));
206 void* aligned =
static_cast<void*
>(
static_cast<uint8_t*
>(original) + offset);
207 if (offset != old_offset) {
208 const void* src =
static_cast<const void*
>(
static_cast<uint8_t*
>(original) + old_offset);
209 std::size_t count = (std::min)(new_size, old_size);
210 std::memmove(aligned, src, count);
213 *(
static_cast<uint8_t*
>(aligned) - 1) =
static_cast<uint8_t
>(offset - 1);
220EIGEN_DEVICE_FUNC
inline void* aligned_malloc(std::size_t size) {
221 if (size == 0)
return nullptr;
224#if (EIGEN_DEFAULT_ALIGN_BYTES == 0) || EIGEN_MALLOC_ALREADY_ALIGNED
226 check_that_malloc_is_allowed();
227 EIGEN_USING_STD(malloc)
228 result = malloc(size);
230#if EIGEN_DEFAULT_ALIGN_BYTES == 16
231 eigen_assert((size < 16 || (std::size_t(result) % 16) == 0) &&
232 "System's malloc returned an unaligned pointer. Compile with EIGEN_MALLOC_ALREADY_ALIGNED=0 to fallback "
233 "to handmade aligned memory allocator.");
236 result = handmade_aligned_malloc(size);
239 if (!result && size) throw_std_bad_alloc();
245EIGEN_DEVICE_FUNC
inline void aligned_free(
void* ptr) {
246#if (EIGEN_DEFAULT_ALIGN_BYTES == 0) || EIGEN_MALLOC_ALREADY_ALIGNED
248 if (ptr !=
nullptr) {
249 check_that_free_is_allowed();
250 EIGEN_USING_STD(free)
255 handmade_aligned_free(ptr);
264EIGEN_DEVICE_FUNC
inline void* aligned_realloc(
void* ptr, std::size_t new_size, std::size_t old_size) {
265 if (ptr ==
nullptr)
return aligned_malloc(new_size);
266 if (old_size == new_size)
return ptr;
273#if (EIGEN_DEFAULT_ALIGN_BYTES == 0) || EIGEN_MALLOC_ALREADY_ALIGNED
274 EIGEN_UNUSED_VARIABLE(old_size);
276 check_that_malloc_is_allowed();
277 EIGEN_USING_STD(realloc)
278 result = realloc(ptr, new_size);
280 result = handmade_aligned_realloc(ptr, new_size, old_size);
283 if (!result && new_size) throw_std_bad_alloc();
296EIGEN_DEVICE_FUNC
inline void* conditional_aligned_malloc(std::size_t size) {
297 return aligned_malloc(size);
301EIGEN_DEVICE_FUNC
inline void* conditional_aligned_malloc<false>(std::size_t size) {
302 if (size == 0)
return nullptr;
304 check_that_malloc_is_allowed();
305 EIGEN_USING_STD(malloc)
306 void* result = malloc(size);
308 if (!result && size) throw_std_bad_alloc();
314EIGEN_DEVICE_FUNC
inline void conditional_aligned_free(
void* ptr) {
319EIGEN_DEVICE_FUNC
inline void conditional_aligned_free<false>(
void* ptr) {
320 if (ptr !=
nullptr) {
321 check_that_free_is_allowed();
322 EIGEN_USING_STD(free)
328EIGEN_DEVICE_FUNC
inline void* conditional_aligned_realloc(
void* ptr, std::size_t new_size, std::size_t old_size) {
329 return aligned_realloc(ptr, new_size, old_size);
333EIGEN_DEVICE_FUNC
inline void* conditional_aligned_realloc<false>(
void* ptr, std::size_t new_size,
334 std::size_t old_size) {
335 if (ptr ==
nullptr)
return conditional_aligned_malloc<false>(new_size);
336 if (old_size == new_size)
return ptr;
338 conditional_aligned_free<false>(ptr);
342 check_that_malloc_is_allowed();
343 EIGEN_USING_STD(realloc)
344 void* result = realloc(ptr, new_size);
345 if (!result) throw_std_bad_alloc();
357EIGEN_DEVICE_FUNC
inline void destruct_elements_of_array(T* ptr, std::size_t size) {
360 while (size) ptr[--size].~T();
367EIGEN_DEVICE_FUNC
inline T* default_construct_elements_of_array(T* ptr, std::size_t size) {
370 for (i = 0; i < size; ++i) ::new (ptr + i) T;
373 destruct_elements_of_array(ptr, i);
383EIGEN_DEVICE_FUNC
inline T* copy_construct_elements_of_array(T* ptr,
const T* src, std::size_t size) {
386 for (i = 0; i < size; ++i) ::new (ptr + i) T(*(src + i));
389 destruct_elements_of_array(ptr, i);
399EIGEN_DEVICE_FUNC
inline T* move_construct_elements_of_array(T* ptr, T* src, std::size_t size) {
402 for (i = 0; i < size; ++i) ::new (ptr + i) T(std::move(*(src + i)));
405 destruct_elements_of_array(ptr, i);
416EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void check_size_for_overflow(std::size_t size) {
417 constexpr std::size_t max_elements = (std::numeric_limits<std::ptrdiff_t>::max)() /
sizeof(T);
418 if (size > max_elements) throw_std_bad_alloc();
426EIGEN_DEVICE_FUNC
inline T* aligned_new(std::size_t size) {
427 check_size_for_overflow<T>(size);
428 T* result =
static_cast<T*
>(aligned_malloc(
sizeof(T) * size));
429 EIGEN_TRY { default_construct_elements_of_array(result, size); }
431 aligned_free(result);
437template <
typename T,
bool Align>
438EIGEN_DEVICE_FUNC
inline T* conditional_aligned_new(std::size_t size) {
439 check_size_for_overflow<T>(size);
440 T* result =
static_cast<T*
>(conditional_aligned_malloc<Align>(
sizeof(T) * size));
441 EIGEN_TRY { default_construct_elements_of_array(result, size); }
443 conditional_aligned_free<Align>(result);
453EIGEN_DEVICE_FUNC
inline void aligned_delete(T* ptr, std::size_t size) {
454 destruct_elements_of_array<T>(ptr, size);
461template <
typename T,
bool Align>
462EIGEN_DEVICE_FUNC
inline void conditional_aligned_delete(T* ptr, std::size_t size) {
463 destruct_elements_of_array<T>(ptr, size);
464 conditional_aligned_free<Align>(ptr);
467template <
typename T,
bool Align>
468EIGEN_DEVICE_FUNC
inline T* conditional_aligned_realloc_new(T* pts, std::size_t new_size, std::size_t old_size) {
469 check_size_for_overflow<T>(new_size);
470 check_size_for_overflow<T>(old_size);
476 T* result =
static_cast<T*
>(conditional_aligned_malloc<Align>(
sizeof(T) * new_size));
477 const std::size_t copy_size = (std::min)(old_size, new_size);
478 bool tail_constructed =
false;
481 if (new_size > copy_size) {
482 default_construct_elements_of_array(result + copy_size, new_size - copy_size);
483 tail_constructed =
true;
485 move_construct_elements_of_array(result, pts, copy_size);
488 conditional_aligned_delete<T, Align>(pts, old_size);
492 if (tail_constructed) destruct_elements_of_array(result + copy_size, new_size - copy_size);
493 conditional_aligned_free<Align>(result);
500template <
typename T,
bool Align>
501EIGEN_DEVICE_FUNC
inline T* conditional_aligned_new_auto(std::size_t size) {
502 if (size == 0)
return nullptr;
503 check_size_for_overflow<T>(size);
504 T* result =
static_cast<T*
>(conditional_aligned_malloc<Align>(
sizeof(T) * size));
505 EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
506 EIGEN_TRY { default_construct_elements_of_array(result, size); }
508 conditional_aligned_free<Align>(result);
515template <
typename T,
bool Align>
516EIGEN_DEVICE_FUNC
inline T* conditional_aligned_realloc_new_auto(T* pts, std::size_t new_size, std::size_t old_size) {
517 EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
518 return conditional_aligned_realloc_new<T, Align>(pts, new_size, old_size);
521 check_size_for_overflow<T>(new_size);
522 check_size_for_overflow<T>(old_size);
523 EIGEN_IF_CONSTEXPR (std::is_trivially_copyable<T>::value) {
524 return static_cast<T*
>(
525 conditional_aligned_realloc<Align>(
static_cast<void*
>(pts),
sizeof(T) * new_size,
sizeof(T) * old_size));
532 if (new_size == old_size)
return pts;
533 T* result =
static_cast<T*
>(conditional_aligned_malloc<Align>(
sizeof(T) * new_size));
534 EIGEN_TRY { move_construct_elements_of_array(result, pts, (std::min)(old_size, new_size)); }
536 conditional_aligned_free<Align>(result);
539 conditional_aligned_free<Align>(pts);
543template <
typename T,
bool Align>
544EIGEN_DEVICE_FUNC
inline void conditional_aligned_delete_auto(T* ptr, std::size_t size) {
545 EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) destruct_elements_of_array<T>(ptr, size);
546 conditional_aligned_free<Align>(ptr);
569template <
int Alignment,
typename Scalar,
typename Index>
570EIGEN_DEVICE_FUNC
inline Index first_aligned(
const Scalar* array, Index size) {
571 constexpr Index ScalarSize =
sizeof(Scalar);
572 constexpr Index AlignmentSize = Alignment / ScalarSize;
573 constexpr Index AlignmentMask = AlignmentSize - 1;
575 EIGEN_IF_CONSTEXPR (Alignment <=
alignof(Scalar)) {
578 }
else EIGEN_IF_CONSTEXPR (Alignment <= ScalarSize) {
580 return (std::uintptr_t(array) & (std::uintptr_t(Alignment) - 1)) ? size : 0;
581 }
else EIGEN_IF_CONSTEXPR ((Alignment % ScalarSize) != 0) {
585 }
else if (std::uintptr_t(array) & (
sizeof(Scalar) - 1)) {
589 Index first = (AlignmentSize - (Index((std::uintptr_t(array) /
sizeof(Scalar))) & AlignmentMask)) & AlignmentMask;
590 return (first < size) ? first : size;
596template <
typename Scalar,
typename Index>
597EIGEN_DEVICE_FUNC
inline Index first_default_aligned(
const Scalar* array, Index size) {
598 using DefaultPacketType =
typename packet_traits<Scalar>::type;
599 return first_aligned<unpacket_traits<DefaultPacketType>::alignment>(array, size);
604template <
typename Index>
605inline Index first_multiple(Index size, Index base) {
606 return ((size + base - 1) / base) * base;
612template <
typename T,
bool UseMemcpy>
613struct smart_copy_helper;
616EIGEN_DEVICE_FUNC
void smart_copy(
const T* start,
const T* end, T* target) {
617 smart_copy_helper<T, std::is_trivially_copyable<T>::value>::run(start, end, target);
621struct smart_copy_helper<T, true> {
622 EIGEN_DEVICE_FUNC
static inline void run(
const T* start,
const T* end, T* target) {
623 std::ptrdiff_t count = end - start;
624 if (count <= 0)
return;
625 eigen_internal_assert(start != 0 && end != 0 && target != 0);
626 EIGEN_USING_STD(memcpy)
627 memcpy(target, start,
static_cast<std::size_t
>(count) *
sizeof(T));
633struct smart_copy_helper<T, false> {
634 EIGEN_DEVICE_FUNC
static inline void run(
const T* EIGEN_RESTRICT start,
const T* end, T* EIGEN_RESTRICT target) {
635 std::copy(start, end, target);
642constexpr std::conditional_t<std::is_move_assignable<T>::value, T&&,
const T&> move_if_assignable(T& x)
noexcept {
648template <typename T, bool Move = std::is_move_assignable<T>::value>
649struct move_or_copy_helper {
650 static void run(T* start, T* end, T* target) { std::move(start, end, target); }
651 static void run_backward(T* start, T* end, T* target_end) { std::move_backward(start, end, target_end); }
655struct move_or_copy_helper<T, false> {
656 static void run(T* start, T* end, T* target) { std::copy(start, end, target); }
657 static void run_backward(T* start, T* end, T* target_end) { std::copy_backward(start, end, target_end); }
661inline void move_or_copy(T* start, T* end, T* target) {
662 move_or_copy_helper<T>::run(start, end, target);
666inline void move_or_copy_backward(T* start, T* end, T* target_end) {
667 move_or_copy_helper<T>::run_backward(start, end, target_end);
676#if !defined EIGEN_ALLOCA && !defined EIGEN_GPU_COMPILE_PHASE
677#if EIGEN_OS_LINUX || EIGEN_OS_MAC || (defined alloca)
678#define EIGEN_ALLOCA alloca
680#define EIGEN_ALLOCA _alloca
689#if defined(__clang__) && defined(__thumb__)
695#ifndef EIGEN_STACK_ALIGN_BYTES
696#if defined(EIGEN_VECTORIZE_SME) && EIGEN_DEFAULT_ALIGN_BYTES < 64
697#define EIGEN_STACK_ALIGN_BYTES 64
699#define EIGEN_STACK_ALIGN_BYTES EIGEN_DEFAULT_ALIGN_BYTES
704EIGEN_DEVICE_FUNC
inline void* scratch_malloc(std::size_t size) {
705#if EIGEN_STACK_ALIGN_BYTES > EIGEN_DEFAULT_ALIGN_BYTES
706 void* result = handmade_aligned_malloc(size, EIGEN_STACK_ALIGN_BYTES);
707 if (!result && size) throw_std_bad_alloc();
710 return aligned_malloc(size);
713EIGEN_DEVICE_FUNC
inline void scratch_free(
void* ptr) {
714#if EIGEN_STACK_ALIGN_BYTES > EIGEN_DEFAULT_ALIGN_BYTES
715 handmade_aligned_free(ptr);
723EIGEN_DEVICE_FUNC
inline T* scratch_malloc(std::size_t size) {
724 EIGEN_IF_CONSTEXPR (
alignof(T) <= EIGEN_STACK_ALIGN_BYTES ||
725 (EIGEN_STACK_ALIGN_BYTES == 0 &&
alignof(T) <=
alignof(std::max_align_t))) {
726 return static_cast<T*
>(scratch_malloc(size));
729 constexpr std::size_t overhead =
alignof(T) - 1 +
sizeof(
void*);
731 if (size < overhead) throw_std_bad_alloc();
732 check_that_malloc_is_allowed();
733 void* original = std::malloc(size);
734 if (!original) throw_std_bad_alloc();
736 char* start =
static_cast<char*
>(original) +
sizeof(
void*);
737 const std::uintptr_t offset =
reinterpret_cast<std::uintptr_t
>(start) %
alignof(T);
738 const std::uintptr_t shift = offset > 0 ?
alignof(T) - offset : 0;
739 void* aligned = start + shift;
740 std::memcpy(
static_cast<char*
>(aligned) -
sizeof(
void*),
reinterpret_cast<const void*
>(&original),
sizeof(
void*));
741 return static_cast<T*
>(aligned);
746EIGEN_DEVICE_FUNC
inline void scratch_free(
void* ptr) {
747 EIGEN_IF_CONSTEXPR (
alignof(T) <= EIGEN_STACK_ALIGN_BYTES ||
748 (EIGEN_STACK_ALIGN_BYTES == 0 &&
alignof(T) <=
alignof(std::max_align_t))) {
752 std::memcpy(
reinterpret_cast<void*
>(&original),
static_cast<char*
>(ptr) -
sizeof(
void*),
sizeof(
void*));
753 check_that_free_is_allowed();
760EIGEN_DEVICE_FUNC
inline T* scratch_new(std::size_t size) {
761 check_size_for_overflow<T>(size);
762 T* result = scratch_malloc<T>(
sizeof(T) * size);
763 EIGEN_TRY { default_construct_elements_of_array(result, size); }
765 scratch_free<T>(result);
773EIGEN_DEVICE_FUNC
inline void scratch_delete(T* ptr, std::size_t size) {
774 destruct_elements_of_array<T>(ptr, size);
775 scratch_free<T>(ptr);
781class aligned_stack_memory_handler {
783 aligned_stack_memory_handler(
const aligned_stack_memory_handler&) =
delete;
784 aligned_stack_memory_handler& operator=(
const aligned_stack_memory_handler&) =
delete;
792 EIGEN_DEVICE_FUNC aligned_stack_memory_handler(T* ptr, std::size_t size,
bool dealloc)
793 : m_ptr(ptr), m_size(size), m_deallocate(dealloc) {
794 EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
795 if (m_ptr) Eigen::internal::default_construct_elements_of_array(m_ptr, size);
798 EIGEN_DEVICE_FUNC ~aligned_stack_memory_handler() {
799 EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
800 if (m_ptr) Eigen::internal::destruct_elements_of_array<T>(m_ptr, m_size);
802 if (m_deallocate) Eigen::internal::scratch_free<T>(m_ptr);
813template <
typename Xpr,
int NbEvaluations,
814 bool MapExternalBuffer = nested_eval<Xpr, NbEvaluations>::Evaluate && Xpr::MaxSizeAtCompileTime == Dynamic>
815struct local_nested_eval_wrapper {
816 static constexpr bool NeedExternalBuffer =
false;
817 using Scalar =
typename Xpr::Scalar;
818 using ObjectType =
typename nested_eval<Xpr, NbEvaluations>::type;
821 EIGEN_DEVICE_FUNC local_nested_eval_wrapper(
const Xpr& xpr, Scalar* ptr) : object(xpr) {
822 EIGEN_UNUSED_VARIABLE(ptr);
823 eigen_internal_assert(ptr == 0);
827template <
typename Xpr,
int NbEvaluations>
828struct local_nested_eval_wrapper<Xpr, NbEvaluations, true> {
829 static constexpr bool NeedExternalBuffer =
true;
830 using Scalar =
typename Xpr::Scalar;
831 using PlainObject =
typename plain_object_eval<Xpr>::type;
832 using ObjectType = Map<PlainObject, EIGEN_DEFAULT_ALIGN_BYTES>;
835 EIGEN_DEVICE_FUNC local_nested_eval_wrapper(
const Xpr& xpr, Scalar* ptr)
836 : object(ptr == 0 ? Eigen::internal::scratch_malloc<Scalar>(sizeof(Scalar) * xpr.size()) : ptr, xpr.rows(),
838 m_deallocate(ptr == 0) {
839 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
840 if (
object.data()) Eigen::internal::default_construct_elements_of_array(
object.data(),
object.size());
845 EIGEN_DEVICE_FUNC ~local_nested_eval_wrapper() {
846 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
847 if (
object.data()) Eigen::internal::destruct_elements_of_array(
object.data(),
object.size());
849 if (m_deallocate) Eigen::internal::scratch_free<Scalar>(
object.data());
886#if defined(EIGEN_ALLOCA) && !defined(EIGEN_NO_ALLOCA)
888#if EIGEN_STACK_ALIGN_BYTES > 0
892#if ((EIGEN_COMP_GNUC || EIGEN_COMP_CLANG) && !EIGEN_COMP_NVHPC && !EIGEN_COMP_ICC)
893#define EIGEN_ALIGNED_ALLOCA(SIZE) __builtin_alloca_with_align(SIZE, CHAR_BIT* EIGEN_STACK_ALIGN_BYTES)
895EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void* eigen_aligned_alloca_helper(
void* ptr) {
896 constexpr std::uintptr_t mask = EIGEN_STACK_ALIGN_BYTES - 1;
897 std::uintptr_t ptr_int = std::uintptr_t(ptr);
898 std::uintptr_t aligned_ptr_int = (ptr_int + mask) & ~mask;
899 std::uintptr_t offset = aligned_ptr_int - ptr_int;
900 return static_cast<void*
>(
static_cast<uint8_t*
>(ptr) + offset);
902#define EIGEN_ALIGNED_ALLOCA(SIZE) eigen_aligned_alloca_helper(EIGEN_ALLOCA(SIZE + EIGEN_STACK_ALIGN_BYTES - 1))
906#define EIGEN_ALIGNED_ALLOCA(SIZE) EIGEN_ALLOCA(SIZE)
910#define EIGEN_TYPED_ALIGNED_ALLOCA(TYPE, SIZE) \
911 (alignof(TYPE) <= EIGEN_STACK_ALIGN_BYTES \
912 ? EIGEN_ALIGNED_ALLOCA(SIZE) \
913 : reinterpret_cast<void*>( \
914 (reinterpret_cast<std::uintptr_t>(EIGEN_ALLOCA((SIZE) + alignof(TYPE) - 1)) + alignof(TYPE) - 1) & \
915 ~(std::uintptr_t(alignof(TYPE)) - 1)))
919#define ei_declare_aligned_stack_constructed_variable(TYPE, NAME, SIZE, BUFFER) \
920 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
921 TYPE* NAME = (BUFFER) != 0 \
923 : reinterpret_cast<TYPE*>((sizeof(TYPE) * (SIZE) <= EIGEN_STACK_ALLOCATION_LIMIT) \
924 ? EIGEN_TYPED_ALIGNED_ALLOCA(TYPE, sizeof(TYPE) * (SIZE)) \
925 : Eigen::internal::scratch_malloc<TYPE>(sizeof(TYPE) * (SIZE))); \
926 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME, _stack_memory_destructor)( \
927 (BUFFER) == 0 ? NAME : 0, SIZE, sizeof(TYPE) * (SIZE) > EIGEN_STACK_ALLOCATION_LIMIT)
930#define ei_declare_local_nested_eval(XPR_T, XPR, N, NAME) \
931 Eigen::internal::local_nested_eval_wrapper<XPR_T, N> EIGEN_CAT(NAME, _wrapper)( \
932 XPR, reinterpret_cast<typename XPR_T::Scalar*>( \
933 ((Eigen::internal::local_nested_eval_wrapper<XPR_T, N>::NeedExternalBuffer) && \
934 ((sizeof(typename XPR_T::Scalar) * (XPR).size()) <= EIGEN_STACK_ALLOCATION_LIMIT)) \
935 ? EIGEN_TYPED_ALIGNED_ALLOCA(typename XPR_T::Scalar, sizeof(typename XPR_T::Scalar) * (XPR).size()) \
937 typename Eigen::internal::local_nested_eval_wrapper<XPR_T, N>::ObjectType NAME(EIGEN_CAT(NAME, _wrapper).object)
943#define ei_declare_aligned_stack_constructed_variable(TYPE, NAME, SIZE, BUFFER) \
944 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
945 TYPE* NAME = (BUFFER) != 0 ? (BUFFER) : Eigen::internal::scratch_malloc<TYPE>(sizeof(TYPE) * (SIZE)); \
946 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME, _stack_memory_destructor)( \
947 (BUFFER) == 0 ? NAME : 0, SIZE, true)
950#define ei_declare_local_nested_eval(XPR_T, XPR, N, NAME) \
951 typename Eigen::internal::nested_eval<XPR_T, N>::type NAME(XPR)
959#if EIGEN_HAS_CXX17_OVERALIGN
963#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign)
964#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
965#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW
966#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar, Size)
971#if EIGEN_MAX_ALIGN_BYTES != 0 && !defined(EIGEN_HIP_DEVICE_COMPILE)
972#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
973 EIGEN_DEVICE_FUNC void* operator new(std::size_t size, const std::nothrow_t&) noexcept { \
974 EIGEN_TRY { return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); } \
975 EIGEN_CATCH(...) { return 0; } \
977 EIGEN_DEVICE_FUNC void* operator new[](std::size_t size, const std::nothrow_t&) noexcept { \
978 EIGEN_TRY { return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); } \
979 EIGEN_CATCH(...) { return 0; } \
981#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) \
982 EIGEN_DEVICE_FUNC void* operator new(std::size_t size) { \
983 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
985 EIGEN_DEVICE_FUNC void* operator new[](std::size_t size) { \
986 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
988 EIGEN_DEVICE_FUNC void operator delete(void* ptr) noexcept { \
989 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
991 EIGEN_DEVICE_FUNC void operator delete[](void* ptr) noexcept { \
992 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
994 EIGEN_DEVICE_FUNC void operator delete(void* ptr, std::size_t ) noexcept { \
995 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
997 EIGEN_DEVICE_FUNC void operator delete[](void* ptr, std::size_t ) noexcept { \
998 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
1003 EIGEN_DEVICE_FUNC static void* operator new(std::size_t size, void* ptr) { return ::operator new(size, ptr); } \
1004 EIGEN_DEVICE_FUNC static void* operator new[](std::size_t size, void* ptr) { return ::operator new[](size, ptr); } \
1005 EIGEN_DEVICE_FUNC void operator delete(void* memory, void* ptr) noexcept { return ::operator delete(memory, ptr); } \
1006 EIGEN_DEVICE_FUNC void operator delete[](void* memory, void* ptr) noexcept { \
1007 return ::operator delete[](memory, ptr); \
1010 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
1011 EIGEN_DEVICE_FUNC void operator delete(void* ptr, const std::nothrow_t&) noexcept { \
1012 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
1014 EIGEN_DEVICE_FUNC void operator delete[](void* ptr, const std::nothrow_t&) noexcept { \
1015 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
1017 using eigen_aligned_operator_new_marker_type = void;
1019#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
1022#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(true)
1023#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar, Size) \
1024 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF( \
1025 bool(((Size) != Eigen::Dynamic) && \
1026 (((EIGEN_MAX_ALIGN_BYTES >= 16) && ((sizeof(Scalar) * size_t(Size)) % (EIGEN_MAX_ALIGN_BYTES) == 0)) || \
1027 ((EIGEN_MAX_ALIGN_BYTES >= 32) && ((sizeof(Scalar) * size_t(Size)) % (EIGEN_MAX_ALIGN_BYTES / 2) == 0)) || \
1028 ((EIGEN_MAX_ALIGN_BYTES >= 64) && ((sizeof(Scalar) * size_t(Size)) % (EIGEN_MAX_ALIGN_BYTES / 4) == 0)))))
1059class aligned_allocator {
1061 using size_type = std::size_t;
1062 using difference_type = std::ptrdiff_t;
1064 using const_pointer =
const T*;
1065 using reference = T&;
1066 using const_reference =
const T&;
1067 using value_type = T;
1071 using other = aligned_allocator<U>;
1074 aligned_allocator() =
default;
1076 aligned_allocator(
const aligned_allocator&) =
default;
1079 aligned_allocator(
const aligned_allocator<U>&) {}
1082 constexpr bool operator==(
const aligned_allocator<U>&)
const noexcept {
1086 constexpr bool operator!=(
const aligned_allocator<U>&)
const noexcept {
1090#if EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_STRICT_AT_LEAST(7, 0, 0)
1094 size_type max_size()
const {
return (std::numeric_limits<std::ptrdiff_t>::max)() /
sizeof(T); }
1097 pointer allocate(size_type num,
const void* =
nullptr) {
1098 internal::check_size_for_overflow<T>(num);
1099 return static_cast<pointer
>(internal::aligned_malloc(num *
sizeof(T)));
1102 void deallocate(pointer p, size_type ) { internal::aligned_free(p); }
1107#if EIGEN_OS_LINUX && !defined(EIGEN_NO_CPU_CACHE_SYSFS)
1109#define EIGEN_CPU_CACHE_SYSFS 1
1112#if !defined(EIGEN_NO_CPUID)
1113#if EIGEN_COMP_GNUC && EIGEN_ARCH_i386_OR_x86_64
1114#if defined(__PIC__) && EIGEN_ARCH_i386
1116#define EIGEN_CPUID(abcd, func, id) \
1117 __asm__ __volatile__("xchgl %%ebx, %k1;cpuid; xchgl %%ebx,%k1" \
1118 : "=a"(abcd[0]), "=&r"(abcd[1]), "=c"(abcd[2]), "=d"(abcd[3]) \
1119 : "a"(func), "c"(id));
1120#elif defined(__PIC__) && EIGEN_ARCH_x86_64
1124#define EIGEN_CPUID(abcd, func, id) \
1125 __asm__ __volatile__("xchg{q}\t{%%}rbx, %q1; cpuid; xchg{q}\t{%%}rbx, %q1" \
1126 : "=a"(abcd[0]), "=&r"(abcd[1]), "=c"(abcd[2]), "=d"(abcd[3]) \
1127 : "0"(func), "2"(id));
1130#define EIGEN_CPUID(abcd, func, id) \
1131 __asm__ __volatile__("cpuid" : "=a"(abcd[0]), "=b"(abcd[1]), "=c"(abcd[2]), "=d"(abcd[3]) : "0"(func), "2"(id));
1133#elif EIGEN_COMP_MSVC
1134#if EIGEN_ARCH_i386_OR_x86_64
1135#define EIGEN_CPUID(abcd, func, id) __cpuidex((int*)abcd, func, id)
1144inline bool cpuid_is_vendor(
int abcd[4],
const int vendor[3]) {
1145 return abcd[1] == vendor[0] && abcd[3] == vendor[1] && abcd[2] == vendor[2];
1148inline void queryCacheSizes_intel_direct(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3) {
1154 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1155 EIGEN_CPUID(abcd, 0x4, cache_id);
1156 cache_type = (abcd[0] & 0x0F) >> 0;
1157 if (cache_type == 1 || cache_type == 3)
1159 int cache_level = (abcd[0] & 0xE0) >> 5;
1160 int ways = (abcd[1] & 0xFFC00000) >> 22;
1161 int partitions = (abcd[1] & 0x003FF000) >> 12;
1162 int line_size = (abcd[1] & 0x00000FFF) >> 0;
1163 int sets = (abcd[2]);
1165 std::ptrdiff_t cache_size =
1166 static_cast<std::ptrdiff_t
>(ways + 1) * (partitions + 1) * (line_size + 1) * (sets + 1);
1168 switch (cache_level) {
1183 }
while (cache_type > 0 && cache_id < 16);
1186inline void queryCacheSizes_intel_codes(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3) {
1188 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1190 EIGEN_CPUID(abcd, 0x00000002, 0);
1191 unsigned char* bytes =
reinterpret_cast<unsigned char*
>(abcd) + 2;
1192 bool check_for_p2_core2 =
false;
1193 for (
int i = 0; i < 14; ++i) {
1292 check_for_p2_core2 =
true;
1376 if (check_for_p2_core2 && l2 == l3) l3 = 0;
1382inline void queryCacheSizes_intel(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3,
int max_std_funcs) {
1383 if (max_std_funcs >= 4)
1384 queryCacheSizes_intel_direct(l1, l2, l3);
1385 else if (max_std_funcs >= 2)
1386 queryCacheSizes_intel_codes(l1, l2, l3);
1391inline void queryCacheSizes_amd(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3) {
1393 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1396 EIGEN_CPUID(abcd, 0x80000000, 0);
1397 if (
static_cast<numext::uint32_t
>(abcd[0]) >=
static_cast<numext::uint32_t
>(0x80000006)) {
1398 EIGEN_CPUID(abcd, 0x80000005, 0);
1399 l1 = (abcd[2] >> 24) * 1024;
1400 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1401 EIGEN_CPUID(abcd, 0x80000006, 0);
1402 l2 = (abcd[2] >> 16) * 1024;
1403 l3 =
static_cast<std::ptrdiff_t
>((abcd[3] & 0xFFFC000) >> 18) * 512 * 1024;
1410#ifdef EIGEN_CPU_CACHE_SYSFS
1413struct CpuCacheTopology {
1414 std::ptrdiff_t l1 = 0;
1415 std::ptrdiff_t l2 = 0;
1416 std::ptrdiff_t l3 = 0;
1417 std::ptrdiff_t l3_per_cpu = 0;
1423inline bool readSysfsLine(
const char* root,
const char* relative,
char (&value)[Size]) {
1425 const int length = std::snprintf(path,
sizeof(path),
"%s/%s", root, relative);
1426 if (length <= 0 || length >=
static_cast<int>(
sizeof(path)))
return false;
1427 const int fd = ::open(path, O_RDONLY | O_CLOEXEC);
1428 if (fd < 0)
return false;
1429 const ssize_t bytes_read = ::read(fd, value, Size - 1);
1431 if (bytes_read <= 0)
return false;
1432 value[bytes_read] =
'\0';
1433 char* newline = std::strchr(value,
'\n');
1434 if (newline !=
nullptr) {
1442inline bool readCpuCacheAttribute(
const char* root,
int cpu,
int index,
const char* name,
char (&value)[Size]) {
1444 const int length = std::snprintf(relative,
sizeof(relative),
"cpu%d/cache/index%d/%s", cpu, index, name);
1445 return length > 0 && length < static_cast<int>(
sizeof(relative)) && readSysfsLine(root, relative, value);
1449inline std::ptrdiff_t parseCpuCacheSize(
const char* text) {
1450 char* suffix =
nullptr;
1451 const std::ptrdiff_t value = std::strtol(text, &suffix, 10);
1452 if (value <= 0)
return 0;
1455 const std::ptrdiff_t multiplier = (*suffix ==
'K' || *suffix ==
'k') ? 1024
1456 : (*suffix ==
'M' || *suffix ==
'm') ? 1024 * 1024
1458 return value * multiplier;
1462inline bool isCpuListTerminator(
const char* text) {
return *text ==
'\0' || *text ==
'\n' || *text ==
'\r'; }
1468template <
typename Visitor>
1469inline bool parseCpuList(
const char* text, Visitor&& visit) {
1470 if (isCpuListTerminator(text))
return false;
1473 const long max_cpu_id = 1 << 16;
1474 const char* cursor = text;
1476 char* end =
nullptr;
1477 const long first = std::strtol(cursor, &end, 10);
1478 if (end == cursor || first < 0)
return false;
1482 last = std::strtol(cursor, &end, 10);
1483 if (end == cursor || last < first)
return false;
1485 if (last >= max_cpu_id)
return false;
1486 visit(
static_cast<int>(first),
static_cast<int>(last));
1495 return isCpuListTerminator(end);
1502inline int parseCpuListCount(
const char* text) {
1504 return parseCpuList(text, [&count](
int first,
int last) { count += last - first + 1; }) ? count : 0;
1514inline CpuCacheTopology readCpuCacheTopologySysfs(
const char* root,
int cpu) {
1515 CpuCacheTopology topology;
1518 for (
int index = 0; index < 16; ++index) {
1523 if (!readCpuCacheAttribute(root, cpu, index,
"level", value))
break;
1524 const long level = std::strtol(value,
nullptr, 10);
1525 std::ptrdiff_t* target =
1526 level == 1 ? &topology.l1 : (level == 2 ? &topology.l2 : (level == 3 ? &topology.l3 :
nullptr));
1527 if (target ==
nullptr || *target > 0)
continue;
1531 if (!readCpuCacheAttribute(root, cpu, index,
"type", value) || value[0] ==
'I')
continue;
1533 if (!readCpuCacheAttribute(root, cpu, index,
"size", value))
continue;
1534 const std::ptrdiff_t size = parseCpuCacheSize(value);
1535 if (size <= 0)
continue;
1538 if (level == 3 && readCpuCacheAttribute(root, cpu, index,
"shared_cpu_list", value) &&
1539 std::strchr(value,
'\n') !=
nullptr) {
1540 const int sharing = parseCpuListCount(value);
1541 if (sharing > 0) topology.l3_per_cpu = size / sharing;
1548inline std::ptrdiff_t smallerReportedCacheSize(std::ptrdiff_t a, std::ptrdiff_t b) {
1549 return a == 0 ? b : (b == 0 ? a : (std::min)(a, b));
1557template <
typename IsAllowed>
1558inline CpuCacheTopology queryCpuCacheTopologySysfs(
const char* root, IsAllowed&& is_allowed) {
1559 CpuCacheTopology topology;
1561 if (!readSysfsLine(root,
"online", online) || std::strchr(online,
'\n') ==
nullptr)
return topology;
1563 if (!parseCpuList(online, [](
int,
int) {}))
return topology;
1564 parseCpuList(online, [&](
int first,
int last) {
1565 for (
int cpu = first; cpu <= last; ++cpu) {
1566 if (!is_allowed(cpu))
continue;
1567 const CpuCacheTopology candidate = readCpuCacheTopologySysfs(root, cpu);
1568 topology.l1 = smallerReportedCacheSize(topology.l1, candidate.l1);
1569 topology.l2 = smallerReportedCacheSize(topology.l2, candidate.l2);
1570 topology.l3 = smallerReportedCacheSize(topology.l3, candidate.l3);
1571 topology.l3_per_cpu = smallerReportedCacheSize(topology.l3_per_cpu, candidate.l3_per_cpu);
1583inline CpuCacheTopology queryCpuCacheTopologySysfs(
const char* root =
"/sys/devices/system/cpu") {
1589 if (sched_getaffinity(0,
sizeof(allowed), &allowed) == 0) {
1590 return queryCpuCacheTopologySysfs(root,
1591 [&allowed](
int cpu) {
return cpu < CPU_SETSIZE && CPU_ISSET(cpu, &allowed); });
1594 return queryCpuCacheTopologySysfs(root, [](
int) {
return true; });
1602inline void queryCacheSizes(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3, std::ptrdiff_t& l3_per_cpu) {
1606 const int GenuineIntel[] = {0x756e6547, 0x49656e69, 0x6c65746e};
1607 const int AuthenticAMD[] = {0x68747541, 0x69746e65, 0x444d4163};
1608 const int AMDisbetter_[] = {0x69444d41, 0x74656273, 0x21726574};
1611 const int HygonGenuine[] = {0x6f677948, 0x6e65476e, 0x656e6975};
1614 EIGEN_CPUID(abcd, 0x0, 0);
1615 int max_std_funcs = abcd[0];
1616 if (cpuid_is_vendor(abcd, GenuineIntel))
1617 queryCacheSizes_intel(l1, l2, l3, max_std_funcs);
1618 else if (cpuid_is_vendor(abcd, AuthenticAMD) || cpuid_is_vendor(abcd, AMDisbetter_) ||
1619 cpuid_is_vendor(abcd, HygonGenuine))
1620 queryCacheSizes_amd(l1, l2, l3);
1623 queryCacheSizes_intel(l1, l2, l3, max_std_funcs);
1646 std::size_t val_size =
sizeof(val);
1648 val_size =
sizeof(val);
1649 if (sysctlbyname(
"hw.perflevel0.l1dcachesize", &val, &val_size,
nullptr, 0) == 0 && val > 0)
1652 val_size =
sizeof(val);
1653 if (sysctlbyname(
"hw.l1dcachesize", &val, &val_size,
nullptr, 0) == 0) l1 = val;
1656 val_size =
sizeof(val);
1657 if (sysctlbyname(
"hw.l2cachesize", &val, &val_size,
nullptr, 0) == 0) l2 = val;
1659 val_size =
sizeof(val);
1660 if (sysctlbyname(
"hw.l3cachesize", &val, &val_size,
nullptr, 0) == 0 && val > 0) l3 = val;
1662#elif EIGEN_OS_UNIX && defined(_SC_LEVEL1_DCACHE_SIZE)
1664 l1 = sysconf(_SC_LEVEL1_DCACHE_SIZE);
1665 l2 = sysconf(_SC_LEVEL2_CACHE_SIZE);
1666 l3 = sysconf(_SC_LEVEL3_CACHE_SIZE);
1670#ifdef EIGEN_CPU_CACHE_SYSFS
1675 const CpuCacheTopology topology = queryCpuCacheTopologySysfs();
1676 if (l1 <= 0) l1 = topology.l1;
1677 if (l2 <= 0) l2 = topology.l2;
1678 if (l3 <= 0) l3 = topology.l3;
1679 l3_per_cpu = topology.l3_per_cpu;
1685inline void queryCacheSizes(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3) {
1686 std::ptrdiff_t l3_per_cpu;
1687 queryCacheSizes(l1, l2, l3, l3_per_cpu);
1692inline std::ptrdiff_t queryL1CacheSize() {
1693 std::ptrdiff_t l1(-1), l2, l3;
1694 queryCacheSizes(l1, l2, l3);
1700inline std::ptrdiff_t queryTopLevelCacheSize() {
1701 std::ptrdiff_t l1, l2(-1), l3(-1);
1702 queryCacheSizes(l1, l2, l3);
1703 return (std::max)(l2, l3);
1710#if !defined(EIGEN_GPU_COMPILE_PHASE) && EIGEN_COMP_CXXVER >= 20 && defined(__cpp_lib_constexpr_dynamic_alloc) && \
1711 __cpp_lib_constexpr_dynamic_alloc >= 201907L
1712using std::construct_at;
1714template <
class T,
class... Args>
1715EIGEN_DEVICE_FUNC T* construct_at(T* p, Args&&... args) {
1716 return ::new (
const_cast<void*
>(
static_cast<const volatile void*
>(p))) T(std::forward<Args>(args)...);
1725#if EIGEN_COMP_CXXVER >= 17
1726using std::destroy_at;
1729EIGEN_DEVICE_FUNC
void destroy_at(T* p) {
1735#if !defined(EIGEN_DONT_ASSUME_ALIGNED) && __has_feature(memory_sanitizer) && (EIGEN_ARCH_ARM || EIGEN_ARCH_ARM64)
1736#define EIGEN_DONT_ASSUME_ALIGNED
1739#if !defined(EIGEN_DONT_ASSUME_ALIGNED) && defined(__cpp_lib_assume_aligned) && (__cpp_lib_assume_aligned >= 201811L)
1740template <std::
size_t N,
typename T>
1741EIGEN_DEVICE_FUNC
constexpr T* assume_aligned(T* ptr) {
1742 return std::assume_aligned<N, T>(ptr);
1744#elif !defined(EIGEN_DONT_ASSUME_ALIGNED) && EIGEN_HAS_BUILTIN(__builtin_assume_aligned)
1745template <std::
size_t N,
typename T>
1746EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC T* assume_aligned(T* ptr) {
1747 return static_cast<T*
>(__builtin_assume_aligned(ptr, N));
1750template <std::
size_t N,
typename T>
1751EIGEN_DEVICE_FUNC
constexpr T* assume_aligned(T* ptr) {