Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
Memory.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2015 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Copyright (C) 2008-2009 Benoit Jacob <jacob.benoit.1@gmail.com>
6// Copyright (C) 2009 Kenneth Riddile <kfriddile@yahoo.com>
7// Copyright (C) 2010 Hauke Heibel <hauke.heibel@gmail.com>
8// Copyright (C) 2010 Thomas Capricelli <orzel@freehackers.org>
9// Copyright (C) 2013 Pavel Holoborodko <pavel@holoborodko.com>
10//
11// This Source Code Form is subject to the terms of the Mozilla
12// Public License v. 2.0. If a copy of the MPL was not distributed
13// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
14// SPDX-License-Identifier: MPL-2.0
15
16/*****************************************************************************
17*** Platform checks for aligned malloc functions ***
18*****************************************************************************/
19
20#ifndef EIGEN_MEMORY_H
21#define EIGEN_MEMORY_H
22
23#ifndef EIGEN_MALLOC_ALREADY_ALIGNED
24
25// Try to determine automatically if malloc is already aligned.
26
27// On 64-bit systems, glibc's malloc returns 16-byte-aligned pointers, see:
28// http://www.gnu.org/s/libc/manual/html_node/Aligned-Memory-Blocks.html
29// This is true at least since glibc 2.8.
30// This leaves the question how to detect 64-bit. According to this document,
31// http://gcc.fyxm.net/summit/2003/Porting%20to%2064%20bit.pdf
32// page 114, "[The] LP64 model [...] is used by all 64-bit UNIX ports" so it's indeed
33// quite safe, at least within the context of glibc, to equate 64-bit with LP64.
34#if defined(__GLIBC__) && ((__GLIBC__ >= 2 && __GLIBC_MINOR__ >= 8) || __GLIBC__ > 2) && defined(__LP64__) && \
35 !defined(__SANITIZE_ADDRESS__) && (EIGEN_DEFAULT_ALIGN_BYTES == 16)
36#define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 1
37#else
38#define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 0
39#endif
40
41// FreeBSD 6 seems to have 16-byte aligned malloc
42// See http://svn.freebsd.org/viewvc/base/stable/6/lib/libc/stdlib/malloc.c?view=markup
43// FreeBSD 7 seems to have 16-byte aligned malloc except on ARM and MIPS architectures
44// See http://svn.freebsd.org/viewvc/base/stable/7/lib/libc/stdlib/malloc.c?view=markup
45#if defined(__FreeBSD__) && !(EIGEN_ARCH_ARM || EIGEN_ARCH_MIPS) && (EIGEN_DEFAULT_ALIGN_BYTES == 16)
46#define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 1
47#else
48#define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 0
49#endif
50
51#if (EIGEN_OS_MAC && (EIGEN_DEFAULT_ALIGN_BYTES == 16)) || (EIGEN_OS_WIN64 && (EIGEN_DEFAULT_ALIGN_BYTES == 16)) || \
52 EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED || EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED
53#define EIGEN_MALLOC_ALREADY_ALIGNED 1
54#else
55#define EIGEN_MALLOC_ALREADY_ALIGNED 0
56#endif
57
58#endif
59
60#ifndef EIGEN_MALLOC_CHECK_THREAD_LOCAL
61
62// Check whether we can use the thread_local keyword to allow or disallow
63// allocating memory with per-thread granularity, by means of the
64// set_is_malloc_allowed() function.
65#ifndef EIGEN_AVOID_THREAD_LOCAL
66
67#if !defined(EIGEN_GPU_COMPILE_PHASE)
68#define EIGEN_MALLOC_CHECK_THREAD_LOCAL thread_local
69#else
70#define EIGEN_MALLOC_CHECK_THREAD_LOCAL
71#endif
72
73#else // EIGEN_AVOID_THREAD_LOCAL
74#define EIGEN_MALLOC_CHECK_THREAD_LOCAL
75#endif // EIGEN_AVOID_THREAD_LOCAL
76
77#endif
78
79// IWYU pragma: private
80#include "../InternalHeaderCheck.h"
81
82namespace Eigen {
83
84namespace internal {
85
86/*****************************************************************************
87*** Implementation of portable aligned versions of malloc/free/realloc ***
88*****************************************************************************/
89
90#ifdef EIGEN_NO_MALLOC
91EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() {
92 eigen_assert(false && "heap allocation is forbidden (EIGEN_NO_MALLOC is defined)");
93}
94EIGEN_DEVICE_FUNC inline void check_that_free_is_allowed() {
95 eigen_assert(false && "heap deallocation is forbidden (EIGEN_NO_MALLOC is defined)");
96}
97#elif defined EIGEN_RUNTIME_NO_MALLOC
98EIGEN_DEVICE_FUNC inline bool is_malloc_allowed_impl(bool update, bool new_value = false) {
99 EIGEN_MALLOC_CHECK_THREAD_LOCAL static bool value = true;
100 if (update == 1) value = new_value;
101 return value;
102}
103EIGEN_DEVICE_FUNC inline bool is_malloc_allowed() { return is_malloc_allowed_impl(false); }
104EIGEN_DEVICE_FUNC inline bool set_is_malloc_allowed(bool new_value) { return is_malloc_allowed_impl(true, new_value); }
105EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() {
106 eigen_assert(is_malloc_allowed() &&
107 "heap allocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and set_is_malloc_allowed is false)");
108}
109EIGEN_DEVICE_FUNC inline bool is_free_allowed_impl(bool update, bool new_value = false) {
110 EIGEN_MALLOC_CHECK_THREAD_LOCAL static bool value = true;
111 if (update == 1) value = new_value;
112 return value;
113}
114EIGEN_DEVICE_FUNC inline bool is_free_allowed() { return is_free_allowed_impl(false); }
115EIGEN_DEVICE_FUNC inline bool set_is_free_allowed(bool new_value) { return is_free_allowed_impl(true, new_value); }
116EIGEN_DEVICE_FUNC inline void check_that_free_is_allowed() {
117 eigen_assert(is_free_allowed() &&
118 "heap deallocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and set_is_free_allowed is false)");
119}
120#else
121EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() {}
122EIGEN_DEVICE_FUNC inline void check_that_free_is_allowed() {}
123#endif
124
125EIGEN_DEVICE_FUNC inline void throw_std_bad_alloc() {
126#ifdef EIGEN_EXCEPTIONS
127 throw std::bad_alloc();
128#else
129 std::size_t huge = static_cast<std::size_t>(-1);
130#if defined(EIGEN_HIPCC)
131 //
132 // calls to "::operator new" are to be treated as opaque function calls (i.e no inlining),
133 // and as a consequence the code in the #else block triggers the hipcc warning :
134 // "no overloaded function has restriction specifiers that are compatible with the ambient context"
135 //
136 // "throw_std_bad_alloc" has the EIGEN_DEVICE_FUNC attribute, so it seems that hipcc expects
137 // the same on "operator new"
138 // Reverting code back to the old version in this #if block for the hipcc compiler
139 //
140 new int[huge];
141#else
142 void* unused = ::operator new(huge);
143 EIGEN_UNUSED_VARIABLE(unused);
144#endif
145#endif
146}
147
148/*****************************************************************************
149*** Implementation of handmade aligned functions ***
150*****************************************************************************/
151
152/* ----- Hand made implementations of aligned malloc/free and realloc ----- */
153
157EIGEN_DEVICE_FUNC inline void* handmade_aligned_malloc(std::size_t size,
158 std::size_t alignment = EIGEN_DEFAULT_ALIGN_BYTES) {
159 eigen_assert(alignment >= sizeof(void*) && alignment <= 256 && (alignment & (alignment - 1)) == 0 &&
160 "Alignment must be at least sizeof(void*), less than or equal to 256, and a power of 2");
161
162 check_that_malloc_is_allowed();
163 // No object may exceed PTRDIFF_MAX bytes (malloc rejects larger ones), and size + alignment must not wrap.
164 if (size > static_cast<std::size_t>((std::numeric_limits<std::ptrdiff_t>::max)()) - alignment) return nullptr;
165 EIGEN_USING_STD(malloc)
166 void* original = malloc(size + alignment);
167 if (original == nullptr) return nullptr;
168 std::size_t offset = alignment - (reinterpret_cast<std::size_t>(original) & (alignment - 1));
169 void* aligned = static_cast<void*>(static_cast<uint8_t*>(original) + offset);
170 // Store offset - 1, since it is guaranteed to be at least 1.
171 *(static_cast<uint8_t*>(aligned) - 1) = static_cast<uint8_t>(offset - 1);
172 return aligned;
173}
174
176EIGEN_DEVICE_FUNC inline void handmade_aligned_free(void* ptr) {
177 if (ptr != nullptr) {
178 std::size_t offset = static_cast<std::size_t>(*(static_cast<uint8_t*>(ptr) - 1)) + 1;
179 void* original = static_cast<void*>(static_cast<uint8_t*>(ptr) - offset);
180
181 check_that_free_is_allowed();
182 EIGEN_USING_STD(free)
183 free(original);
184 }
185}
186
192EIGEN_DEVICE_FUNC inline void* handmade_aligned_realloc(void* ptr, std::size_t new_size, std::size_t old_size,
193 std::size_t alignment = EIGEN_DEFAULT_ALIGN_BYTES) {
194 if (ptr == nullptr) return handmade_aligned_malloc(new_size, alignment);
195 std::size_t old_offset = static_cast<std::size_t>(*(static_cast<uint8_t*>(ptr) - 1)) + 1;
196 void* old_original = static_cast<uint8_t*>(ptr) - old_offset;
197
198 check_that_malloc_is_allowed();
199 // Same bound as handmade_aligned_malloc; fail as realloc does, leaving the old block untouched.
200 if (new_size > static_cast<std::size_t>((std::numeric_limits<std::ptrdiff_t>::max)()) - alignment) return nullptr;
201 EIGEN_USING_STD(realloc)
202 void* original = realloc(old_original, new_size + alignment);
203 if (original == nullptr) return nullptr;
204 if (original == old_original) return ptr;
205 std::size_t offset = alignment - (reinterpret_cast<std::size_t>(original) & (alignment - 1));
206 void* aligned = static_cast<void*>(static_cast<uint8_t*>(original) + offset);
207 if (offset != old_offset) {
208 const void* src = static_cast<const void*>(static_cast<uint8_t*>(original) + old_offset);
209 std::size_t count = (std::min)(new_size, old_size);
210 std::memmove(aligned, src, count);
211 }
212 // Store offset - 1, since it is guaranteed to be at least 1.
213 *(static_cast<uint8_t*>(aligned) - 1) = static_cast<uint8_t>(offset - 1);
214 return aligned;
215}
216
220EIGEN_DEVICE_FUNC inline void* aligned_malloc(std::size_t size) {
221 if (size == 0) return nullptr;
222
223 void* result;
224#if (EIGEN_DEFAULT_ALIGN_BYTES == 0) || EIGEN_MALLOC_ALREADY_ALIGNED
225
226 check_that_malloc_is_allowed();
227 EIGEN_USING_STD(malloc)
228 result = malloc(size);
229
230#if EIGEN_DEFAULT_ALIGN_BYTES == 16
231 eigen_assert((size < 16 || (std::size_t(result) % 16) == 0) &&
232 "System's malloc returned an unaligned pointer. Compile with EIGEN_MALLOC_ALREADY_ALIGNED=0 to fallback "
233 "to handmade aligned memory allocator.");
234#endif
235#else
236 result = handmade_aligned_malloc(size);
237#endif
238
239 if (!result && size) throw_std_bad_alloc();
240
241 return result;
242}
243
245EIGEN_DEVICE_FUNC inline void aligned_free(void* ptr) {
246#if (EIGEN_DEFAULT_ALIGN_BYTES == 0) || EIGEN_MALLOC_ALREADY_ALIGNED
247
248 if (ptr != nullptr) {
249 check_that_free_is_allowed();
250 EIGEN_USING_STD(free)
251 free(ptr);
252 }
253
254#else
255 handmade_aligned_free(ptr);
256#endif
257}
258
264EIGEN_DEVICE_FUNC inline void* aligned_realloc(void* ptr, std::size_t new_size, std::size_t old_size) {
265 if (ptr == nullptr) return aligned_malloc(new_size);
266 if (old_size == new_size) return ptr;
267 if (new_size == 0) {
268 aligned_free(ptr);
269 return nullptr;
270 }
271
272 void* result;
273#if (EIGEN_DEFAULT_ALIGN_BYTES == 0) || EIGEN_MALLOC_ALREADY_ALIGNED
274 EIGEN_UNUSED_VARIABLE(old_size);
275
276 check_that_malloc_is_allowed();
277 EIGEN_USING_STD(realloc)
278 result = realloc(ptr, new_size);
279#else
280 result = handmade_aligned_realloc(ptr, new_size, old_size);
281#endif
282
283 if (!result && new_size) throw_std_bad_alloc();
284
285 return result;
286}
287
288/*****************************************************************************
289*** Implementation of conditionally aligned functions ***
290*****************************************************************************/
291
295template <bool Align>
296EIGEN_DEVICE_FUNC inline void* conditional_aligned_malloc(std::size_t size) {
297 return aligned_malloc(size);
298}
299
300template <>
301EIGEN_DEVICE_FUNC inline void* conditional_aligned_malloc<false>(std::size_t size) {
302 if (size == 0) return nullptr;
303
304 check_that_malloc_is_allowed();
305 EIGEN_USING_STD(malloc)
306 void* result = malloc(size);
307
308 if (!result && size) throw_std_bad_alloc();
309 return result;
310}
311
313template <bool Align>
314EIGEN_DEVICE_FUNC inline void conditional_aligned_free(void* ptr) {
315 aligned_free(ptr);
316}
317
318template <>
319EIGEN_DEVICE_FUNC inline void conditional_aligned_free<false>(void* ptr) {
320 if (ptr != nullptr) {
321 check_that_free_is_allowed();
322 EIGEN_USING_STD(free)
323 free(ptr);
324 }
325}
326
327template <bool Align>
328EIGEN_DEVICE_FUNC inline void* conditional_aligned_realloc(void* ptr, std::size_t new_size, std::size_t old_size) {
329 return aligned_realloc(ptr, new_size, old_size);
330}
331
332template <>
333EIGEN_DEVICE_FUNC inline void* conditional_aligned_realloc<false>(void* ptr, std::size_t new_size,
334 std::size_t old_size) {
335 if (ptr == nullptr) return conditional_aligned_malloc<false>(new_size);
336 if (old_size == new_size) return ptr;
337 if (new_size == 0) {
338 conditional_aligned_free<false>(ptr);
339 return nullptr;
340 }
341
342 check_that_malloc_is_allowed();
343 EIGEN_USING_STD(realloc)
344 void* result = realloc(ptr, new_size);
345 if (!result) throw_std_bad_alloc();
346 return result;
347}
348
349/*****************************************************************************
350*** Construction/destruction of array elements ***
351*****************************************************************************/
352
356template <typename T>
357EIGEN_DEVICE_FUNC inline void destruct_elements_of_array(T* ptr, std::size_t size) {
358 // always destruct an array starting from the end.
359 if (ptr)
360 while (size) ptr[--size].~T();
361}
362
366template <typename T>
367EIGEN_DEVICE_FUNC inline T* default_construct_elements_of_array(T* ptr, std::size_t size) {
368 std::size_t i = 0;
369 EIGEN_TRY {
370 for (i = 0; i < size; ++i) ::new (ptr + i) T;
371 }
372 EIGEN_CATCH(...) {
373 destruct_elements_of_array(ptr, i);
374 EIGEN_THROW;
375 }
376 return ptr;
377}
378
382template <typename T>
383EIGEN_DEVICE_FUNC inline T* copy_construct_elements_of_array(T* ptr, const T* src, std::size_t size) {
384 std::size_t i = 0;
385 EIGEN_TRY {
386 for (i = 0; i < size; ++i) ::new (ptr + i) T(*(src + i));
387 }
388 EIGEN_CATCH(...) {
389 destruct_elements_of_array(ptr, i);
390 EIGEN_THROW;
391 }
392 return ptr;
393}
394
398template <typename T>
399EIGEN_DEVICE_FUNC inline T* move_construct_elements_of_array(T* ptr, T* src, std::size_t size) {
400 std::size_t i = 0;
401 EIGEN_TRY {
402 for (i = 0; i < size; ++i) ::new (ptr + i) T(std::move(*(src + i)));
403 }
404 EIGEN_CATCH(...) {
405 destruct_elements_of_array(ptr, i);
406 EIGEN_THROW;
407 }
408 return ptr;
409}
410
411/*****************************************************************************
412*** Implementation of aligned new/delete-like functions ***
413*****************************************************************************/
414
415template <typename T>
416EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void check_size_for_overflow(std::size_t size) {
417 constexpr std::size_t max_elements = (std::numeric_limits<std::ptrdiff_t>::max)() / sizeof(T);
418 if (size > max_elements) throw_std_bad_alloc();
419}
420
425template <typename T>
426EIGEN_DEVICE_FUNC inline T* aligned_new(std::size_t size) {
427 check_size_for_overflow<T>(size);
428 T* result = static_cast<T*>(aligned_malloc(sizeof(T) * size));
429 EIGEN_TRY { default_construct_elements_of_array(result, size); }
430 EIGEN_CATCH(...) {
431 aligned_free(result);
432 EIGEN_THROW;
433 }
434 return result;
435}
436
437template <typename T, bool Align>
438EIGEN_DEVICE_FUNC inline T* conditional_aligned_new(std::size_t size) {
439 check_size_for_overflow<T>(size);
440 T* result = static_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T) * size));
441 EIGEN_TRY { default_construct_elements_of_array(result, size); }
442 EIGEN_CATCH(...) {
443 conditional_aligned_free<Align>(result);
444 EIGEN_THROW;
445 }
446 return result;
447}
448
452template <typename T>
453EIGEN_DEVICE_FUNC inline void aligned_delete(T* ptr, std::size_t size) {
454 destruct_elements_of_array<T>(ptr, size);
455 aligned_free(ptr);
456}
457
461template <typename T, bool Align>
462EIGEN_DEVICE_FUNC inline void conditional_aligned_delete(T* ptr, std::size_t size) {
463 destruct_elements_of_array<T>(ptr, size);
464 conditional_aligned_free<Align>(ptr);
465}
466
467template <typename T, bool Align>
468EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new(T* pts, std::size_t new_size, std::size_t old_size) {
469 check_size_for_overflow<T>(new_size);
470 check_size_for_overflow<T>(old_size);
471
472 // If elements need to be explicitly initialized, we cannot simply realloc
473 // (or memcpy) the memory block - each element needs to be reconstructed.
474 // Otherwise, objects that contain internal pointers like mpfr or
475 // AnnoyingScalar can be pointing to the wrong thing.
476 T* result = static_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T) * new_size));
477 const std::size_t copy_size = (std::min)(old_size, new_size);
478 bool tail_constructed = false;
479 EIGEN_TRY {
480 // A failing default constructor must not leave the source elements moved-from.
481 if (new_size > copy_size) {
482 default_construct_elements_of_array(result + copy_size, new_size - copy_size);
483 tail_constructed = true;
484 }
485 move_construct_elements_of_array(result, pts, copy_size);
486
487 // Delete old elements.
488 conditional_aligned_delete<T, Align>(pts, old_size);
489 }
490 EIGEN_CATCH(...) {
491 // The construction helpers clean up their own partial ranges, but not the completed tail.
492 if (tail_constructed) destruct_elements_of_array(result + copy_size, new_size - copy_size);
493 conditional_aligned_free<Align>(result);
494 EIGEN_THROW;
495 }
496
497 return result;
498}
499
500template <typename T, bool Align>
501EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(std::size_t size) {
502 if (size == 0) return nullptr; // short-cut. Also fixes Bug 884
503 check_size_for_overflow<T>(size);
504 T* result = static_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T) * size));
505 EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
506 EIGEN_TRY { default_construct_elements_of_array(result, size); }
507 EIGEN_CATCH(...) {
508 conditional_aligned_free<Align>(result);
509 EIGEN_THROW;
510 }
511 }
512 return result;
513}
514
515template <typename T, bool Align>
516EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new_auto(T* pts, std::size_t new_size, std::size_t old_size) {
517 EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
518 return conditional_aligned_realloc_new<T, Align>(pts, new_size, old_size);
519 }
520
521 check_size_for_overflow<T>(new_size);
522 check_size_for_overflow<T>(old_size);
523 EIGEN_IF_CONSTEXPR (std::is_trivially_copyable<T>::value) {
524 return static_cast<T*>(
525 conditional_aligned_realloc<Align>(static_cast<void*>(pts), sizeof(T) * new_size, sizeof(T) * old_size));
526 }
527
528 // realloc relocates bytes, which only a trivially copyable T allows: move the elements instead. As in
529 // conditional_aligned_new_auto and conditional_aligned_delete_auto, T gets no default constructor or destructor call.
530 // Eigen's own callers do not reach this: dense conservativeResize() copies such a T into a new object, and
531 // CompressedStorage relocates only its stored entries, since old_size counts its uninitialized capacity too.
532 if (new_size == old_size) return pts;
533 T* result = static_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T) * new_size));
534 EIGEN_TRY { move_construct_elements_of_array(result, pts, (std::min)(old_size, new_size)); }
535 EIGEN_CATCH(...) {
536 conditional_aligned_free<Align>(result);
537 EIGEN_THROW;
538 }
539 conditional_aligned_free<Align>(pts);
540 return result;
541}
542
543template <typename T, bool Align>
544EIGEN_DEVICE_FUNC inline void conditional_aligned_delete_auto(T* ptr, std::size_t size) {
545 EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) destruct_elements_of_array<T>(ptr, size);
546 conditional_aligned_free<Align>(ptr);
547}
548
549/****************************************************************************/
550
569template <int Alignment, typename Scalar, typename Index>
570EIGEN_DEVICE_FUNC inline Index first_aligned(const Scalar* array, Index size) {
571 constexpr Index ScalarSize = sizeof(Scalar);
572 constexpr Index AlignmentSize = Alignment / ScalarSize;
573 constexpr Index AlignmentMask = AlignmentSize - 1;
574
575 EIGEN_IF_CONSTEXPR (Alignment <= alignof(Scalar)) {
576 // A valid Scalar* is already aligned to at least alignof(Scalar).
577 return 0;
578 } else EIGEN_IF_CONSTEXPR (Alignment <= ScalarSize) {
579 // Stepping by sizeof(Scalar) preserves address modulo Alignment, so either element 0 is aligned or none is.
580 return (std::uintptr_t(array) & (std::uintptr_t(Alignment) - 1)) ? size : 0;
581 } else EIGEN_IF_CONSTEXPR ((Alignment % ScalarSize) != 0) {
582 // The requested alignment is not a multiple of the scalar size. Consequently, no element of the array is well
583 // aligned.
584 return size;
585 } else if (std::uintptr_t(array) & (sizeof(Scalar) - 1)) {
586 // The array is not aligned to the size of a single scalar. Consequently, no element of the array is well aligned.
587 return size;
588 } else {
589 Index first = (AlignmentSize - (Index((std::uintptr_t(array) / sizeof(Scalar))) & AlignmentMask)) & AlignmentMask;
590 return (first < size) ? first : size;
591 }
592}
593
596template <typename Scalar, typename Index>
597EIGEN_DEVICE_FUNC inline Index first_default_aligned(const Scalar* array, Index size) {
598 using DefaultPacketType = typename packet_traits<Scalar>::type;
599 return first_aligned<unpacket_traits<DefaultPacketType>::alignment>(array, size);
600}
601
604template <typename Index>
605inline Index first_multiple(Index size, Index base) {
606 return ((size + base - 1) / base) * base;
607}
608
609// Copies [start, end) to the non-overlapping range at target: memcpy where T is trivially copyable, which is the
610// exact precondition of a byte copy, and element-wise assignment otherwise. NumTraits<T>::RequireInitialization says
611// whether T needs construction before first use, not whether its bytes may be duplicated.
612template <typename T, bool UseMemcpy>
613struct smart_copy_helper;
614
615template <typename T>
616EIGEN_DEVICE_FUNC void smart_copy(const T* start, const T* end, T* target) {
617 smart_copy_helper<T, std::is_trivially_copyable<T>::value>::run(start, end, target);
618}
619
620template <typename T>
621struct smart_copy_helper<T, true> {
622 EIGEN_DEVICE_FUNC static inline void run(const T* start, const T* end, T* target) {
623 std::ptrdiff_t count = end - start;
624 if (count <= 0) return;
625 eigen_internal_assert(start != 0 && end != 0 && target != 0);
626 EIGEN_USING_STD(memcpy)
627 memcpy(target, start, static_cast<std::size_t>(count) * sizeof(T));
628 }
629};
630
631// The ranges do not overlap, and saying so lets the compiler turn an assignment that copies fields into memcpy.
632template <typename T>
633struct smart_copy_helper<T, false> {
634 EIGEN_DEVICE_FUNC static inline void run(const T* EIGEN_RESTRICT start, const T* end, T* EIGEN_RESTRICT target) {
635 std::copy(start, end, target);
636 }
637};
638
639// std::move(x) where `T = std::move(x)` compiles, and x otherwise, so that a T that deletes its move assignment but
640// keeps its copy assignment is copied rather than rejected.
641template <typename T>
642constexpr std::conditional_t<std::is_move_assignable<T>::value, T&&, const T&> move_if_assignable(T& x) noexcept {
643 return std::move(x);
644}
645
646// std::move and std::move_backward where T can be move-assigned, std::copy and std::copy_backward otherwise. The
647// ranges may overlap as those algorithms allow.
648template <typename T, bool Move = std::is_move_assignable<T>::value>
649struct move_or_copy_helper {
650 static void run(T* start, T* end, T* target) { std::move(start, end, target); }
651 static void run_backward(T* start, T* end, T* target_end) { std::move_backward(start, end, target_end); }
652};
653
654template <typename T>
655struct move_or_copy_helper<T, false> {
656 static void run(T* start, T* end, T* target) { std::copy(start, end, target); }
657 static void run_backward(T* start, T* end, T* target_end) { std::copy_backward(start, end, target_end); }
658};
659
660template <typename T>
661inline void move_or_copy(T* start, T* end, T* target) {
662 move_or_copy_helper<T>::run(start, end, target);
663}
664
665template <typename T>
666inline void move_or_copy_backward(T* start, T* end, T* target_end) {
667 move_or_copy_helper<T>::run_backward(start, end, target_end);
668}
669
670/*****************************************************************************
671*** Implementation of runtime stack allocation (falling back to malloc) ***
672*****************************************************************************/
673
674// you can overwrite Eigen's default behavior regarding alloca by defining EIGEN_ALLOCA
675// to the appropriate stack allocation function
676#if !defined EIGEN_ALLOCA && !defined EIGEN_GPU_COMPILE_PHASE
677#if EIGEN_OS_LINUX || EIGEN_OS_MAC || (defined alloca)
678#define EIGEN_ALLOCA alloca
679#elif EIGEN_COMP_MSVC
680#define EIGEN_ALLOCA _alloca
681#endif
682#endif
683
684// With clang -Oz -mthumb, alloca changes the stack pointer in a way that is
685// not allowed in Thumb2. -DEIGEN_STACK_ALLOCATION_LIMIT=0 doesn't work because
686// the compiler still emits bad code because stack allocation checks use "<=".
687// TODO: Eliminate after https://bugs.llvm.org/show_bug.cgi?id=23772
688// is fixed.
689#if defined(__clang__) && defined(__thumb__)
690#undef EIGEN_ALLOCA
691#endif
692
693// The SME GEMM kernel reads its packed panels a streaming vector at a time and slows down when they straddle
694// 64-byte lines, so SME builds align the internal temporaries to 64 bytes, heap and stack alike.
695#ifndef EIGEN_STACK_ALIGN_BYTES
696#if defined(EIGEN_VECTORIZE_SME) && EIGEN_DEFAULT_ALIGN_BYTES < 64
697#define EIGEN_STACK_ALIGN_BYTES 64
698#else
699#define EIGEN_STACK_ALIGN_BYTES EIGEN_DEFAULT_ALIGN_BYTES
700#endif
701#endif
702
704EIGEN_DEVICE_FUNC inline void* scratch_malloc(std::size_t size) {
705#if EIGEN_STACK_ALIGN_BYTES > EIGEN_DEFAULT_ALIGN_BYTES
706 void* result = handmade_aligned_malloc(size, EIGEN_STACK_ALIGN_BYTES);
707 if (!result && size) throw_std_bad_alloc();
708 return result;
709#else
710 return aligned_malloc(size);
711#endif
712}
713EIGEN_DEVICE_FUNC inline void scratch_free(void* ptr) {
714#if EIGEN_STACK_ALIGN_BYTES > EIGEN_DEFAULT_ALIGN_BYTES
715 handmade_aligned_free(ptr);
716#else
717 aligned_free(ptr);
718#endif
719}
720
722template <typename T>
723EIGEN_DEVICE_FUNC inline T* scratch_malloc(std::size_t size) {
724 EIGEN_IF_CONSTEXPR (alignof(T) <= EIGEN_STACK_ALIGN_BYTES ||
725 (EIGEN_STACK_ALIGN_BYTES == 0 && alignof(T) <= alignof(std::max_align_t))) {
726 return static_cast<T*>(scratch_malloc(size));
727 } else {
728 // Unlike the byte-offset allocator, this also supports alignments greater than 256.
729 constexpr std::size_t overhead = alignof(T) - 1 + sizeof(void*);
730 size += overhead;
731 if (size < overhead) throw_std_bad_alloc();
732 check_that_malloc_is_allowed();
733 void* original = std::malloc(size);
734 if (!original) throw_std_bad_alloc();
735 // std::align is host-only with CUDA.
736 char* start = static_cast<char*>(original) + sizeof(void*);
737 const std::uintptr_t offset = reinterpret_cast<std::uintptr_t>(start) % alignof(T);
738 const std::uintptr_t shift = offset > 0 ? alignof(T) - offset : 0;
739 void* aligned = start + shift;
740 std::memcpy(static_cast<char*>(aligned) - sizeof(void*), reinterpret_cast<const void*>(&original), sizeof(void*));
741 return static_cast<T*>(aligned);
742 }
743}
744
745template <typename T>
746EIGEN_DEVICE_FUNC inline void scratch_free(void* ptr) {
747 EIGEN_IF_CONSTEXPR (alignof(T) <= EIGEN_STACK_ALIGN_BYTES ||
748 (EIGEN_STACK_ALIGN_BYTES == 0 && alignof(T) <= alignof(std::max_align_t))) {
749 scratch_free(ptr);
750 } else if (ptr) {
751 void* original;
752 std::memcpy(reinterpret_cast<void*>(&original), static_cast<char*>(ptr) - sizeof(void*), sizeof(void*));
753 check_that_free_is_allowed();
754 std::free(original);
755 }
756}
757
759template <typename T>
760EIGEN_DEVICE_FUNC inline T* scratch_new(std::size_t size) {
761 check_size_for_overflow<T>(size);
762 T* result = scratch_malloc<T>(sizeof(T) * size);
763 EIGEN_TRY { default_construct_elements_of_array(result, size); }
764 EIGEN_CATCH(...) {
765 scratch_free<T>(result);
766 EIGEN_THROW;
767 }
768 return result;
769}
770
772template <typename T>
773EIGEN_DEVICE_FUNC inline void scratch_delete(T* ptr, std::size_t size) {
774 destruct_elements_of_array<T>(ptr, size);
775 scratch_free<T>(ptr);
776}
777
778// This helper class construct the allocated memory, and takes care of destructing and freeing the handled data
779// at destruction time. In practice this helper class is mainly useful to avoid memory leak in case of exceptions.
780template <typename T>
781class aligned_stack_memory_handler {
782 public:
783 aligned_stack_memory_handler(const aligned_stack_memory_handler&) = delete;
784 aligned_stack_memory_handler& operator=(const aligned_stack_memory_handler&) = delete;
785
786 /* Creates a stack_memory_handler responsible for the buffer \a ptr of size \a size.
787 * Note that \a ptr can be 0 regardless of the other parameters.
788 * This constructor takes care of constructing/initializing the elements of the buffer if required by the scalar type
789 *T (see NumTraits<T>::RequireInitialization). In this case, the buffer elements will also be destructed when this
790 *handler will be destructed. Finally, if \a dealloc is true, then the pointer \a ptr is freed.
791 **/
792 EIGEN_DEVICE_FUNC aligned_stack_memory_handler(T* ptr, std::size_t size, bool dealloc)
793 : m_ptr(ptr), m_size(size), m_deallocate(dealloc) {
794 EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
795 if (m_ptr) Eigen::internal::default_construct_elements_of_array(m_ptr, size);
796 }
797 }
798 EIGEN_DEVICE_FUNC ~aligned_stack_memory_handler() {
799 EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
800 if (m_ptr) Eigen::internal::destruct_elements_of_array<T>(m_ptr, m_size);
801 }
802 if (m_deallocate) Eigen::internal::scratch_free<T>(m_ptr);
803 }
804
805 protected:
806 T* m_ptr;
807 std::size_t m_size;
808 bool m_deallocate;
809};
810
811#ifdef EIGEN_ALLOCA
812
813template <typename Xpr, int NbEvaluations,
814 bool MapExternalBuffer = nested_eval<Xpr, NbEvaluations>::Evaluate && Xpr::MaxSizeAtCompileTime == Dynamic>
815struct local_nested_eval_wrapper {
816 static constexpr bool NeedExternalBuffer = false;
817 using Scalar = typename Xpr::Scalar;
818 using ObjectType = typename nested_eval<Xpr, NbEvaluations>::type;
819 ObjectType object;
820
821 EIGEN_DEVICE_FUNC local_nested_eval_wrapper(const Xpr& xpr, Scalar* ptr) : object(xpr) {
822 EIGEN_UNUSED_VARIABLE(ptr);
823 eigen_internal_assert(ptr == 0);
824 }
825};
826
827template <typename Xpr, int NbEvaluations>
828struct local_nested_eval_wrapper<Xpr, NbEvaluations, true> {
829 static constexpr bool NeedExternalBuffer = true;
830 using Scalar = typename Xpr::Scalar;
831 using PlainObject = typename plain_object_eval<Xpr>::type;
832 using ObjectType = Map<PlainObject, EIGEN_DEFAULT_ALIGN_BYTES>;
833 ObjectType object;
834
835 EIGEN_DEVICE_FUNC local_nested_eval_wrapper(const Xpr& xpr, Scalar* ptr)
836 : object(ptr == 0 ? Eigen::internal::scratch_malloc<Scalar>(sizeof(Scalar) * xpr.size()) : ptr, xpr.rows(),
837 xpr.cols()),
838 m_deallocate(ptr == 0) {
839 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
840 if (object.data()) Eigen::internal::default_construct_elements_of_array(object.data(), object.size());
841 }
842 object = xpr;
843 }
844
845 EIGEN_DEVICE_FUNC ~local_nested_eval_wrapper() {
846 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
847 if (object.data()) Eigen::internal::destruct_elements_of_array(object.data(), object.size());
848 }
849 if (m_deallocate) Eigen::internal::scratch_free<Scalar>(object.data());
850 }
851
852 private:
853 bool m_deallocate;
854};
855
856#endif // EIGEN_ALLOCA
857
858} // end namespace internal
859
886#if defined(EIGEN_ALLOCA) && !defined(EIGEN_NO_ALLOCA)
887
888#if EIGEN_STACK_ALIGN_BYTES > 0
889// We always manually re-align the result of EIGEN_ALLOCA.
890// If alloca is already aligned, the compiler should be smart enough to optimize away the re-alignment.
891
892#if ((EIGEN_COMP_GNUC || EIGEN_COMP_CLANG) && !EIGEN_COMP_NVHPC && !EIGEN_COMP_ICC)
893#define EIGEN_ALIGNED_ALLOCA(SIZE) __builtin_alloca_with_align(SIZE, CHAR_BIT* EIGEN_STACK_ALIGN_BYTES)
894#else
895EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void* eigen_aligned_alloca_helper(void* ptr) {
896 constexpr std::uintptr_t mask = EIGEN_STACK_ALIGN_BYTES - 1;
897 std::uintptr_t ptr_int = std::uintptr_t(ptr);
898 std::uintptr_t aligned_ptr_int = (ptr_int + mask) & ~mask;
899 std::uintptr_t offset = aligned_ptr_int - ptr_int;
900 return static_cast<void*>(static_cast<uint8_t*>(ptr) + offset);
901}
902#define EIGEN_ALIGNED_ALLOCA(SIZE) eigen_aligned_alloca_helper(EIGEN_ALLOCA(SIZE + EIGEN_STACK_ALIGN_BYTES - 1))
903#endif
904
905#else
906#define EIGEN_ALIGNED_ALLOCA(SIZE) EIGEN_ALLOCA(SIZE)
907#endif
908
909// Preserve the existing allocation path unless the scalar needs stronger alignment.
910#define EIGEN_TYPED_ALIGNED_ALLOCA(TYPE, SIZE) \
911 (alignof(TYPE) <= EIGEN_STACK_ALIGN_BYTES \
912 ? EIGEN_ALIGNED_ALLOCA(SIZE) \
913 : reinterpret_cast<void*>( \
914 (reinterpret_cast<std::uintptr_t>(EIGEN_ALLOCA((SIZE) + alignof(TYPE) - 1)) + alignof(TYPE) - 1) & \
915 ~(std::uintptr_t(alignof(TYPE)) - 1)))
916
917// TYPE and NAME form declarations, not expressions.
918// NOLINTBEGIN(bugprone-macro-parentheses)
919#define ei_declare_aligned_stack_constructed_variable(TYPE, NAME, SIZE, BUFFER) \
920 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
921 TYPE* NAME = (BUFFER) != 0 \
922 ? (BUFFER) \
923 : reinterpret_cast<TYPE*>((sizeof(TYPE) * (SIZE) <= EIGEN_STACK_ALLOCATION_LIMIT) \
924 ? EIGEN_TYPED_ALIGNED_ALLOCA(TYPE, sizeof(TYPE) * (SIZE)) \
925 : Eigen::internal::scratch_malloc<TYPE>(sizeof(TYPE) * (SIZE))); \
926 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME, _stack_memory_destructor)( \
927 (BUFFER) == 0 ? NAME : 0, SIZE, sizeof(TYPE) * (SIZE) > EIGEN_STACK_ALLOCATION_LIMIT)
928// NOLINTEND(bugprone-macro-parentheses)
929
930#define ei_declare_local_nested_eval(XPR_T, XPR, N, NAME) \
931 Eigen::internal::local_nested_eval_wrapper<XPR_T, N> EIGEN_CAT(NAME, _wrapper)( \
932 XPR, reinterpret_cast<typename XPR_T::Scalar*>( \
933 ((Eigen::internal::local_nested_eval_wrapper<XPR_T, N>::NeedExternalBuffer) && \
934 ((sizeof(typename XPR_T::Scalar) * (XPR).size()) <= EIGEN_STACK_ALLOCATION_LIMIT)) \
935 ? EIGEN_TYPED_ALIGNED_ALLOCA(typename XPR_T::Scalar, sizeof(typename XPR_T::Scalar) * (XPR).size()) \
936 : 0)); \
937 typename Eigen::internal::local_nested_eval_wrapper<XPR_T, N>::ObjectType NAME(EIGEN_CAT(NAME, _wrapper).object)
938
939#else
940
941// TYPE and NAME form declarations, not expressions.
942// NOLINTBEGIN(bugprone-macro-parentheses)
943#define ei_declare_aligned_stack_constructed_variable(TYPE, NAME, SIZE, BUFFER) \
944 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
945 TYPE* NAME = (BUFFER) != 0 ? (BUFFER) : Eigen::internal::scratch_malloc<TYPE>(sizeof(TYPE) * (SIZE)); \
946 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME, _stack_memory_destructor)( \
947 (BUFFER) == 0 ? NAME : 0, SIZE, true)
948// NOLINTEND(bugprone-macro-parentheses)
949
950#define ei_declare_local_nested_eval(XPR_T, XPR, N, NAME) \
951 typename Eigen::internal::nested_eval<XPR_T, N>::type NAME(XPR)
952
953#endif
954
955/*****************************************************************************
956*** Implementation of EIGEN_MAKE_ALIGNED_OPERATOR_NEW [_IF] ***
957*****************************************************************************/
958
959#if EIGEN_HAS_CXX17_OVERALIGN
960
961// C++17 -> no need to bother about alignment anymore :)
962
963#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign)
964#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
965#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW
966#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar, Size)
967
968#else
969
970// HIP does not support new/delete on device.
971#if EIGEN_MAX_ALIGN_BYTES != 0 && !defined(EIGEN_HIP_DEVICE_COMPILE)
972#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
973 EIGEN_DEVICE_FUNC void* operator new(std::size_t size, const std::nothrow_t&) noexcept { \
974 EIGEN_TRY { return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); } \
975 EIGEN_CATCH(...) { return 0; } \
976 } \
977 EIGEN_DEVICE_FUNC void* operator new[](std::size_t size, const std::nothrow_t&) noexcept { \
978 EIGEN_TRY { return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); } \
979 EIGEN_CATCH(...) { return 0; } \
980 }
981#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) \
982 EIGEN_DEVICE_FUNC void* operator new(std::size_t size) { \
983 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
984 } \
985 EIGEN_DEVICE_FUNC void* operator new[](std::size_t size) { \
986 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
987 } \
988 EIGEN_DEVICE_FUNC void operator delete(void* ptr) noexcept { \
989 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
990 } \
991 EIGEN_DEVICE_FUNC void operator delete[](void* ptr) noexcept { \
992 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
993 } \
994 EIGEN_DEVICE_FUNC void operator delete(void* ptr, std::size_t /* sz */) noexcept { \
995 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
996 } \
997 EIGEN_DEVICE_FUNC void operator delete[](void* ptr, std::size_t /* sz */) noexcept { \
998 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
999 } \
1000 /* in-place new and delete. since (at least afaik) there is no actual */ \
1001 /* memory allocated we can safely let the default implementation handle */ \
1002 /* this particular case. */ \
1003 EIGEN_DEVICE_FUNC static void* operator new(std::size_t size, void* ptr) { return ::operator new(size, ptr); } \
1004 EIGEN_DEVICE_FUNC static void* operator new[](std::size_t size, void* ptr) { return ::operator new[](size, ptr); } \
1005 EIGEN_DEVICE_FUNC void operator delete(void* memory, void* ptr) noexcept { return ::operator delete(memory, ptr); } \
1006 EIGEN_DEVICE_FUNC void operator delete[](void* memory, void* ptr) noexcept { \
1007 return ::operator delete[](memory, ptr); \
1008 } \
1009 /* nothrow-new (returns zero instead of std::bad_alloc) */ \
1010 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
1011 EIGEN_DEVICE_FUNC void operator delete(void* ptr, const std::nothrow_t&) noexcept { \
1012 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
1013 } \
1014 EIGEN_DEVICE_FUNC void operator delete[](void* ptr, const std::nothrow_t&) noexcept { \
1015 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
1016 } \
1017 using eigen_aligned_operator_new_marker_type = void;
1018#else
1019#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
1020#endif
1021
1022#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(true)
1023#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar, Size) \
1024 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF( \
1025 bool(((Size) != Eigen::Dynamic) && \
1026 (((EIGEN_MAX_ALIGN_BYTES >= 16) && ((sizeof(Scalar) * size_t(Size)) % (EIGEN_MAX_ALIGN_BYTES) == 0)) || \
1027 ((EIGEN_MAX_ALIGN_BYTES >= 32) && ((sizeof(Scalar) * size_t(Size)) % (EIGEN_MAX_ALIGN_BYTES / 2) == 0)) || \
1028 ((EIGEN_MAX_ALIGN_BYTES >= 64) && ((sizeof(Scalar) * size_t(Size)) % (EIGEN_MAX_ALIGN_BYTES / 4) == 0)))))
1029
1030#endif
1031
1032/****************************************************************************/
1033
1058template <class T>
1059class aligned_allocator {
1060 public:
1061 using size_type = std::size_t;
1062 using difference_type = std::ptrdiff_t;
1063 using pointer = T*;
1064 using const_pointer = const T*;
1065 using reference = T&;
1066 using const_reference = const T&;
1067 using value_type = T;
1068
1069 template <class U>
1070 struct rebind {
1071 using other = aligned_allocator<U>;
1072 };
1073
1074 aligned_allocator() = default;
1075
1076 aligned_allocator(const aligned_allocator&) = default;
1077
1078 template <class U>
1079 aligned_allocator(const aligned_allocator<U>&) {}
1080
1081 template <class U>
1082 constexpr bool operator==(const aligned_allocator<U>&) const noexcept {
1083 return true;
1084 }
1085 template <class U>
1086 constexpr bool operator!=(const aligned_allocator<U>&) const noexcept {
1087 return false;
1088 }
1089
1090#if EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_STRICT_AT_LEAST(7, 0, 0)
1091 // In gcc std::allocator::max_size() is bugged making gcc triggers a warning:
1092 // eigen/Eigen/src/Core/util/Memory.h:189:12: warning: argument 1 value '18446744073709551612' exceeds maximum object
1093 // size 9223372036854775807 See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87544
1094 size_type max_size() const { return (std::numeric_limits<std::ptrdiff_t>::max)() / sizeof(T); }
1095#endif
1096
1097 pointer allocate(size_type num, const void* /*hint*/ = nullptr) {
1098 internal::check_size_for_overflow<T>(num);
1099 return static_cast<pointer>(internal::aligned_malloc(num * sizeof(T)));
1100 }
1101
1102 void deallocate(pointer p, size_type /*num*/) { internal::aligned_free(p); }
1103};
1104
1105//---------- Cache sizes ----------
1106
1107#if EIGEN_OS_LINUX && !defined(EIGEN_NO_CPU_CACHE_SYSFS)
1108// Linux publishes the cache topology under sysfs on every architecture.
1109#define EIGEN_CPU_CACHE_SYSFS 1
1110#endif
1111
1112#if !defined(EIGEN_NO_CPUID)
1113#if EIGEN_COMP_GNUC && EIGEN_ARCH_i386_OR_x86_64
1114#if defined(__PIC__) && EIGEN_ARCH_i386
1115// Case for x86 with PIC
1116#define EIGEN_CPUID(abcd, func, id) \
1117 __asm__ __volatile__("xchgl %%ebx, %k1;cpuid; xchgl %%ebx,%k1" \
1118 : "=a"(abcd[0]), "=&r"(abcd[1]), "=c"(abcd[2]), "=d"(abcd[3]) \
1119 : "a"(func), "c"(id));
1120#elif defined(__PIC__) && EIGEN_ARCH_x86_64
1121// Case for x64 with PIC. In theory this is only a problem with recent gcc and with medium or large code model, not with
1122// the default small code model. However, we cannot detect which code model is used, and the xchg overhead is negligible
1123// anyway.
1124#define EIGEN_CPUID(abcd, func, id) \
1125 __asm__ __volatile__("xchg{q}\t{%%}rbx, %q1; cpuid; xchg{q}\t{%%}rbx, %q1" \
1126 : "=a"(abcd[0]), "=&r"(abcd[1]), "=c"(abcd[2]), "=d"(abcd[3]) \
1127 : "0"(func), "2"(id));
1128#else
1129// Case for x86_64 or x86 w/o PIC
1130#define EIGEN_CPUID(abcd, func, id) \
1131 __asm__ __volatile__("cpuid" : "=a"(abcd[0]), "=b"(abcd[1]), "=c"(abcd[2]), "=d"(abcd[3]) : "0"(func), "2"(id));
1132#endif
1133#elif EIGEN_COMP_MSVC
1134#if EIGEN_ARCH_i386_OR_x86_64
1135#define EIGEN_CPUID(abcd, func, id) __cpuidex((int*)abcd, func, id)
1136#endif
1137#endif
1138#endif
1139
1140namespace internal {
1141
1142#ifdef EIGEN_CPUID
1143
1144inline bool cpuid_is_vendor(int abcd[4], const int vendor[3]) {
1145 return abcd[1] == vendor[0] && abcd[3] == vendor[1] && abcd[2] == vendor[2];
1146}
1147
1148inline void queryCacheSizes_intel_direct(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3) {
1149 int abcd[4];
1150 l1 = l2 = l3 = 0;
1151 int cache_id = 0;
1152 int cache_type = 0;
1153 do {
1154 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1155 EIGEN_CPUID(abcd, 0x4, cache_id);
1156 cache_type = (abcd[0] & 0x0F) >> 0;
1157 if (cache_type == 1 || cache_type == 3) // data or unified cache
1158 {
1159 int cache_level = (abcd[0] & 0xE0) >> 5; // A[7:5]
1160 int ways = (abcd[1] & 0xFFC00000) >> 22; // B[31:22]
1161 int partitions = (abcd[1] & 0x003FF000) >> 12; // B[21:12]
1162 int line_size = (abcd[1] & 0x00000FFF) >> 0; // B[11:0]
1163 int sets = (abcd[2]); // C[31:0]
1164
1165 std::ptrdiff_t cache_size =
1166 static_cast<std::ptrdiff_t>(ways + 1) * (partitions + 1) * (line_size + 1) * (sets + 1);
1167
1168 switch (cache_level) {
1169 case 1:
1170 l1 = cache_size;
1171 break;
1172 case 2:
1173 l2 = cache_size;
1174 break;
1175 case 3:
1176 l3 = cache_size;
1177 break;
1178 default:
1179 break;
1180 }
1181 }
1182 cache_id++;
1183 } while (cache_type > 0 && cache_id < 16);
1184}
1185
1186inline void queryCacheSizes_intel_codes(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3) {
1187 int abcd[4];
1188 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1189 l1 = l2 = l3 = 0;
1190 EIGEN_CPUID(abcd, 0x00000002, 0);
1191 unsigned char* bytes = reinterpret_cast<unsigned char*>(abcd) + 2;
1192 bool check_for_p2_core2 = false;
1193 for (int i = 0; i < 14; ++i) {
1194 switch (bytes[i]) {
1195 case 0x0A:
1196 l1 = 8;
1197 break; // 0Ah data L1 cache, 8 KB, 2 ways, 32 byte lines
1198 case 0x0C:
1199 l1 = 16;
1200 break; // 0Ch data L1 cache, 16 KB, 4 ways, 32 byte lines
1201 case 0x0E:
1202 l1 = 24;
1203 break; // 0Eh data L1 cache, 24 KB, 6 ways, 64 byte lines
1204 case 0x10:
1205 l1 = 16;
1206 break; // 10h data L1 cache, 16 KB, 4 ways, 32 byte lines (IA-64)
1207 case 0x15:
1208 l1 = 16;
1209 break; // 15h code L1 cache, 16 KB, 4 ways, 32 byte lines (IA-64)
1210 case 0x2C:
1211 l1 = 32;
1212 break; // 2Ch data L1 cache, 32 KB, 8 ways, 64 byte lines
1213 case 0x30:
1214 l1 = 32;
1215 break; // 30h code L1 cache, 32 KB, 8 ways, 64 byte lines
1216 case 0x60:
1217 l1 = 16;
1218 break; // 60h data L1 cache, 16 KB, 8 ways, 64 byte lines, sectored
1219 case 0x66:
1220 l1 = 8;
1221 break; // 66h data L1 cache, 8 KB, 4 ways, 64 byte lines, sectored
1222 case 0x67:
1223 l1 = 16;
1224 break; // 67h data L1 cache, 16 KB, 4 ways, 64 byte lines, sectored
1225 case 0x68:
1226 l1 = 32;
1227 break; // 68h data L1 cache, 32 KB, 4 ways, 64 byte lines, sectored
1228 case 0x1A:
1229 l2 = 96;
1230 break; // code and data L2 cache, 96 KB, 6 ways, 64 byte lines (IA-64)
1231 case 0x22:
1232 l3 = 512;
1233 break; // code and data L3 cache, 512 KB, 4 ways (!), 64 byte lines, dual-sectored
1234 case 0x23:
1235 l3 = 1024;
1236 break; // code and data L3 cache, 1024 KB, 8 ways, 64 byte lines, dual-sectored
1237 case 0x25:
1238 l3 = 2048;
1239 break; // code and data L3 cache, 2048 KB, 8 ways, 64 byte lines, dual-sectored
1240 case 0x29:
1241 l3 = 4096;
1242 break; // code and data L3 cache, 4096 KB, 8 ways, 64 byte lines, dual-sectored
1243 case 0x39:
1244 l2 = 128;
1245 break; // code and data L2 cache, 128 KB, 4 ways, 64 byte lines, sectored
1246 case 0x3A:
1247 l2 = 192;
1248 break; // code and data L2 cache, 192 KB, 6 ways, 64 byte lines, sectored
1249 case 0x3B:
1250 l2 = 128;
1251 break; // code and data L2 cache, 128 KB, 2 ways, 64 byte lines, sectored
1252 case 0x3C:
1253 l2 = 256;
1254 break; // code and data L2 cache, 256 KB, 4 ways, 64 byte lines, sectored
1255 case 0x3D:
1256 l2 = 384;
1257 break; // code and data L2 cache, 384 KB, 6 ways, 64 byte lines, sectored
1258 case 0x3E:
1259 l2 = 512;
1260 break; // code and data L2 cache, 512 KB, 4 ways, 64 byte lines, sectored
1261 case 0x40:
1262 l2 = 0;
1263 break; // no integrated L2 cache (P6 core) or L3 cache (P4 core)
1264 case 0x41:
1265 l2 = 128;
1266 break; // code and data L2 cache, 128 KB, 4 ways, 32 byte lines
1267 case 0x42:
1268 l2 = 256;
1269 break; // code and data L2 cache, 256 KB, 4 ways, 32 byte lines
1270 case 0x43:
1271 l2 = 512;
1272 break; // code and data L2 cache, 512 KB, 4 ways, 32 byte lines
1273 case 0x44:
1274 l2 = 1024;
1275 break; // code and data L2 cache, 1024 KB, 4 ways, 32 byte lines
1276 case 0x45:
1277 l2 = 2048;
1278 break; // code and data L2 cache, 2048 KB, 4 ways, 32 byte lines
1279 case 0x46:
1280 l3 = 4096;
1281 break; // code and data L3 cache, 4096 KB, 4 ways, 64 byte lines
1282 case 0x47:
1283 l3 = 8192;
1284 break; // code and data L3 cache, 8192 KB, 8 ways, 64 byte lines
1285 case 0x48:
1286 l2 = 3072;
1287 break; // code and data L2 cache, 3072 KB, 12 ways, 64 byte lines
1288 case 0x49:
1289 if (l2 != 0)
1290 l3 = 4096;
1291 else {
1292 check_for_p2_core2 = true;
1293 l3 = l2 = 4096;
1294 }
1295 break; // code and data L3 cache, 4096 KB, 16 ways, 64 byte lines (P4) or L2 for core2
1296 case 0x4A:
1297 l3 = 6144;
1298 break; // code and data L3 cache, 6144 KB, 12 ways, 64 byte lines
1299 case 0x4B:
1300 l3 = 8192;
1301 break; // code and data L3 cache, 8192 KB, 16 ways, 64 byte lines
1302 case 0x4C:
1303 l3 = 12288;
1304 break; // code and data L3 cache, 12288 KB, 12 ways, 64 byte lines
1305 case 0x4D:
1306 l3 = 16384;
1307 break; // code and data L3 cache, 16384 KB, 16 ways, 64 byte lines
1308 case 0x4E:
1309 l2 = 6144;
1310 break; // code and data L2 cache, 6144 KB, 24 ways, 64 byte lines
1311 case 0x78:
1312 l2 = 1024;
1313 break; // code and data L2 cache, 1024 KB, 4 ways, 64 byte lines
1314 case 0x79:
1315 l2 = 128;
1316 break; // code and data L2 cache, 128 KB, 8 ways, 64 byte lines, dual-sectored
1317 case 0x7A:
1318 l2 = 256;
1319 break; // code and data L2 cache, 256 KB, 8 ways, 64 byte lines, dual-sectored
1320 case 0x7B:
1321 l2 = 512;
1322 break; // code and data L2 cache, 512 KB, 8 ways, 64 byte lines, dual-sectored
1323 case 0x7C:
1324 l2 = 1024;
1325 break; // code and data L2 cache, 1024 KB, 8 ways, 64 byte lines, dual-sectored
1326 case 0x7D:
1327 l2 = 2048;
1328 break; // code and data L2 cache, 2048 KB, 8 ways, 64 byte lines
1329 case 0x7E:
1330 l2 = 256;
1331 break; // code and data L2 cache, 256 KB, 8 ways, 128 byte lines, sect. (IA-64)
1332 case 0x7F:
1333 l2 = 512;
1334 break; // code and data L2 cache, 512 KB, 2 ways, 64 byte lines
1335 case 0x80:
1336 l2 = 512;
1337 break; // code and data L2 cache, 512 KB, 8 ways, 64 byte lines
1338 case 0x81:
1339 l2 = 128;
1340 break; // code and data L2 cache, 128 KB, 8 ways, 32 byte lines
1341 case 0x82:
1342 l2 = 256;
1343 break; // code and data L2 cache, 256 KB, 8 ways, 32 byte lines
1344 case 0x83:
1345 l2 = 512;
1346 break; // code and data L2 cache, 512 KB, 8 ways, 32 byte lines
1347 case 0x84:
1348 l2 = 1024;
1349 break; // code and data L2 cache, 1024 KB, 8 ways, 32 byte lines
1350 case 0x85:
1351 l2 = 2048;
1352 break; // code and data L2 cache, 2048 KB, 8 ways, 32 byte lines
1353 case 0x86:
1354 l2 = 512;
1355 break; // code and data L2 cache, 512 KB, 4 ways, 64 byte lines
1356 case 0x87:
1357 l2 = 1024;
1358 break; // code and data L2 cache, 1024 KB, 8 ways, 64 byte lines
1359 case 0x88:
1360 l3 = 2048;
1361 break; // code and data L3 cache, 2048 KB, 4 ways, 64 byte lines (IA-64)
1362 case 0x89:
1363 l3 = 4096;
1364 break; // code and data L3 cache, 4096 KB, 4 ways, 64 byte lines (IA-64)
1365 case 0x8A:
1366 l3 = 8192;
1367 break; // code and data L3 cache, 8192 KB, 4 ways, 64 byte lines (IA-64)
1368 case 0x8D:
1369 l3 = 3072;
1370 break; // code and data L3 cache, 3072 KB, 12 ways, 128 byte lines (IA-64)
1371
1372 default:
1373 break;
1374 }
1375 }
1376 if (check_for_p2_core2 && l2 == l3) l3 = 0;
1377 l1 *= 1024;
1378 l2 *= 1024;
1379 l3 *= 1024;
1380}
1381
1382inline void queryCacheSizes_intel(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3, int max_std_funcs) {
1383 if (max_std_funcs >= 4)
1384 queryCacheSizes_intel_direct(l1, l2, l3);
1385 else if (max_std_funcs >= 2)
1386 queryCacheSizes_intel_codes(l1, l2, l3);
1387 else
1388 l1 = l2 = l3 = 0;
1389}
1390
1391inline void queryCacheSizes_amd(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3) {
1392 int abcd[4];
1393 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1394
1395 // First query the max supported function.
1396 EIGEN_CPUID(abcd, 0x80000000, 0);
1397 if (static_cast<numext::uint32_t>(abcd[0]) >= static_cast<numext::uint32_t>(0x80000006)) {
1398 EIGEN_CPUID(abcd, 0x80000005, 0);
1399 l1 = (abcd[2] >> 24) * 1024; // C[31:24] = L1 size in KB
1400 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1401 EIGEN_CPUID(abcd, 0x80000006, 0);
1402 l2 = (abcd[2] >> 16) * 1024; // C[31;16] = l2 cache size in KB
1403 l3 = static_cast<std::ptrdiff_t>((abcd[3] & 0xFFFC000) >> 18) * 512 * 1024; // D[31;18] = l3 cache size in 512KB
1404 } else {
1405 l1 = l2 = l3 = 0;
1406 }
1407}
1408#endif
1409
1410#ifdef EIGEN_CPU_CACHE_SYSFS
1411
1413struct CpuCacheTopology {
1414 std::ptrdiff_t l1 = 0;
1415 std::ptrdiff_t l2 = 0;
1416 std::ptrdiff_t l3 = 0;
1417 std::ptrdiff_t l3_per_cpu = 0;
1418};
1419
1422template <int Size>
1423inline bool readSysfsLine(const char* root, const char* relative, char (&value)[Size]) {
1424 char path[512];
1425 const int length = std::snprintf(path, sizeof(path), "%s/%s", root, relative);
1426 if (length <= 0 || length >= static_cast<int>(sizeof(path))) return false;
1427 const int fd = ::open(path, O_RDONLY | O_CLOEXEC);
1428 if (fd < 0) return false;
1429 const ssize_t bytes_read = ::read(fd, value, Size - 1);
1430 ::close(fd);
1431 if (bytes_read <= 0) return false;
1432 value[bytes_read] = '\0';
1433 char* newline = std::strchr(value, '\n');
1434 if (newline != nullptr) {
1435 newline[1] = '\0';
1436 }
1437 return true;
1438}
1439
1441template <int Size>
1442inline bool readCpuCacheAttribute(const char* root, int cpu, int index, const char* name, char (&value)[Size]) {
1443 char relative[64];
1444 const int length = std::snprintf(relative, sizeof(relative), "cpu%d/cache/index%d/%s", cpu, index, name);
1445 return length > 0 && length < static_cast<int>(sizeof(relative)) && readSysfsLine(root, relative, value);
1446}
1447
1449inline std::ptrdiff_t parseCpuCacheSize(const char* text) {
1450 char* suffix = nullptr;
1451 const std::ptrdiff_t value = std::strtol(text, &suffix, 10);
1452 if (value <= 0) return 0;
1453 // The kernel writes kibibytes ("%uK"), but only scale on an explicit unit: reading a bare byte
1454 // count as kibibytes would overstate a cache 1024x, which is much worse than the reverse.
1455 const std::ptrdiff_t multiplier = (*suffix == 'K' || *suffix == 'k') ? 1024
1456 : (*suffix == 'M' || *suffix == 'm') ? 1024 * 1024
1457 : 1;
1458 return value * multiplier;
1459}
1460
1462inline bool isCpuListTerminator(const char* text) { return *text == '\0' || *text == '\n' || *text == '\r'; }
1463
1468template <typename Visitor>
1469inline bool parseCpuList(const char* text, Visitor&& visit) {
1470 if (isCpuListTerminator(text)) return false;
1471 // Far above the 8192 CPUs current kernels can number, so a larger id means the line is not a cpu list;
1472 // bounding it also keeps a caller that walks the ranges from visiting ids that cannot exist.
1473 const long max_cpu_id = 1 << 16;
1474 const char* cursor = text;
1475 for (;;) {
1476 char* end = nullptr;
1477 const long first = std::strtol(cursor, &end, 10);
1478 if (end == cursor || first < 0) return false;
1479 long last = first;
1480 if (*end == '-') {
1481 cursor = end + 1;
1482 last = std::strtol(cursor, &end, 10);
1483 if (end == cursor || last < first) return false;
1484 }
1485 if (last >= max_cpu_id) return false;
1486 visit(static_cast<int>(first), static_cast<int>(last));
1487 // A comma promises another range, so a list ending on one is malformed: going round the loop
1488 // lands on the terminator and fails the strtol above.
1489 if (*end == ',') {
1490 cursor = end + 1;
1491 continue;
1492 }
1493 // Anything other than a separator or the end of the line means the format is not what this
1494 // parser assumes.
1495 return isCpuListTerminator(end);
1496 }
1497}
1498
1502inline int parseCpuListCount(const char* text) {
1503 int count = 0;
1504 return parseCpuList(text, [&count](int first, int last) { count += last - first + 1; }) ? count : 0;
1505}
1506
1514inline CpuCacheTopology readCpuCacheTopologySysfs(const char* root, int cpu) {
1515 CpuCacheTopology topology;
1516 // One directory per cache, numbered contiguously from zero. The bound only guards a malformed
1517 // sysfs; it is far above what any current CPU reports.
1518 for (int index = 0; index < 16; ++index) {
1519 // A shared_cpu_list can be long on a large machine, and truncating it would undercount the
1520 // sharers and hand out too large a share.
1521 char value[512];
1522
1523 if (!readCpuCacheAttribute(root, cpu, index, "level", value)) break;
1524 const long level = std::strtol(value, nullptr, 10);
1525 std::ptrdiff_t* target =
1526 level == 1 ? &topology.l1 : (level == 2 ? &topology.l2 : (level == 3 ? &topology.l3 : nullptr));
1527 if (target == nullptr || *target > 0) continue;
1528
1529 // "Data", "Instruction", or "Unified". An instruction cache never holds the operands a product
1530 // blocks for, so it must not be mistaken for the L1 data cache.
1531 if (!readCpuCacheAttribute(root, cpu, index, "type", value) || value[0] == 'I') continue;
1532
1533 if (!readCpuCacheAttribute(root, cpu, index, "size", value)) continue;
1534 const std::ptrdiff_t size = parseCpuCacheSize(value);
1535 if (size <= 0) continue;
1536 *target = size;
1537
1538 if (level == 3 && readCpuCacheAttribute(root, cpu, index, "shared_cpu_list", value) &&
1539 std::strchr(value, '\n') != nullptr) {
1540 const int sharing = parseCpuListCount(value);
1541 if (sharing > 0) topology.l3_per_cpu = size / sharing;
1542 }
1543 }
1544 return topology;
1545}
1546
1548inline std::ptrdiff_t smallerReportedCacheSize(std::ptrdiff_t a, std::ptrdiff_t b) {
1549 return a == 0 ? b : (b == 0 ? a : (std::min)(a, b));
1550}
1551
1557template <typename IsAllowed>
1558inline CpuCacheTopology queryCpuCacheTopologySysfs(const char* root, IsAllowed&& is_allowed) {
1559 CpuCacheTopology topology;
1560 char online[512];
1561 if (!readSysfsLine(root, "online", online) || std::strchr(online, '\n') == nullptr) return topology;
1562 // Validate the whole line before acting on any range of it, so that a malformed list reads as "unknown".
1563 if (!parseCpuList(online, [](int, int) {})) return topology;
1564 parseCpuList(online, [&](int first, int last) {
1565 for (int cpu = first; cpu <= last; ++cpu) {
1566 if (!is_allowed(cpu)) continue;
1567 const CpuCacheTopology candidate = readCpuCacheTopologySysfs(root, cpu);
1568 topology.l1 = smallerReportedCacheSize(topology.l1, candidate.l1);
1569 topology.l2 = smallerReportedCacheSize(topology.l2, candidate.l2);
1570 topology.l3 = smallerReportedCacheSize(topology.l3, candidate.l3);
1571 topology.l3_per_cpu = smallerReportedCacheSize(topology.l3_per_cpu, candidate.l3_per_cpu);
1572 }
1573 });
1574 return topology;
1575}
1576
1583inline CpuCacheTopology queryCpuCacheTopologySysfs(const char* root = "/sys/devices/system/cpu") {
1584#ifdef CPU_SETSIZE
1585 // The affinity API is a GNU extension that glibc, musl and bionic expose under _GNU_SOURCE, which g++ and
1586 // clang++ predefine for C++; CPU_SETSIZE is defined exactly when it is exposed. The query fails on a machine
1587 // with more CPU ids than cpu_set_t holds.
1588 cpu_set_t allowed;
1589 if (sched_getaffinity(0, sizeof(allowed), &allowed) == 0) {
1590 return queryCpuCacheTopologySysfs(root,
1591 [&allowed](int cpu) { return cpu < CPU_SETSIZE && CPU_ISSET(cpu, &allowed); });
1592 }
1593#endif
1594 return queryCpuCacheTopologySysfs(root, [](int) { return true; });
1595}
1596
1597#endif // EIGEN_CPU_CACHE_SYSFS
1598
1602inline void queryCacheSizes(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3, std::ptrdiff_t& l3_per_cpu) {
1603 l3_per_cpu = 0;
1604#ifdef EIGEN_CPUID
1605 int abcd[4];
1606 const int GenuineIntel[] = {0x756e6547, 0x49656e69, 0x6c65746e};
1607 const int AuthenticAMD[] = {0x68747541, 0x69746e65, 0x444d4163};
1608 const int AMDisbetter_[] = {0x69444d41, 0x74656273, 0x21726574}; // "AMDisbetter!"
1609 // Hygon Dhyana is a Zen-based joint-venture line that returns "HygonGenuine"
1610 // and shares AMD's CPUID layout for cache descriptors (AMD Family 17h).
1611 const int HygonGenuine[] = {0x6f677948, 0x6e65476e, 0x656e6975}; // "HygonGenuine"
1612
1613 // identify the CPU vendor
1614 EIGEN_CPUID(abcd, 0x0, 0);
1615 int max_std_funcs = abcd[0];
1616 if (cpuid_is_vendor(abcd, GenuineIntel))
1617 queryCacheSizes_intel(l1, l2, l3, max_std_funcs);
1618 else if (cpuid_is_vendor(abcd, AuthenticAMD) || cpuid_is_vendor(abcd, AMDisbetter_) ||
1619 cpuid_is_vendor(abcd, HygonGenuine))
1620 queryCacheSizes_amd(l1, l2, l3);
1621 else
1622 // by default let's use Intel's API
1623 queryCacheSizes_intel(l1, l2, l3, max_std_funcs);
1624
1625 // here is the list of other vendors:
1626 // ||cpuid_is_vendor(abcd,"VIA VIA VIA ")
1627 // ||cpuid_is_vendor(abcd,"CyrixInstead")
1628 // ||cpuid_is_vendor(abcd,"CentaurHauls")
1629 // ||cpuid_is_vendor(abcd,"GenuineTMx86")
1630 // ||cpuid_is_vendor(abcd,"TransmetaCPU")
1631 // ||cpuid_is_vendor(abcd,"RiseRiseRise")
1632 // ||cpuid_is_vendor(abcd,"Geode by NSC")
1633 // ||cpuid_is_vendor(abcd,"SiS SiS SiS ")
1634 // ||cpuid_is_vendor(abcd,"UMC UMC UMC ")
1635 // ||cpuid_is_vendor(abcd,"NexGenDriven")
1636#elif EIGEN_OS_MAC
1637 // On macOS (including Apple Silicon), use sysctlbyname to query cache sizes.
1638 // The sysctl values are 64-bit, so read into int64_t and convert.
1639 // For L1, prefer P-core (perflevel0) size since compute-heavy work like GEMM
1640 // is typically scheduled on performance cores. L1 is per-core so always safe.
1641 // For L2, use the generic hw.l2cachesize which is more conservative (reports
1642 // the smaller E-core cluster L2 on heterogeneous chips). The P-core L2 is
1643 // shared among all P-cores and would overestimate per-core capacity.
1644 {
1645 int64_t val = 0;
1646 std::size_t val_size = sizeof(val);
1647 l1 = -1;
1648 val_size = sizeof(val);
1649 if (sysctlbyname("hw.perflevel0.l1dcachesize", &val, &val_size, nullptr, 0) == 0 && val > 0)
1650 l1 = val;
1651 else {
1652 val_size = sizeof(val);
1653 if (sysctlbyname("hw.l1dcachesize", &val, &val_size, nullptr, 0) == 0) l1 = val;
1654 }
1655 l2 = -1;
1656 val_size = sizeof(val);
1657 if (sysctlbyname("hw.l2cachesize", &val, &val_size, nullptr, 0) == 0) l2 = val;
1658 l3 = -1;
1659 val_size = sizeof(val);
1660 if (sysctlbyname("hw.l3cachesize", &val, &val_size, nullptr, 0) == 0 && val > 0) l3 = val;
1661 }
1662#elif EIGEN_OS_UNIX && defined(_SC_LEVEL1_DCACHE_SIZE)
1663 // A glibc extension: POSIX specifies no cache queries, and musl defines none of these names.
1664 l1 = sysconf(_SC_LEVEL1_DCACHE_SIZE);
1665 l2 = sysconf(_SC_LEVEL2_CACHE_SIZE);
1666 l3 = sysconf(_SC_LEVEL3_CACHE_SIZE);
1667#else
1668 l1 = l2 = l3 = -1;
1669#endif
1670#ifdef EIGEN_CPU_CACHE_SYSFS
1671 // glibc answers the _SC_LEVEL*_CACHE_SIZE queries from CPUID and so only implements them on x86; every
1672 // other architecture gets 0, and musl has no such queries at all. Whatever the platform left unknown comes
1673 // from the topology Linux publishes on every architecture, as does the L3 share, so that all four numbers
1674 // describe the same CPUs.
1675 const CpuCacheTopology topology = queryCpuCacheTopologySysfs();
1676 if (l1 <= 0) l1 = topology.l1;
1677 if (l2 <= 0) l2 = topology.l2;
1678 if (l3 <= 0) l3 = topology.l3;
1679 l3_per_cpu = topology.l3_per_cpu;
1680#endif
1681}
1682
1685inline void queryCacheSizes(std::ptrdiff_t& l1, std::ptrdiff_t& l2, std::ptrdiff_t& l3) {
1686 std::ptrdiff_t l3_per_cpu;
1687 queryCacheSizes(l1, l2, l3, l3_per_cpu);
1688}
1689
1692inline std::ptrdiff_t queryL1CacheSize() {
1693 std::ptrdiff_t l1(-1), l2, l3;
1694 queryCacheSizes(l1, l2, l3);
1695 return l1;
1696}
1697
1700inline std::ptrdiff_t queryTopLevelCacheSize() {
1701 std::ptrdiff_t l1, l2(-1), l3(-1);
1702 queryCacheSizes(l1, l2, l3);
1703 return (std::max)(l2, l3);
1704}
1705
1709
1710#if !defined(EIGEN_GPU_COMPILE_PHASE) && EIGEN_COMP_CXXVER >= 20 && defined(__cpp_lib_constexpr_dynamic_alloc) && \
1711 __cpp_lib_constexpr_dynamic_alloc >= 201907L
1712using std::construct_at;
1713#else
1714template <class T, class... Args>
1715EIGEN_DEVICE_FUNC T* construct_at(T* p, Args&&... args) {
1716 return ::new (const_cast<void*>(static_cast<const volatile void*>(p))) T(std::forward<Args>(args)...);
1717}
1718#endif
1719
1725#if EIGEN_COMP_CXXVER >= 17
1726using std::destroy_at;
1727#else
1728template <class T>
1729EIGEN_DEVICE_FUNC void destroy_at(T* p) {
1730 p->~T();
1731}
1732#endif
1733
1734// FIXME(rmlarsen): Work around missing linker symbol with msan on ARM.
1735#if !defined(EIGEN_DONT_ASSUME_ALIGNED) && __has_feature(memory_sanitizer) && (EIGEN_ARCH_ARM || EIGEN_ARCH_ARM64)
1736#define EIGEN_DONT_ASSUME_ALIGNED
1737#endif
1738
1739#if !defined(EIGEN_DONT_ASSUME_ALIGNED) && defined(__cpp_lib_assume_aligned) && (__cpp_lib_assume_aligned >= 201811L)
1740template <std::size_t N, typename T>
1741EIGEN_DEVICE_FUNC constexpr T* assume_aligned(T* ptr) {
1742 return std::assume_aligned<N, T>(ptr);
1743}
1744#elif !defined(EIGEN_DONT_ASSUME_ALIGNED) && EIGEN_HAS_BUILTIN(__builtin_assume_aligned)
1745template <std::size_t N, typename T>
1746EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC T* assume_aligned(T* ptr) {
1747 return static_cast<T*>(__builtin_assume_aligned(ptr, N));
1748}
1749#else
1750template <std::size_t N, typename T>
1751EIGEN_DEVICE_FUNC constexpr T* assume_aligned(T* ptr) {
1752 return ptr;
1753}
1754#endif
1755
1756} // end namespace internal
1757
1758} // end namespace Eigen
1759
1760#endif // EIGEN_MEMORY_H