Memory.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Copyright (C) 2008-2009 Benoit Jacob <jacob.benoit.1@gmail.com>
6// Copyright (C) 2009 Kenneth Riddile <kfriddile@yahoo.com>
7// Copyright (C) 2010 Hauke Heibel <hauke.heibel@gmail.com>
8// Copyright (C) 2010 Thomas Capricelli <orzel@freehackers.org>
9//
10// This Source Code Form is subject to the terms of the Mozilla
11// Public License v. 2.0. If a copy of the MPL was not distributed
12// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
13
14
15/*****************************************************************************
16*** Platform checks for aligned malloc functions ***
17*****************************************************************************/
18
19#ifndef EIGEN_MEMORY_H
20#define EIGEN_MEMORY_H
21
22// On 64-bit systems, glibc's malloc returns 16-byte-aligned pointers, see:
23// http://www.gnu.org/s/libc/manual/html_node/Aligned-Memory-Blocks.html
24// This is true at least since glibc 2.8.
25// This leaves the question how to detect 64-bit. According to this document,
26// http://gcc.fyxm.net/summit/2003/Porting%20to%2064%20bit.pdf
27// page 114, "[The] LP64 model [...] is used by all 64-bit UNIX ports" so it's indeed
28// quite safe, at least within the context of glibc, to equate 64-bit with LP64.
29#if defined(__GLIBC__) && ((__GLIBC__>=2 && __GLIBC_MINOR__ >= 8) || __GLIBC__>2) \
30 && defined(__LP64__)
31 #define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 1
32#else
33 #define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 0
34#endif
35
36// FreeBSD 6 seems to have 16-byte aligned malloc
37// See http://svn.freebsd.org/viewvc/base/stable/6/lib/libc/stdlib/malloc.c?view=markup
38// FreeBSD 7 seems to have 16-byte aligned malloc except on ARM and MIPS architectures
39// See http://svn.freebsd.org/viewvc/base/stable/7/lib/libc/stdlib/malloc.c?view=markup
40#if defined(__FreeBSD__) && !defined(__arm__) && !defined(__mips__)
41 #define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 1
42#else
43 #define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 0
44#endif
45
46#if defined(__APPLE__) \
47 || defined(_WIN64) \
48 || EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED \
49 || EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED
50 #define EIGEN_MALLOC_ALREADY_ALIGNED 1
51#else
52 #define EIGEN_MALLOC_ALREADY_ALIGNED 0
53#endif
54
55#if ((defined __QNXNTO__) || (defined _GNU_SOURCE) || ((defined _XOPEN_SOURCE) && (_XOPEN_SOURCE >= 600))) \
56 && (defined _POSIX_ADVISORY_INFO) && (_POSIX_ADVISORY_INFO > 0)
57 #define EIGEN_HAS_POSIX_MEMALIGN 1
58#else
59 #define EIGEN_HAS_POSIX_MEMALIGN 0
60#endif
61
62#ifdef EIGEN_VECTORIZE_SSE
63 #define EIGEN_HAS_MM_MALLOC 1
64#else
65 #define EIGEN_HAS_MM_MALLOC 0
66#endif
67
68namespace Eigen {
69
70namespace internal {
71
72inline void throw_std_bad_alloc()
73{
74 #ifdef EIGEN_EXCEPTIONS
75 throw std::bad_alloc();
76 #else
77 std::size_t huge = -1;
78 new int[huge];
79 #endif
80}
81
82/*****************************************************************************
83*** Implementation of handmade aligned functions ***
84*****************************************************************************/
85
86/* ----- Hand made implementations of aligned malloc/free and realloc ----- */
87
91inline void* handmade_aligned_malloc(std::size_t size)
92{
93 void *original = std::malloc(size+16);
94 if (original == 0) return 0;
95 void *aligned = reinterpret_cast<void*>((reinterpret_cast<std::size_t>(original) & ~(std::size_t(15))) + 16);
96 *(reinterpret_cast<void**>(aligned) - 1) = original;
97 return aligned;
98}
99
101inline void handmade_aligned_free(void *ptr)
102{
103 if (ptr) std::free(*(reinterpret_cast<void**>(ptr) - 1));
104}
105
111inline void* handmade_aligned_realloc(void* ptr, std::size_t size, std::size_t = 0)
112{
113 if (ptr == 0) return handmade_aligned_malloc(size);
114 void *original = *(reinterpret_cast<void**>(ptr) - 1);
115 std::ptrdiff_t previous_offset = static_cast<char *>(ptr)-static_cast<char *>(original);
116 original = std::realloc(original,size+16);
117 if (original == 0) return 0;
118 void *aligned = reinterpret_cast<void*>((reinterpret_cast<std::size_t>(original) & ~(std::size_t(15))) + 16);
119 void *previous_aligned = static_cast<char *>(original)+previous_offset;
120 if(aligned!=previous_aligned)
121 std::memmove(aligned, previous_aligned, size);
122
123 *(reinterpret_cast<void**>(aligned) - 1) = original;
124 return aligned;
125}
126
127/*****************************************************************************
128*** Implementation of generic aligned realloc (when no realloc can be used)***
129*****************************************************************************/
130
131void* aligned_malloc(std::size_t size);
132void aligned_free(void *ptr);
133
139inline void* generic_aligned_realloc(void* ptr, size_t size, size_t old_size)
140{
141 if (ptr==0)
142 return aligned_malloc(size);
143
144 if (size==0)
145 {
146 aligned_free(ptr);
147 return 0;
148 }
149
150 void* newptr = aligned_malloc(size);
151 if (newptr == 0)
152 {
153 #ifdef EIGEN_HAS_ERRNO
154 errno = ENOMEM; // according to the standard
155 #endif
156 return 0;
157 }
158
159 if (ptr != 0)
160 {
161 std::memcpy(newptr, ptr, (std::min)(size,old_size));
162 aligned_free(ptr);
163 }
164
165 return newptr;
166}
167
168/*****************************************************************************
169*** Implementation of portable aligned versions of malloc/free/realloc ***
170*****************************************************************************/
171
172#ifdef EIGEN_NO_MALLOC
173inline void check_that_malloc_is_allowed()
174{
175 eigen_assert(false && "heap allocation is forbidden (EIGEN_NO_MALLOC is defined)");
176}
177#elif defined EIGEN_RUNTIME_NO_MALLOC
178inline bool is_malloc_allowed_impl(bool update, bool new_value = false)
179{
180 static bool value = true;
181 if (update == 1)
182 value = new_value;
183 return value;
184}
185inline bool is_malloc_allowed() { return is_malloc_allowed_impl(false); }
186inline bool set_is_malloc_allowed(bool new_value) { return is_malloc_allowed_impl(true, new_value); }
187inline void check_that_malloc_is_allowed()
188{
189 eigen_assert(is_malloc_allowed() && "heap allocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and g_is_malloc_allowed is false)");
190}
191#else
192inline void check_that_malloc_is_allowed()
193{}
194#endif
195
199inline void* aligned_malloc(size_t size)
200{
201 check_that_malloc_is_allowed();
202
203 void *result;
204 #if !EIGEN_ALIGN
205 result = std::malloc(size);
206 #elif EIGEN_MALLOC_ALREADY_ALIGNED
207 result = std::malloc(size);
208 #elif EIGEN_HAS_POSIX_MEMALIGN
209 if(posix_memalign(&result, 16, size)) result = 0;
210 #elif EIGEN_HAS_MM_MALLOC
211 result = _mm_malloc(size, 16);
212#elif defined(_MSC_VER) && (!defined(_WIN32_WCE))
213 result = _aligned_malloc(size, 16);
214 #else
215 result = handmade_aligned_malloc(size);
216 #endif
217
218 if(!result && size)
219 throw_std_bad_alloc();
220
221 return result;
222}
223
225inline void aligned_free(void *ptr)
226{
227 #if !EIGEN_ALIGN
228 std::free(ptr);
229 #elif EIGEN_MALLOC_ALREADY_ALIGNED
230 std::free(ptr);
231 #elif EIGEN_HAS_POSIX_MEMALIGN
232 std::free(ptr);
233 #elif EIGEN_HAS_MM_MALLOC
234 _mm_free(ptr);
235 #elif defined(_MSC_VER) && (!defined(_WIN32_WCE))
236 _aligned_free(ptr);
237 #else
238 handmade_aligned_free(ptr);
239 #endif
240}
241
247inline void* aligned_realloc(void *ptr, size_t new_size, size_t old_size)
248{
249 EIGEN_UNUSED_VARIABLE(old_size);
250
251 void *result;
252#if !EIGEN_ALIGN
253 result = std::realloc(ptr,new_size);
254#elif EIGEN_MALLOC_ALREADY_ALIGNED
255 result = std::realloc(ptr,new_size);
256#elif EIGEN_HAS_POSIX_MEMALIGN
257 result = generic_aligned_realloc(ptr,new_size,old_size);
258#elif EIGEN_HAS_MM_MALLOC
259 // The defined(_mm_free) is just here to verify that this MSVC version
260 // implements _mm_malloc/_mm_free based on the corresponding _aligned_
261 // functions. This may not always be the case and we just try to be safe.
262 #if defined(_MSC_VER) && defined(_mm_free)
263 result = _aligned_realloc(ptr,new_size,16);
264 #else
265 result = generic_aligned_realloc(ptr,new_size,old_size);
266 #endif
267#elif defined(_MSC_VER)
268 result = _aligned_realloc(ptr,new_size,16);
269#else
270 result = handmade_aligned_realloc(ptr,new_size,old_size);
271#endif
272
273 if (!result && new_size)
274 throw_std_bad_alloc();
275
276 return result;
277}
278
279/*****************************************************************************
280*** Implementation of conditionally aligned functions ***
281*****************************************************************************/
282
286template<bool Align> inline void* conditional_aligned_malloc(size_t size)
287{
288 return aligned_malloc(size);
289}
290
291template<> inline void* conditional_aligned_malloc<false>(size_t size)
292{
293 check_that_malloc_is_allowed();
294
295 void *result = std::malloc(size);
296 if(!result && size)
297 throw_std_bad_alloc();
298 return result;
299}
300
302template<bool Align> inline void conditional_aligned_free(void *ptr)
303{
304 aligned_free(ptr);
305}
306
307template<> inline void conditional_aligned_free<false>(void *ptr)
308{
309 std::free(ptr);
310}
311
312template<bool Align> inline void* conditional_aligned_realloc(void* ptr, size_t new_size, size_t old_size)
313{
314 return aligned_realloc(ptr, new_size, old_size);
315}
316
317template<> inline void* conditional_aligned_realloc<false>(void* ptr, size_t new_size, size_t)
318{
319 return std::realloc(ptr, new_size);
320}
321
322/*****************************************************************************
323*** Construction/destruction of array elements ***
324*****************************************************************************/
325
329template<typename T> inline T* construct_elements_of_array(T *ptr, size_t size)
330{
331 for (size_t i=0; i < size; ++i) ::new (ptr + i) T;
332 return ptr;
333}
334
338template<typename T> inline void destruct_elements_of_array(T *ptr, size_t size)
339{
340 // always destruct an array starting from the end.
341 if(ptr)
342 while(size) ptr[--size].~T();
343}
344
345/*****************************************************************************
346*** Implementation of aligned new/delete-like functions ***
347*****************************************************************************/
348
349template<typename T>
350EIGEN_ALWAYS_INLINE void check_size_for_overflow(size_t size)
351{
352 if(size > size_t(-1) / sizeof(T))
353 throw_std_bad_alloc();
354}
355
360template<typename T> inline T* aligned_new(size_t size)
361{
362 check_size_for_overflow<T>(size);
363 T *result = reinterpret_cast<T*>(aligned_malloc(sizeof(T)*size));
364 return construct_elements_of_array(result, size);
365}
366
367template<typename T, bool Align> inline T* conditional_aligned_new(size_t size)
368{
369 check_size_for_overflow<T>(size);
370 T *result = reinterpret_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T)*size));
371 return construct_elements_of_array(result, size);
372}
373
377template<typename T> inline void aligned_delete(T *ptr, size_t size)
378{
379 destruct_elements_of_array<T>(ptr, size);
380 aligned_free(ptr);
381}
382
386template<typename T, bool Align> inline void conditional_aligned_delete(T *ptr, size_t size)
387{
388 destruct_elements_of_array<T>(ptr, size);
389 conditional_aligned_free<Align>(ptr);
390}
391
392template<typename T, bool Align> inline T* conditional_aligned_realloc_new(T* pts, size_t new_size, size_t old_size)
393{
394 check_size_for_overflow<T>(new_size);
395 check_size_for_overflow<T>(old_size);
396 if(new_size < old_size)
397 destruct_elements_of_array(pts+new_size, old_size-new_size);
398 T *result = reinterpret_cast<T*>(conditional_aligned_realloc<Align>(reinterpret_cast<void*>(pts), sizeof(T)*new_size, sizeof(T)*old_size));
399 if(new_size > old_size)
400 construct_elements_of_array(result+old_size, new_size-old_size);
401 return result;
402}
403
404
405template<typename T, bool Align> inline T* conditional_aligned_new_auto(size_t size)
406{
407 check_size_for_overflow<T>(size);
408 T *result = reinterpret_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T)*size));
409 if(NumTraits<T>::RequireInitialization)
410 construct_elements_of_array(result, size);
411 return result;
412}
413
414template<typename T, bool Align> inline T* conditional_aligned_realloc_new_auto(T* pts, size_t new_size, size_t old_size)
415{
416 check_size_for_overflow<T>(new_size);
417 check_size_for_overflow<T>(old_size);
418 if(NumTraits<T>::RequireInitialization && (new_size < old_size))
419 destruct_elements_of_array(pts+new_size, old_size-new_size);
420 T *result = reinterpret_cast<T*>(conditional_aligned_realloc<Align>(reinterpret_cast<void*>(pts), sizeof(T)*new_size, sizeof(T)*old_size));
421 if(NumTraits<T>::RequireInitialization && (new_size > old_size))
422 construct_elements_of_array(result+old_size, new_size-old_size);
423 return result;
424}
425
426template<typename T, bool Align> inline void conditional_aligned_delete_auto(T *ptr, size_t size)
427{
428 if(NumTraits<T>::RequireInitialization)
429 destruct_elements_of_array<T>(ptr, size);
430 conditional_aligned_free<Align>(ptr);
431}
432
433/****************************************************************************/
434
451template<typename Scalar, typename Index>
452static inline Index first_aligned(const Scalar* array, Index size)
453{
454 enum { PacketSize = packet_traits<Scalar>::size,
455 PacketAlignedMask = PacketSize-1
456 };
457
458 if(PacketSize==1)
459 {
460 // Either there is no vectorization, or a packet consists of exactly 1 scalar so that all elements
461 // of the array have the same alignment.
462 return 0;
463 }
464 else if(size_t(array) & (sizeof(Scalar)-1))
465 {
466 // There is vectorization for this scalar type, but the array is not aligned to the size of a single scalar.
467 // Consequently, no element of the array is well aligned.
468 return size;
469 }
470 else
471 {
472 return std::min<Index>( (PacketSize - (Index((size_t(array)/sizeof(Scalar))) & PacketAlignedMask))
473 & PacketAlignedMask, size);
474 }
475}
476
477
478// std::copy is much slower than memcpy, so let's introduce a smart_copy which
479// use memcpy on trivial types, i.e., on types that does not require an initialization ctor.
480template<typename T, bool UseMemcpy> struct smart_copy_helper;
481
482template<typename T> void smart_copy(const T* start, const T* end, T* target)
483{
484 smart_copy_helper<T,!NumTraits<T>::RequireInitialization>::run(start, end, target);
485}
486
487template<typename T> struct smart_copy_helper<T,true> {
488 static inline void run(const T* start, const T* end, T* target)
489 { memcpy(target, start, std::ptrdiff_t(end)-std::ptrdiff_t(start)); }
490};
491
492template<typename T> struct smart_copy_helper<T,false> {
493 static inline void run(const T* start, const T* end, T* target)
494 { std::copy(start, end, target); }
495};
496
497
498/*****************************************************************************
499*** Implementation of runtime stack allocation (falling back to malloc) ***
500*****************************************************************************/
501
502// you can overwrite Eigen's default behavior regarding alloca by defining EIGEN_ALLOCA
503// to the appropriate stack allocation function
504#ifndef EIGEN_ALLOCA
505 #if (defined __linux__)
506 #define EIGEN_ALLOCA alloca
507 #elif defined(_MSC_VER)
508 #define EIGEN_ALLOCA _alloca
509 #endif
510#endif
511
512// This helper class construct the allocated memory, and takes care of destructing and freeing the handled data
513// at destruction time. In practice this helper class is mainly useful to avoid memory leak in case of exceptions.
514template<typename T> class aligned_stack_memory_handler
515{
516 public:
517 /* Creates a stack_memory_handler responsible for the buffer \a ptr of size \a size.
518 * Note that \a ptr can be 0 regardless of the other parameters.
519 * This constructor takes care of constructing/initializing the elements of the buffer if required by the scalar type T (see NumTraits<T>::RequireInitialization).
520 * In this case, the buffer elements will also be destructed when this handler will be destructed.
521 * Finally, if \a dealloc is true, then the pointer \a ptr is freed.
522 **/
523 aligned_stack_memory_handler(T* ptr, size_t size, bool dealloc)
524 : m_ptr(ptr), m_size(size), m_deallocate(dealloc)
525 {
526 if(NumTraits<T>::RequireInitialization && m_ptr)
527 Eigen::internal::construct_elements_of_array(m_ptr, size);
528 }
529 ~aligned_stack_memory_handler()
530 {
531 if(NumTraits<T>::RequireInitialization && m_ptr)
532 Eigen::internal::destruct_elements_of_array<T>(m_ptr, m_size);
533 if(m_deallocate)
534 Eigen::internal::aligned_free(m_ptr);
535 }
536 protected:
537 T* m_ptr;
538 size_t m_size;
539 bool m_deallocate;
540};
541
542} // end namespace internal
543
559#ifdef EIGEN_ALLOCA
560
561 #ifdef __arm__
562 #define EIGEN_ALIGNED_ALLOCA(SIZE) reinterpret_cast<void*>((reinterpret_cast<size_t>(EIGEN_ALLOCA(SIZE+16)) & ~(size_t(15))) + 16)
563 #else
564 #define EIGEN_ALIGNED_ALLOCA EIGEN_ALLOCA
565 #endif
566
567 #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \
568 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
569 TYPE* NAME = (BUFFER)!=0 ? (BUFFER) \
570 : reinterpret_cast<TYPE*>( \
571 (sizeof(TYPE)*SIZE<=EIGEN_STACK_ALLOCATION_LIMIT) ? EIGEN_ALIGNED_ALLOCA(sizeof(TYPE)*SIZE) \
572 : Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE) ); \
573 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,sizeof(TYPE)*SIZE>EIGEN_STACK_ALLOCATION_LIMIT)
574
575#else
576
577 #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \
578 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
579 TYPE* NAME = (BUFFER)!=0 ? BUFFER : reinterpret_cast<TYPE*>(Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE)); \
580 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,true)
581
582#endif
583
584
585/*****************************************************************************
586*** Implementation of EIGEN_MAKE_ALIGNED_OPERATOR_NEW [_IF] ***
587*****************************************************************************/
588
589#if EIGEN_ALIGN
590 #ifdef EIGEN_EXCEPTIONS
591 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
592 void* operator new(size_t size, const std::nothrow_t&) throw() { \
593 try { return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); } \
594 catch (...) { return 0; } \
595 return 0; \
596 }
597 #else
598 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
599 void* operator new(size_t size, const std::nothrow_t&) throw() { \
600 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
601 }
602 #endif
603
604 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) \
605 void *operator new(size_t size) { \
606 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
607 } \
608 void *operator new[](size_t size) { \
609 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
610 } \
611 void operator delete(void * ptr) throw() { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
612 void operator delete[](void * ptr) throw() { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
613 /* in-place new and delete. since (at least afaik) there is no actual */ \
614 /* memory allocated we can safely let the default implementation handle */ \
615 /* this particular case. */ \
616 static void *operator new(size_t size, void *ptr) { return ::operator new(size,ptr); } \
617 void operator delete(void * memory, void *ptr) throw() { return ::operator delete(memory,ptr); } \
618 /* nothrow-new (returns zero instead of std::bad_alloc) */ \
619 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
620 void operator delete(void *ptr, const std::nothrow_t&) throw() { \
621 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
622 } \
623 typedef void eigen_aligned_operator_new_marker_type;
624#else
625 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
626#endif
627
628#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(true)
629#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar,Size) \
630 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(bool(((Size)!=Eigen::Dynamic) && ((sizeof(Scalar)*(Size))%16==0)))
631
632/****************************************************************************/
633
650template<class T>
651class aligned_allocator
652{
653public:
654 typedef size_t size_type;
655 typedef std::ptrdiff_t difference_type;
656 typedef T* pointer;
657 typedef const T* const_pointer;
658 typedef T& reference;
659 typedef const T& const_reference;
660 typedef T value_type;
661
662 template<class U>
663 struct rebind
664 {
665 typedef aligned_allocator<U> other;
666 };
667
668 pointer address( reference value ) const
669 {
670 return &value;
671 }
672
673 const_pointer address( const_reference value ) const
674 {
675 return &value;
676 }
677
678 aligned_allocator()
679 {
680 }
681
682 aligned_allocator( const aligned_allocator& )
683 {
684 }
685
686 template<class U>
687 aligned_allocator( const aligned_allocator<U>& )
688 {
689 }
690
691 ~aligned_allocator()
692 {
693 }
694
695 size_type max_size() const
696 {
697 return (std::numeric_limits<size_type>::max)();
698 }
699
700 pointer allocate( size_type num, const void* hint = 0 )
701 {
702 EIGEN_UNUSED_VARIABLE(hint);
703 internal::check_size_for_overflow<T>(num);
704 return static_cast<pointer>( internal::aligned_malloc( num * sizeof(T) ) );
705 }
706
707 void construct( pointer p, const T& value )
708 {
709 ::new( p ) T( value );
710 }
711
712 void destroy( pointer p )
713 {
714 p->~T();
715 }
716
717 void deallocate( pointer p, size_type /*num*/ )
718 {
719 internal::aligned_free( p );
720 }
721
722 bool operator!=(const aligned_allocator<T>& ) const
723 { return false; }
724
725 bool operator==(const aligned_allocator<T>& ) const
726 { return true; }
727};
728
729//---------- Cache sizes ----------
730
731#if !defined(EIGEN_NO_CPUID)
732# if defined(__GNUC__) && ( defined(__i386__) || defined(__x86_64__) )
733# if defined(__PIC__) && defined(__i386__)
734 // Case for x86 with PIC
735# define EIGEN_CPUID(abcd,func,id) \
736 __asm__ __volatile__ ("xchgl %%ebx, %%esi;cpuid; xchgl %%ebx,%%esi": "=a" (abcd[0]), "=S" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "a" (func), "c" (id));
737# else
738 // Case for x86_64 or x86 w/o PIC
739# define EIGEN_CPUID(abcd,func,id) \
740 __asm__ __volatile__ ("cpuid": "=a" (abcd[0]), "=b" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "a" (func), "c" (id) );
741# endif
742# elif defined(_MSC_VER)
743# if (_MSC_VER > 1500) && ( defined(_M_IX86) || defined(_M_X64) )
744# define EIGEN_CPUID(abcd,func,id) __cpuidex((int*)abcd,func,id)
745# endif
746# endif
747#endif
748
749namespace internal {
750
751#ifdef EIGEN_CPUID
752
753inline bool cpuid_is_vendor(int abcd[4], const char* vendor)
754{
755 return abcd[1]==(reinterpret_cast<const int*>(vendor))[0] && abcd[3]==(reinterpret_cast<const int*>(vendor))[1] && abcd[2]==(reinterpret_cast<const int*>(vendor))[2];
756}
757
758inline void queryCacheSizes_intel_direct(int& l1, int& l2, int& l3)
759{
760 int abcd[4];
761 l1 = l2 = l3 = 0;
762 int cache_id = 0;
763 int cache_type = 0;
764 do {
765 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
766 EIGEN_CPUID(abcd,0x4,cache_id);
767 cache_type = (abcd[0] & 0x0F) >> 0;
768 if(cache_type==1||cache_type==3) // data or unified cache
769 {
770 int cache_level = (abcd[0] & 0xE0) >> 5; // A[7:5]
771 int ways = (abcd[1] & 0xFFC00000) >> 22; // B[31:22]
772 int partitions = (abcd[1] & 0x003FF000) >> 12; // B[21:12]
773 int line_size = (abcd[1] & 0x00000FFF) >> 0; // B[11:0]
774 int sets = (abcd[2]); // C[31:0]
775
776 int cache_size = (ways+1) * (partitions+1) * (line_size+1) * (sets+1);
777
778 switch(cache_level)
779 {
780 case 1: l1 = cache_size; break;
781 case 2: l2 = cache_size; break;
782 case 3: l3 = cache_size; break;
783 default: break;
784 }
785 }
786 cache_id++;
787 } while(cache_type>0 && cache_id<16);
788}
789
790inline void queryCacheSizes_intel_codes(int& l1, int& l2, int& l3)
791{
792 int abcd[4];
793 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
794 l1 = l2 = l3 = 0;
795 EIGEN_CPUID(abcd,0x00000002,0);
796 unsigned char * bytes = reinterpret_cast<unsigned char *>(abcd)+2;
797 bool check_for_p2_core2 = false;
798 for(int i=0; i<14; ++i)
799 {
800 switch(bytes[i])
801 {
802 case 0x0A: l1 = 8; break; // 0Ah data L1 cache, 8 KB, 2 ways, 32 byte lines
803 case 0x0C: l1 = 16; break; // 0Ch data L1 cache, 16 KB, 4 ways, 32 byte lines
804 case 0x0E: l1 = 24; break; // 0Eh data L1 cache, 24 KB, 6 ways, 64 byte lines
805 case 0x10: l1 = 16; break; // 10h data L1 cache, 16 KB, 4 ways, 32 byte lines (IA-64)
806 case 0x15: l1 = 16; break; // 15h code L1 cache, 16 KB, 4 ways, 32 byte lines (IA-64)
807 case 0x2C: l1 = 32; break; // 2Ch data L1 cache, 32 KB, 8 ways, 64 byte lines
808 case 0x30: l1 = 32; break; // 30h code L1 cache, 32 KB, 8 ways, 64 byte lines
809 case 0x60: l1 = 16; break; // 60h data L1 cache, 16 KB, 8 ways, 64 byte lines, sectored
810 case 0x66: l1 = 8; break; // 66h data L1 cache, 8 KB, 4 ways, 64 byte lines, sectored
811 case 0x67: l1 = 16; break; // 67h data L1 cache, 16 KB, 4 ways, 64 byte lines, sectored
812 case 0x68: l1 = 32; break; // 68h data L1 cache, 32 KB, 4 ways, 64 byte lines, sectored
813 case 0x1A: l2 = 96; break; // code and data L2 cache, 96 KB, 6 ways, 64 byte lines (IA-64)
814 case 0x22: l3 = 512; break; // code and data L3 cache, 512 KB, 4 ways (!), 64 byte lines, dual-sectored
815 case 0x23: l3 = 1024; break; // code and data L3 cache, 1024 KB, 8 ways, 64 byte lines, dual-sectored
816 case 0x25: l3 = 2048; break; // code and data L3 cache, 2048 KB, 8 ways, 64 byte lines, dual-sectored
817 case 0x29: l3 = 4096; break; // code and data L3 cache, 4096 KB, 8 ways, 64 byte lines, dual-sectored
818 case 0x39: l2 = 128; break; // code and data L2 cache, 128 KB, 4 ways, 64 byte lines, sectored
819 case 0x3A: l2 = 192; break; // code and data L2 cache, 192 KB, 6 ways, 64 byte lines, sectored
820 case 0x3B: l2 = 128; break; // code and data L2 cache, 128 KB, 2 ways, 64 byte lines, sectored
821 case 0x3C: l2 = 256; break; // code and data L2 cache, 256 KB, 4 ways, 64 byte lines, sectored
822 case 0x3D: l2 = 384; break; // code and data L2 cache, 384 KB, 6 ways, 64 byte lines, sectored
823 case 0x3E: l2 = 512; break; // code and data L2 cache, 512 KB, 4 ways, 64 byte lines, sectored
824 case 0x40: l2 = 0; break; // no integrated L2 cache (P6 core) or L3 cache (P4 core)
825 case 0x41: l2 = 128; break; // code and data L2 cache, 128 KB, 4 ways, 32 byte lines
826 case 0x42: l2 = 256; break; // code and data L2 cache, 256 KB, 4 ways, 32 byte lines
827 case 0x43: l2 = 512; break; // code and data L2 cache, 512 KB, 4 ways, 32 byte lines
828 case 0x44: l2 = 1024; break; // code and data L2 cache, 1024 KB, 4 ways, 32 byte lines
829 case 0x45: l2 = 2048; break; // code and data L2 cache, 2048 KB, 4 ways, 32 byte lines
830 case 0x46: l3 = 4096; break; // code and data L3 cache, 4096 KB, 4 ways, 64 byte lines
831 case 0x47: l3 = 8192; break; // code and data L3 cache, 8192 KB, 8 ways, 64 byte lines
832 case 0x48: l2 = 3072; break; // code and data L2 cache, 3072 KB, 12 ways, 64 byte lines
833 case 0x49: if(l2!=0) l3 = 4096; else {check_for_p2_core2=true; l3 = l2 = 4096;} break;// code and data L3 cache, 4096 KB, 16 ways, 64 byte lines (P4) or L2 for core2
834 case 0x4A: l3 = 6144; break; // code and data L3 cache, 6144 KB, 12 ways, 64 byte lines
835 case 0x4B: l3 = 8192; break; // code and data L3 cache, 8192 KB, 16 ways, 64 byte lines
836 case 0x4C: l3 = 12288; break; // code and data L3 cache, 12288 KB, 12 ways, 64 byte lines
837 case 0x4D: l3 = 16384; break; // code and data L3 cache, 16384 KB, 16 ways, 64 byte lines
838 case 0x4E: l2 = 6144; break; // code and data L2 cache, 6144 KB, 24 ways, 64 byte lines
839 case 0x78: l2 = 1024; break; // code and data L2 cache, 1024 KB, 4 ways, 64 byte lines
840 case 0x79: l2 = 128; break; // code and data L2 cache, 128 KB, 8 ways, 64 byte lines, dual-sectored
841 case 0x7A: l2 = 256; break; // code and data L2 cache, 256 KB, 8 ways, 64 byte lines, dual-sectored
842 case 0x7B: l2 = 512; break; // code and data L2 cache, 512 KB, 8 ways, 64 byte lines, dual-sectored
843 case 0x7C: l2 = 1024; break; // code and data L2 cache, 1024 KB, 8 ways, 64 byte lines, dual-sectored
844 case 0x7D: l2 = 2048; break; // code and data L2 cache, 2048 KB, 8 ways, 64 byte lines
845 case 0x7E: l2 = 256; break; // code and data L2 cache, 256 KB, 8 ways, 128 byte lines, sect. (IA-64)
846 case 0x7F: l2 = 512; break; // code and data L2 cache, 512 KB, 2 ways, 64 byte lines
847 case 0x80: l2 = 512; break; // code and data L2 cache, 512 KB, 8 ways, 64 byte lines
848 case 0x81: l2 = 128; break; // code and data L2 cache, 128 KB, 8 ways, 32 byte lines
849 case 0x82: l2 = 256; break; // code and data L2 cache, 256 KB, 8 ways, 32 byte lines
850 case 0x83: l2 = 512; break; // code and data L2 cache, 512 KB, 8 ways, 32 byte lines
851 case 0x84: l2 = 1024; break; // code and data L2 cache, 1024 KB, 8 ways, 32 byte lines
852 case 0x85: l2 = 2048; break; // code and data L2 cache, 2048 KB, 8 ways, 32 byte lines
853 case 0x86: l2 = 512; break; // code and data L2 cache, 512 KB, 4 ways, 64 byte lines
854 case 0x87: l2 = 1024; break; // code and data L2 cache, 1024 KB, 8 ways, 64 byte lines
855 case 0x88: l3 = 2048; break; // code and data L3 cache, 2048 KB, 4 ways, 64 byte lines (IA-64)
856 case 0x89: l3 = 4096; break; // code and data L3 cache, 4096 KB, 4 ways, 64 byte lines (IA-64)
857 case 0x8A: l3 = 8192; break; // code and data L3 cache, 8192 KB, 4 ways, 64 byte lines (IA-64)
858 case 0x8D: l3 = 3072; break; // code and data L3 cache, 3072 KB, 12 ways, 128 byte lines (IA-64)
859
860 default: break;
861 }
862 }
863 if(check_for_p2_core2 && l2 == l3)
864 l3 = 0;
865 l1 *= 1024;
866 l2 *= 1024;
867 l3 *= 1024;
868}
869
870inline void queryCacheSizes_intel(int& l1, int& l2, int& l3, int max_std_funcs)
871{
872 if(max_std_funcs>=4)
873 queryCacheSizes_intel_direct(l1,l2,l3);
874 else
875 queryCacheSizes_intel_codes(l1,l2,l3);
876}
877
878inline void queryCacheSizes_amd(int& l1, int& l2, int& l3)
879{
880 int abcd[4];
881 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
882 EIGEN_CPUID(abcd,0x80000005,0);
883 l1 = (abcd[2] >> 24) * 1024; // C[31:24] = L1 size in KB
884 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
885 EIGEN_CPUID(abcd,0x80000006,0);
886 l2 = (abcd[2] >> 16) * 1024; // C[31;16] = l2 cache size in KB
887 l3 = ((abcd[3] & 0xFFFC000) >> 18) * 512 * 1024; // D[31;18] = l3 cache size in 512KB
888}
889#endif
890
893inline void queryCacheSizes(int& l1, int& l2, int& l3)
894{
895 #ifdef EIGEN_CPUID
896 int abcd[4];
897
898 // identify the CPU vendor
899 EIGEN_CPUID(abcd,0x0,0);
900 int max_std_funcs = abcd[1];
901 if(cpuid_is_vendor(abcd,"GenuineIntel"))
902 queryCacheSizes_intel(l1,l2,l3,max_std_funcs);
903 else if(cpuid_is_vendor(abcd,"AuthenticAMD") || cpuid_is_vendor(abcd,"AMDisbetter!"))
904 queryCacheSizes_amd(l1,l2,l3);
905 else
906 // by default let's use Intel's API
907 queryCacheSizes_intel(l1,l2,l3,max_std_funcs);
908
909 // here is the list of other vendors:
910// ||cpuid_is_vendor(abcd,"VIA VIA VIA ")
911// ||cpuid_is_vendor(abcd,"CyrixInstead")
912// ||cpuid_is_vendor(abcd,"CentaurHauls")
913// ||cpuid_is_vendor(abcd,"GenuineTMx86")
914// ||cpuid_is_vendor(abcd,"TransmetaCPU")
915// ||cpuid_is_vendor(abcd,"RiseRiseRise")
916// ||cpuid_is_vendor(abcd,"Geode by NSC")
917// ||cpuid_is_vendor(abcd,"SiS SiS SiS ")
918// ||cpuid_is_vendor(abcd,"UMC UMC UMC ")
919// ||cpuid_is_vendor(abcd,"NexGenDriven")
920 #else
921 l1 = l2 = l3 = -1;
922 #endif
923}
924
927inline int queryL1CacheSize()
928{
929 int l1(-1), l2, l3;
930 queryCacheSizes(l1,l2,l3);
931 return l1;
932}
933
936inline int queryTopLevelCacheSize()
937{
938 int l1, l2(-1), l3(-1);
939 queryCacheSizes(l1,l2,l3);
940 return (std::max)(l2,l3);
941}
942
943} // end namespace internal
944
945} // end namespace Eigen
946
947#endif // EIGEN_MEMORY_H
Eigen::internal
Definition LDLT.h:18