Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
ConfigureVectorization.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2018 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Copyright (C) 2020, Arm Limited and Contributors
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_CONFIGURE_VECTORIZATION_H
13#define EIGEN_CONFIGURE_VECTORIZATION_H
14
15// Prepare for using the generic clang backend if requested.
16#if defined(EIGEN_VECTORIZE_GENERIC) && !defined(EIGEN_GENERIC_VECTOR_SIZE_BYTES)
17#define EIGEN_GENERIC_VECTOR_SIZE_BYTES 64
18#endif
19#if defined(EIGEN_VECTORIZE_GENERIC) && !defined(EIGEN_DONT_VECTORIZE) && !defined(EIGEN_DONT_ALIGN)
20#if !EIGEN_ARCH_VECTOR_EXTENSIONS
21#error "The compiler does not support clang vector extensions."
22#endif
23#define EIGEN_VECTORIZE
24#define EIGEN_MAX_ALIGN_BYTES EIGEN_GENERIC_VECTOR_SIZE_BYTES
25#endif
26
27//------------------------------------------------------------------------------------------
28// Static and dynamic alignment control
29//
30// The main purpose of this section is to define EIGEN_MAX_ALIGN_BYTES and EIGEN_MAX_STATIC_ALIGN_BYTES
31// as the maximal boundary in bytes on which dynamically and statically allocated data may be alignment respectively.
32// The values of EIGEN_MAX_ALIGN_BYTES and EIGEN_MAX_STATIC_ALIGN_BYTES can be specified by the user. If not,
33// a default value is automatically computed based on architecture, compiler, and OS.
34//
35// This section also defines macros EIGEN_ALIGN_TO_BOUNDARY(N) and the shortcuts EIGEN_ALIGN{8,16,32,_MAX}
36// to be used to declare statically aligned buffers.
37//------------------------------------------------------------------------------------------
38
39/* EIGEN_ALIGN_TO_BOUNDARY(n) forces data to be n-byte aligned. This is used to satisfy SIMD requirements.
40 * However, we do that EVEN if vectorization (EIGEN_VECTORIZE) is disabled,
41 * so that vectorization doesn't affect binary compatibility.
42 *
43 * If we made alignment depend on whether or not EIGEN_VECTORIZE is defined, it would be impossible to link
44 * vectorized and non-vectorized code.
45 */
46#if (defined EIGEN_CUDACC)
47#define EIGEN_ALIGN_TO_BOUNDARY(n) __align__(n)
48#define EIGEN_ALIGNOF(x) __alignof(x)
49#else
50#define EIGEN_ALIGN_TO_BOUNDARY(n) alignas(n)
51#define EIGEN_ALIGNOF(x) alignof(x)
52#endif
53
54#if defined(__riscv)
55// Defines the default LMUL for RISC-V
56#ifndef EIGEN_RISCV64_DEFAULT_LMUL
57#define EIGEN_RISCV64_DEFAULT_LMUL 1
58#endif
59#endif
60
61// Align to the boundary that avoids false sharing.
62// Pinned to 128 bytes to preserve a stable ABI across architectures and standard libraries
63// and avoid GCC -Winterference-size warnings.
64#ifndef EIGEN_ALIGN_TO_AVOID_FALSE_SHARING
65#define EIGEN_ALIGN_TO_AVOID_FALSE_SHARING EIGEN_ALIGN_TO_BOUNDARY(128)
66#endif
67
68// If the user explicitly disable vectorization, then we also disable alignment
69#if defined(EIGEN_DONT_VECTORIZE)
70#if defined(EIGEN_GPUCC)
71// GPU code is always vectorized and requires memory alignment for
72// statically allocated buffers.
73#define EIGEN_IDEAL_MAX_ALIGN_BYTES 16
74#else
75#define EIGEN_IDEAL_MAX_ALIGN_BYTES 0
76#endif
77#elif defined(EIGEN_VECTORIZE_GENERIC)
78// Generic clang backend overrides native SIMD; align to the generic vector size.
79#define EIGEN_IDEAL_MAX_ALIGN_BYTES EIGEN_GENERIC_VECTOR_SIZE_BYTES
80#elif defined(__AVX512F__)
81// 64 bytes static alignment is preferred only if really required
82#define EIGEN_IDEAL_MAX_ALIGN_BYTES 64
83// Deliberately no SME case: this block runs long before EIGEN_VECTORIZE_SME is
84// defined, so a test for it here is unreachable -- and moving it would be wrong
85// rather than merely late. Raising the value changes the alignment, and so the
86// ABI, of every fixed-size object relative to a NEON build of the same headers;
87// past 16 bytes aligned_malloc also switches to the prefixed
88// handmade_aligned_malloc, so a matrix allocated in an SME translation unit and
89// freed in a non-SME one corrupts the heap. Keeping 16 is not free: the SME GEMM
90// kernel is up to 3.4x faster at small sizes when the result matrix starts on a
91// 64-byte boundary, which is what 64 here would give every Eigen-owned matrix.
92#elif defined(EIGEN_ARM64_USE_SVE) && defined(__ARM_FEATURE_SVE_BITS) && (__ARM_FEATURE_SVE_BITS != 0)
93// A fixed-length SVE packet is __ARM_FEATURE_SVE_BITS/8 bytes wide and asks for
94// exactly that much alignment; a fixed-size object has to be able to offer it or
95// it never reaches the vectorized path. The Alignment enum stops at 128.
96#if __ARM_FEATURE_SVE_BITS <= 1024
97#define EIGEN_IDEAL_MAX_ALIGN_BYTES (__ARM_FEATURE_SVE_BITS / 8)
98#else
99#define EIGEN_IDEAL_MAX_ALIGN_BYTES 128
100#endif
101#elif defined(__AVX__)
102// 32 bytes static alignment is preferred only if really required
103#define EIGEN_IDEAL_MAX_ALIGN_BYTES 32
104#elif defined __HVX__ && (__HVX_LENGTH__ == 128)
105#define EIGEN_IDEAL_MAX_ALIGN_BYTES 128
106#elif defined(EIGEN_RISCV64_USE_RVV10) && defined(__riscv_v_fixed_vlen)
107// As for SVE: a default packet is VLEN * LMUL / 8 bytes and asks for that much alignment, which a
108// fixed-size object must be able to offer to reach the aligned path. The Alignment enum stops at 128.
109// Without a fixed VLEN, falling through to 16 keeps vectorization on, so the RVV block below reports
110// the missing -mrvv-vector-bits=zvl; evaluating the formula would give 0 and disable that block.
111#if (__riscv_v_fixed_vlen * EIGEN_RISCV64_DEFAULT_LMUL / 8) <= 128
112#define EIGEN_IDEAL_MAX_ALIGN_BYTES (__riscv_v_fixed_vlen * EIGEN_RISCV64_DEFAULT_LMUL / 8)
113#else
114#define EIGEN_IDEAL_MAX_ALIGN_BYTES 128
115#endif
116#else
117#define EIGEN_IDEAL_MAX_ALIGN_BYTES 16
118#endif
119
120// EIGEN_MIN_ALIGN_BYTES defines the minimal value for which the notion of explicit alignment makes sense
121#define EIGEN_MIN_ALIGN_BYTES 16
122
123// Defined the boundary (in bytes) on which the data needs to be aligned. Note
124// that unless EIGEN_ALIGN is defined and not equal to 0, the data may not be
125// aligned at all regardless of the value of this #define.
126
127#if (defined(EIGEN_DONT_ALIGN_STATICALLY) || defined(EIGEN_DONT_ALIGN)) && defined(EIGEN_MAX_STATIC_ALIGN_BYTES) && \
128 EIGEN_MAX_STATIC_ALIGN_BYTES > 0
129#error EIGEN_MAX_STATIC_ALIGN_BYTES and EIGEN_DONT_ALIGN[_STATICALLY] are both defined with EIGEN_MAX_STATIC_ALIGN_BYTES!=0. Use EIGEN_MAX_STATIC_ALIGN_BYTES=0 as a synonym of EIGEN_DONT_ALIGN_STATICALLY.
130#endif
131
132// EIGEN_DONT_ALIGN_STATICALLY and EIGEN_DONT_ALIGN are deprecated
133// They imply EIGEN_MAX_STATIC_ALIGN_BYTES=0
134#if defined(EIGEN_DONT_ALIGN_STATICALLY) || defined(EIGEN_DONT_ALIGN)
135#ifdef EIGEN_MAX_STATIC_ALIGN_BYTES
136#undef EIGEN_MAX_STATIC_ALIGN_BYTES
137#endif
138#define EIGEN_MAX_STATIC_ALIGN_BYTES 0
139#endif
140
141#ifndef EIGEN_MAX_STATIC_ALIGN_BYTES
142
143// Try to automatically guess what is the best default value for EIGEN_MAX_STATIC_ALIGN_BYTES
144
145// 16 byte alignment is only useful for vectorization. Since it affects the ABI, we need to enable
146// 16 byte alignment on all platforms where vectorization might be enabled. In theory we could always
147// enable alignment, but it can be a cause of problems on some platforms, so we just disable it in
148// certain common platform (compiler+architecture combinations) to avoid these problems.
149// Only static alignment is really problematic (relies on nonstandard compiler extensions),
150// try to keep heap alignment even when we have to disable static alignment.
151#if EIGEN_COMP_GNUC && !(EIGEN_ARCH_i386_OR_x86_64 || EIGEN_ARCH_ARM_OR_ARM64 || EIGEN_ARCH_PPC || EIGEN_ARCH_IA64 || \
152 EIGEN_ARCH_MIPS || EIGEN_ARCH_LOONGARCH64 || EIGEN_ARCH_RISCV)
153#define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1
154#else
155#define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 0
156#endif
157
158// static alignment is completely disabled with GCC 3, Sun Studio, and QCC/QNX
159#if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT && !EIGEN_COMP_SUNCC && !EIGEN_OS_QNX
160#define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 1
161#else
162#define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 0
163#endif
164
165#if EIGEN_ARCH_WANTS_STACK_ALIGNMENT
166#define EIGEN_MAX_STATIC_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES
167#else
168#define EIGEN_MAX_STATIC_ALIGN_BYTES 0
169#endif
170
171#endif
172
173// If EIGEN_MAX_ALIGN_BYTES is defined, then it is considered as an upper bound for EIGEN_MAX_STATIC_ALIGN_BYTES
174#if defined(EIGEN_MAX_ALIGN_BYTES) && EIGEN_MAX_ALIGN_BYTES < EIGEN_MAX_STATIC_ALIGN_BYTES
175#undef EIGEN_MAX_STATIC_ALIGN_BYTES
176#define EIGEN_MAX_STATIC_ALIGN_BYTES EIGEN_MAX_ALIGN_BYTES
177#endif
178
179#if EIGEN_MAX_STATIC_ALIGN_BYTES == 0 && !defined(EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT)
180#define EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT
181#endif
182
183// At this stage, EIGEN_MAX_STATIC_ALIGN_BYTES>0 is the true test whether we want to align arrays on the stack or not.
184// It takes into account both the user choice to explicitly enable/disable alignment (by setting
185// EIGEN_MAX_STATIC_ALIGN_BYTES) and the architecture config (EIGEN_ARCH_WANTS_STACK_ALIGNMENT). Henceforth, only
186// EIGEN_MAX_STATIC_ALIGN_BYTES should be used.
187
188// Shortcuts to EIGEN_ALIGN_TO_BOUNDARY
189#define EIGEN_ALIGN8 EIGEN_ALIGN_TO_BOUNDARY(8)
190#define EIGEN_ALIGN16 EIGEN_ALIGN_TO_BOUNDARY(16)
191#define EIGEN_ALIGN32 EIGEN_ALIGN_TO_BOUNDARY(32)
192#define EIGEN_ALIGN64 EIGEN_ALIGN_TO_BOUNDARY(64)
193#if EIGEN_MAX_STATIC_ALIGN_BYTES > 0
194#define EIGEN_ALIGN_MAX EIGEN_ALIGN_TO_BOUNDARY(EIGEN_MAX_STATIC_ALIGN_BYTES)
195#else
196#define EIGEN_ALIGN_MAX
197#endif
198
199// Dynamic alignment control
200
201#if defined(EIGEN_DONT_ALIGN) && defined(EIGEN_MAX_ALIGN_BYTES) && EIGEN_MAX_ALIGN_BYTES > 0
202#error EIGEN_MAX_ALIGN_BYTES and EIGEN_DONT_ALIGN are both defined with EIGEN_MAX_ALIGN_BYTES!=0. Use EIGEN_MAX_ALIGN_BYTES=0 as a synonym of EIGEN_DONT_ALIGN.
203#endif
204
205#ifdef EIGEN_DONT_ALIGN
206#ifdef EIGEN_MAX_ALIGN_BYTES
207#undef EIGEN_MAX_ALIGN_BYTES
208#endif
209#define EIGEN_MAX_ALIGN_BYTES 0
210#elif !defined(EIGEN_MAX_ALIGN_BYTES)
211#define EIGEN_MAX_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES
212#endif
213
214#if EIGEN_IDEAL_MAX_ALIGN_BYTES > EIGEN_MAX_ALIGN_BYTES
215#define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES
216#else
217#define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_MAX_ALIGN_BYTES
218#endif
219
220#ifndef EIGEN_UNALIGNED_VECTORIZE
221#define EIGEN_UNALIGNED_VECTORIZE 1
222#endif
223
224//----------------------------------------------------------------------
225
226// if alignment is disabled, then disable vectorization. Note: EIGEN_MAX_ALIGN_BYTES is the proper check, it takes into
227// account both the user's will (EIGEN_MAX_ALIGN_BYTES,EIGEN_DONT_ALIGN) and our own platform checks
228#if EIGEN_MAX_ALIGN_BYTES == 0
229#ifndef EIGEN_DONT_VECTORIZE
230#define EIGEN_DONT_VECTORIZE
231#endif
232#endif
233
234#if EIGEN_COMP_MSVC
235#include <malloc.h> // for _alloca
236#endif
237
238#if !(defined(EIGEN_DONT_VECTORIZE) || defined(EIGEN_GPUCC) || defined(EIGEN_VECTORIZE_GENERIC))
239
240// Whether the ARM SME backend can be built at all. Two hard requirements, both
241// properties of the translation unit rather than of the CPU:
242// - SME2, not just SME: the micro-kernel's multi-vector loads (svld1_*_x2/x4)
243// and svcount_t predicates are SME2 instructions.
244// - scalable (VLA) SVE mode: the kernel derives its ZA-tile geometry from the
245// runtime streaming vector length, so -msve-vector-bits=N would pin it to
246// one SVL and silently miscompute at any other.
247#if defined(__ARM_FEATURE_SME2) && !(defined(__ARM_FEATURE_SVE_BITS) && (__ARM_FEATURE_SVE_BITS != 0))
248#define EIGEN_ARM64_SME_USABLE
249#endif
250
251// ... and whether it is the backend to use. SME is selected automatically when
252// it is usable: unlike SVE it needs no fixed vector length, and it only
253// replaces the GEMM kernels -- everything else keeps the NEON packet path. Two
254// escapes: EIGEN_ARM64_NO_SME turns it off, and EIGEN_ARM64_USE_SVE takes
255// precedence for callers who asked for SVE specifically.
256#if defined(EIGEN_ARM64_SME_USABLE) && !defined(EIGEN_ARM64_NO_SME) && !defined(EIGEN_ARM64_USE_SVE)
257#define EIGEN_ARM64_SME_SELECTED
258#endif
259
260#if defined(EIGEN_ARM64_USE_SME) && defined(EIGEN_ARM64_NO_SME)
261#error "EIGEN_ARM64_USE_SME and EIGEN_ARM64_NO_SME are mutually exclusive."
262#endif
263
264// MSVC reports SSE2 through _M_IX86_FP on 32-bit x86; x86-64 always supports SSE2.
265#if (EIGEN_COMP_MSVC && ((defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || EIGEN_ARCH_x86_64)) || \
266 (!EIGEN_COMP_MSVC && defined(__SSE2__))
267
268// Defines symbols for compile-time detection of which instructions are
269// used.
270// EIGEN_VECTORIZE_YY is defined if and only if the instruction set YY is used
271#define EIGEN_VECTORIZE
272#define EIGEN_VECTORIZE_SSE
273#define EIGEN_VECTORIZE_SSE2
274
275// Detect sse3/ssse3/sse4:
276// gcc and icc defines __SSE3__, ...
277// there is no way to know about this on msvc. You can define EIGEN_VECTORIZE_SSE* if you
278// want to force the use of those instructions with msvc.
279#ifdef __SSE3__
280#define EIGEN_VECTORIZE_SSE3
281#endif
282#ifdef __SSSE3__
283#define EIGEN_VECTORIZE_SSSE3
284#endif
285#ifdef __SSE4_1__
286#define EIGEN_VECTORIZE_SSE4_1
287#endif
288#ifdef __SSE4_2__
289#define EIGEN_VECTORIZE_SSE4_2
290#endif
291#ifdef __AVX__
292#if !defined(EIGEN_USE_SYCL) && !EIGEN_COMP_EMSCRIPTEN
293#define EIGEN_VECTORIZE_AVX
294#endif
295#define EIGEN_VECTORIZE_SSE3
296#define EIGEN_VECTORIZE_SSSE3
297#define EIGEN_VECTORIZE_SSE4_1
298#define EIGEN_VECTORIZE_SSE4_2
299#endif
300#ifdef __AVX2__
301#ifndef EIGEN_USE_SYCL
302#define EIGEN_VECTORIZE_AVX2
303#define EIGEN_VECTORIZE_AVX
304#endif
305#define EIGEN_VECTORIZE_SSE3
306#define EIGEN_VECTORIZE_SSSE3
307#define EIGEN_VECTORIZE_SSE4_1
308#define EIGEN_VECTORIZE_SSE4_2
309#endif
310#if defined(__FMA__) || (EIGEN_COMP_MSVC && defined(__AVX2__))
311// MSVC does not expose a switch dedicated for FMA
312// For MSVC, AVX2 => FMA
313#define EIGEN_VECTORIZE_FMA
314#endif
315#if defined(__AVX512F__)
316#ifndef EIGEN_VECTORIZE_FMA
317#if EIGEN_COMP_GNUC
318#error Please add -mfma to your compiler flags: compiling with -mavx512f alone without SSE/AVX FMA is not supported (bug 1638).
319#else
320#error Please enable FMA in your compiler flags (e.g. -mfma): compiling with AVX512 alone without SSE/AVX FMA is not supported (bug 1638).
321#endif
322#endif
323#ifndef EIGEN_USE_SYCL
324#define EIGEN_VECTORIZE_AVX512
325#define EIGEN_VECTORIZE_AVX2
326#define EIGEN_VECTORIZE_AVX
327#endif
328#define EIGEN_VECTORIZE_FMA
329#define EIGEN_VECTORIZE_SSE3
330#define EIGEN_VECTORIZE_SSSE3
331#define EIGEN_VECTORIZE_SSE4_1
332#define EIGEN_VECTORIZE_SSE4_2
333#ifndef EIGEN_USE_SYCL
334#ifdef __AVX512DQ__
335#define EIGEN_VECTORIZE_AVX512DQ
336#endif
337#ifdef __AVX512ER__
338#define EIGEN_VECTORIZE_AVX512ER
339#endif
340#ifdef __AVX512BF16__
341#define EIGEN_VECTORIZE_AVX512BF16
342#endif
343#ifdef __AVX512VL__
344#define EIGEN_VECTORIZE_AVX512VL
345#endif
346#ifdef __AVX512FP16__
347#if EIGEN_COMP_NVHPC
348// NVC++ exposes AVX512-FP16 inconsistently: older releases define the feature without _Float16/__m512h,
349// and 24.11-26.3 lower compare/blend intrinsics to unresolved __builtin_ia32_*ph* references.
350#elif defined(__AVX512VL__)
351#define EIGEN_VECTORIZE_AVX512FP16
352// Built-in _Float16.
353#define EIGEN_HAS_BUILTIN_FLOAT16 1
354#else
355#if EIGEN_COMP_GNUC
356#error Please add -mavx512vl to your compiler flags: compiling with -mavx512fp16 alone without AVX512-VL is not supported.
357#else
358#error Please enable AVX512-VL in your compiler flags (e.g. -mavx512vl): compiling with AVX512-FP16 alone without AVX512-VL is not supported.
359#endif
360#endif
361#endif
362#endif
363#endif
364
365// Disable AVX support on broken xcode versions
366#if (EIGEN_COMP_CLANGAPPLE == 11000033) && (__MAC_OS_X_VERSION_MIN_REQUIRED == 101500)
367// A nasty bug in the clang compiler shipped with xcode in a common compilation situation
368// when XCode 11.0 and Mac deployment target macOS 10.15 is https://trac.macports.org/ticket/58776#no1
369#ifdef EIGEN_VECTORIZE_AVX
370#undef EIGEN_VECTORIZE_AVX
371#warning \
372 "Disabling AVX support: clang compiler shipped with XCode 11.[012] generates broken assembly with -macosx-version-min=10.15 and AVX enabled. "
373#ifdef EIGEN_VECTORIZE_AVX2
374#undef EIGEN_VECTORIZE_AVX2
375#endif
376#ifdef EIGEN_VECTORIZE_FMA
377#undef EIGEN_VECTORIZE_FMA
378#endif
379#ifdef EIGEN_VECTORIZE_AVX512
380#undef EIGEN_VECTORIZE_AVX512
381#endif
382#ifdef EIGEN_VECTORIZE_AVX512DQ
383#undef EIGEN_VECTORIZE_AVX512DQ
384#endif
385#ifdef EIGEN_VECTORIZE_AVX512ER
386#undef EIGEN_VECTORIZE_AVX512ER
387#endif
388#endif
389// NOTE: Confirmed test failures in XCode 11.0, and XCode 11.2 with -macosx-version-min=10.15 and AVX
390// NOTE using -macosx-version-min=10.15 with Xcode 11.0 results in runtime segmentation faults in many tests, 11.2
391// produce core dumps in 3 tests NOTE using -macosx-version-min=10.14 produces functioning and passing tests in all
392// cases NOTE __clang_version__ "11.0.0 (clang-1100.0.33.8)" XCode 11.0 <- Produces many segfault and core dumping
393// tests
394// with -macosx-version-min=10.15 and AVX
395// NOTE __clang_version__ "11.0.0 (clang-1100.0.33.12)" XCode 11.2 <- Produces 3 core dumping tests with
396// -macosx-version-min=10.15 and AVX
397#endif
398
399// include files
400
401// This extern "C" works around a MINGW-w64 compilation issue
402// https://sourceforge.net/tracker/index.php?func=detail&aid=3018394&group_id=202880&atid=983354
403// In essence, intrin.h is included by windows.h and also declares intrinsics (just as emmintrin.h etc. below do).
404// However, intrin.h uses an extern "C" declaration, and g++ thus complains of duplicate declarations
405// with conflicting linkage. The linkage for intrinsics doesn't matter, but at that stage the compiler doesn't know;
406// so, to avoid compile errors when windows.h is included after Eigen/Core, ensure intrinsics are extern "C" here too.
407// notice that since these are C headers, the extern "C" is theoretically needed anyways.
408extern "C" {
409// ICC and Emscripten need the umbrella header instead of direct *mmintrin.h includes.
410#if EIGEN_COMP_ICC || EIGEN_COMP_EMSCRIPTEN
411#include <immintrin.h>
412#else
413#include <mmintrin.h>
414#include <emmintrin.h>
415#include <xmmintrin.h>
416#ifdef EIGEN_VECTORIZE_SSE3
417#include <pmmintrin.h>
418#endif
419#ifdef EIGEN_VECTORIZE_SSSE3
420#include <tmmintrin.h>
421#endif
422#ifdef EIGEN_VECTORIZE_SSE4_1
423#include <smmintrin.h>
424#endif
425#ifdef EIGEN_VECTORIZE_SSE4_2
426#include <nmmintrin.h>
427#endif
428#if defined(EIGEN_VECTORIZE_AVX) || defined(EIGEN_VECTORIZE_AVX512)
429#include <immintrin.h>
430#endif
431#endif
432} // end extern "C"
433
434#elif defined(__VSX__) && !defined(__APPLE__)
435
436#define EIGEN_VECTORIZE
437#define EIGEN_VECTORIZE_VSX 1
438// The POWER8 (ISA 2.07) vector instructions, such as 64-bit integer lane arithmetic and compares and vec_neg; a
439// POWER7 target has VSX without them.
440#if defined(__POWER8_VECTOR__)
441#define EIGEN_VECTORIZE_POWER8_VECTOR
442#endif
443#define EIGEN_VECTORIZE_FMA
444#include <altivec.h>
445// We need to #undef macros defined by <altivec.h> that conflict with standard C++ names.
446// => use __vector instead of vector
447#undef bool
448#undef vector
449#undef pixel
450
451#elif defined __ALTIVEC__
452
453#define EIGEN_VECTORIZE
454#define EIGEN_VECTORIZE_ALTIVEC
455#define EIGEN_VECTORIZE_FMA
456#include <altivec.h>
457// We need to #undef macros defined by <altivec.h> that conflict with standard C++ names.
458// => use __vector instead of vector
459#undef bool
460#undef vector
461#undef pixel
462
463#elif defined(EIGEN_ARM64_USE_SME) && !defined(EIGEN_ARM64_SME_USABLE)
464
465#error \
466 "EIGEN_ARM64_USE_SME requires a compiler targeting SME2 in scalable (SVE VLA) mode: build with e.g. -march=armv9.2-a+sme2 and without -msve-vector-bits."
467
468#elif ((defined __ARM_NEON) || (defined __ARM_NEON__)) && !(defined EIGEN_ARM64_USE_SVE) && \
469 !(defined EIGEN_ARM64_SME_SELECTED)
470
471#define EIGEN_VECTORIZE
472#define EIGEN_VECTORIZE_NEON
473#include <arm_neon.h>
474
475// We currently require SVE to be enabled explicitly via EIGEN_ARM64_USE_SVE and
476// will not select the backend automatically
477#elif (defined __ARM_FEATURE_SVE) && (defined EIGEN_ARM64_USE_SVE)
478
479#define EIGEN_VECTORIZE
480#define EIGEN_VECTORIZE_SVE
481#include <arm_sve.h>
482
483// Since we depend on knowing SVE vector length at compile-time, we need
484// to ensure a fixed length is set
485#if defined __ARM_FEATURE_SVE_BITS
486#define EIGEN_ARM64_SVE_VL __ARM_FEATURE_SVE_BITS
487
488// Architecture-mandated length constraints.
489static_assert((EIGEN_ARM64_SVE_VL >= 128) && (EIGEN_ARM64_SVE_VL <= 2048) &&
490 ((EIGEN_ARM64_SVE_VL & (EIGEN_ARM64_SVE_VL - 1)) == 0),
491 "SVE vector length must be 2^n for some n in [7, 11]");
492#else
493#error "Eigen requires a fixed SVE vector length but EIGEN_ARM64_SVE_VL is not set."
494#endif
495
496// TriangularMatrixMatrix.h puts a (2 * max(mr, nr))^2 panel of Scalar on the
497// stack, and mr is 3 * PacketSize, so for float -- the widest scalar this backend
498// vectorizes -- the panel is 9 * VL^2 / 64 bytes
499// -- 144 kB at VL=1024 and 576 kB at VL=2048, past the 128 kB default, and the
500// backend does not compile at those lengths without more room. Only raise Eigen's
501// default; an explicit user limit remains authoritative, including 0.
502#if defined(EIGEN_STACK_ALLOCATION_LIMIT_WAS_DEFAULTED) && \
503 EIGEN_STACK_ALLOCATION_LIMIT < (9 * EIGEN_ARM64_SVE_VL * EIGEN_ARM64_SVE_VL / 64)
504#undef EIGEN_STACK_ALLOCATION_LIMIT
505#define EIGEN_STACK_ALLOCATION_LIMIT (9 * EIGEN_ARM64_SVE_VL * EIGEN_ARM64_SVE_VL / 64)
506#endif
507
508// Selected automatically whenever the toolchain can provide it; see
509// EIGEN_ARM64_SME_SELECTED above for the conditions and the opt-out.
510#elif defined(EIGEN_ARM64_SME_SELECTED)
511
512#define EIGEN_VECTORIZE
513#define EIGEN_VECTORIZE_SME
514#include <arm_neon.h>
515#include <arm_sme.h>
516
517// Double-precision outer products (FMOPA into a ZA.D tile) need the optional
518// FEAT_SME_F64F64, which each compiler reports differently: GCC defines the ACLE
519// macro, clang defines no macro but gates the builtin on the target feature.
520// Both halves are needed -- clang otherwise accepts svmopa_za64_f64_m without the
521// feature, so a missed gate faults at run time rather than at build time.
522#if !defined(EIGEN_ARM64_NO_SME_F64F64) && \
523 (defined(__ARM_FEATURE_SME_F64F64) || EIGEN_HAS_BUILTIN(__builtin_sme_svmopa_za64_f64_m))
524#define EIGEN_VECTORIZE_SME_F64F64
525#endif
526
527#elif EIGEN_ARCH_RISCV
528
529#if defined(__riscv_zfh)
530#define EIGEN_HAS_BUILTIN_FLOAT16
531#endif
532
533// We currently require RVV to be enabled explicitly via EIGEN_RISCV64_USE_RVV and
534// will not select the backend automatically
535#if (defined EIGEN_RISCV64_USE_RVV10)
536
537#define EIGEN_VECTORIZE
538#define EIGEN_VECTORIZE_RVV10
539// pmadd and its variants map to the fused vfmadd/vfmsub/vfnmsub/vfnmadd.
540#define EIGEN_VECTORIZE_FMA
541#include <riscv_vector.h>
542
543// Since we depend on knowing RVV vector length at compile-time, we need
544// to ensure a fixed length is set
545#if defined(__riscv_v_fixed_vlen)
546#define EIGEN_RISCV64_RVV_VL __riscv_v_fixed_vlen
547#if __riscv_v_fixed_vlen >= 256
548#undef EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT
549#define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1
550#endif
551#else
552#error "Eigen requires a fixed RVV vector length but -mrvv-vector-bits=zvl is not set."
553#endif
554
555// Raise Eigen's own default only, as the SVE block above does: an explicit limit is a caller
556// policy on stack safety, and rewriting it defeats the policy. nomalloc, bdcsvd, jacobisvd,
557// diagonalview and diagonalmatrices set 0 to switch the check off, and clobbering that re-enables
558// alloca underneath the very tests written to catch it.
559#ifdef EIGEN_STACK_ALLOCATION_LIMIT_WAS_DEFAULTED
560#undef EIGEN_STACK_ALLOCATION_LIMIT
561#if __riscv_v_fixed_vlen <= 512
562#define EIGEN_STACK_ALLOCATION_LIMIT 196608
563#else
564#define EIGEN_STACK_ALLOCATION_LIMIT 393216
565#endif
566#endif
567
568#if defined(__riscv_zvfh) && defined(__riscv_zfh)
569#define EIGEN_VECTORIZE_RVV10FP16
570#elif defined(__riscv_zvfh)
571#if defined(__GNUC__) || defined(__clang__)
572#warning "The Eigen::Half vectorization requires Zfh and Zvfh extensions."
573#elif defined(_MSC_VER)
574#pragma message("The Eigen::Half vectorization requires Zfh and Zvfh extensions.")
575#endif
576#endif
577
578#if defined(__riscv_zvfbfwma)
579#define EIGEN_VECTORIZE_RVV10BF16
580#endif
581
582#endif // defined(EIGEN_ARCH_RISCV)
583
584// ZVector needs the vector float instructions of z14 (__ARCH__ 12); below that, s390x stays scalar.
585#elif (defined __s390x__ && defined __VEC__ && (!defined(__ARCH__) || __ARCH__ >= 12))
586
587#define EIGEN_VECTORIZE
588#define EIGEN_VECTORIZE_ZVECTOR
589#include <vecintrin.h>
590
591#elif defined __mips_msa
592
593// Limit MSA optimizations to little-endian CPUs for now.
594// TODO: Perhaps, eventually support MSA optimizations on big-endian CPUs?
595#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
596#if defined(__LP64__)
597#define EIGEN_MIPS_64
598#else
599#define EIGEN_MIPS_32
600#endif
601#define EIGEN_VECTORIZE
602#define EIGEN_VECTORIZE_MSA
603#include <msa.h>
604#endif
605
606#elif (defined __loongarch64 && defined __loongarch_sx)
607
608#define EIGEN_VECTORIZE
609#define EIGEN_VECTORIZE_LSX
610// pmadd and its variants map to the fused vfmadd/vfmsub/vfnmadd/vfnmsub.
611#define EIGEN_VECTORIZE_FMA
612#include <lsxintrin.h>
613
614#elif defined __HVX__ && (__HVX_LENGTH__ == 128)
615
616#define EIGEN_VECTORIZE
617#define EIGEN_VECTORIZE_HVX
618#include <hexagon_types.h>
619
620#endif
621#endif
622
623// The backend blocks above are the only consumers; do not leak it into user code.
624#undef EIGEN_STACK_ALLOCATION_LIMIT_WAS_DEFAULTED
625
626// Following the Arm ACLE arm_neon.h should also include arm_fp16.h but not all
627// compilers seem to follow this. We therefore include it explicitly.
628// See also: https://bugs.llvm.org/show_bug.cgi?id=47955
629#if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC)
630#include <arm_fp16.h>
631#endif
632
633// Enable FMA for ARM.
634#if defined(__ARM_FEATURE_FMA)
635#define EIGEN_VECTORIZE_FMA
636#endif
637
638// The generic clang backend skips the x86 detection above but fuses its pmadd where __FMA__ is defined, and the
639// kernels that choose constants or error-free products by EIGEN_VECTORIZE_FMA must know it.
640#if defined(EIGEN_VECTORIZE_GENERIC) && !defined(EIGEN_DONT_VECTORIZE) && defined(__FMA__) && \
641 !defined(EIGEN_VECTORIZE_FMA)
642#define EIGEN_VECTORIZE_FMA
643#endif
644
645#if defined(__F16C__) && !defined(EIGEN_GPUCC)
646// We can use the optimized fp16 to float and float to fp16 conversion routines
647#define EIGEN_HAS_FP16_C
648
649#if EIGEN_COMP_GNUC
650// Make sure immintrin.h is included, even if e.g. vectorization is
651// explicitly disabled (see also issue #2395).
652// Note that FP16C intrinsics for gcc and clang are included by immintrin.h,
653// as opposed to emmintrin.h as suggested by Intel:
654// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#othertechs=FP16C&expand=1711
655#include <immintrin.h>
656#endif
657#endif
658
659#if defined EIGEN_CUDACC
660#define EIGEN_VECTORIZE_GPU
661#include <vector_types.h>
662#include <cuda_runtime_api.h>
663#if defined(EIGEN_HAS_CUDA_FP16)
664#include <cuda_fp16.h>
665#endif
666#endif
667
668#if defined(EIGEN_HIPCC)
669#define EIGEN_VECTORIZE_GPU
670#include <hip/hip_vector_types.h>
671#include <hip/hip_fp16.h>
672#define EIGEN_HAS_HIP_BF16
673#include <hip/hip_bfloat16.h>
674#endif
675
677// IWYU pragma: private
678#include "../InternalHeaderCheck.h"
679
689#ifndef EIGEN_SCALAR_MADD_USE_FMA
690#ifdef EIGEN_VECTORIZE_FMA
691#define EIGEN_SCALAR_MADD_USE_FMA 1
692#else
693#define EIGEN_SCALAR_MADD_USE_FMA 0
694#endif
695#endif
696
697namespace Eigen {
698
699inline static const char* SimdInstructionSetsInUse(void) {
700#if defined(EIGEN_VECTORIZE_GENERIC) && !defined(EIGEN_DONT_VECTORIZE)
701 return "Clang vector extensions";
702#elif defined(EIGEN_VECTORIZE_AVX512)
703 return "AVX512, FMA, AVX2, AVX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
704#elif defined(EIGEN_VECTORIZE_AVX)
705 return "AVX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
706#elif defined(EIGEN_VECTORIZE_SSE4_2)
707 return "SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
708#elif defined(EIGEN_VECTORIZE_SSE4_1)
709 return "SSE, SSE2, SSE3, SSSE3, SSE4.1";
710#elif defined(EIGEN_VECTORIZE_SSSE3)
711 return "SSE, SSE2, SSE3, SSSE3";
712#elif defined(EIGEN_VECTORIZE_SSE3)
713 return "SSE, SSE2, SSE3";
714#elif defined(EIGEN_VECTORIZE_SSE2)
715 return "SSE, SSE2";
716#elif defined(EIGEN_VECTORIZE_ALTIVEC)
717 return "AltiVec";
718#elif defined(EIGEN_VECTORIZE_POWER8_VECTOR)
719 return "VSX, POWER8 vector";
720#elif defined(EIGEN_VECTORIZE_VSX)
721 return "VSX";
722#elif defined(EIGEN_VECTORIZE_NEON)
723 return "ARM NEON";
724#elif defined(EIGEN_VECTORIZE_SME)
725 return "ARM SME";
726#elif defined(EIGEN_VECTORIZE_SVE)
727 return "ARM SVE";
728#elif defined(EIGEN_VECTORIZE_ZVECTOR)
729 return "S390X ZVECTOR";
730#elif defined(EIGEN_VECTORIZE_MSA)
731 return "MIPS MSA";
732#elif defined(EIGEN_VECTORIZE_LSX)
733 return "LOONGARCH64 LSX";
734#elif defined(EIGEN_VECTORIZE_RVV10)
735 return "RVV";
736#elif defined(EIGEN_VECTORIZE_HVX)
737 return "Hexagon HVX";
738#else
739 return "None";
740#endif
741}
742
743} // end namespace Eigen
744
745#endif // EIGEN_CONFIGURE_VECTORIZATION_H