12#ifndef EIGEN_CONFIGURE_VECTORIZATION_H
13#define EIGEN_CONFIGURE_VECTORIZATION_H
16#if defined(EIGEN_VECTORIZE_GENERIC) && !defined(EIGEN_GENERIC_VECTOR_SIZE_BYTES)
17#define EIGEN_GENERIC_VECTOR_SIZE_BYTES 64
19#if defined(EIGEN_VECTORIZE_GENERIC) && !defined(EIGEN_DONT_VECTORIZE) && !defined(EIGEN_DONT_ALIGN)
20#if !EIGEN_ARCH_VECTOR_EXTENSIONS
21#error "The compiler does not support clang vector extensions."
23#define EIGEN_VECTORIZE
24#define EIGEN_MAX_ALIGN_BYTES EIGEN_GENERIC_VECTOR_SIZE_BYTES
46#if (defined EIGEN_CUDACC)
47#define EIGEN_ALIGN_TO_BOUNDARY(n) __align__(n)
48#define EIGEN_ALIGNOF(x) __alignof(x)
50#define EIGEN_ALIGN_TO_BOUNDARY(n) alignas(n)
51#define EIGEN_ALIGNOF(x) alignof(x)
56#ifndef EIGEN_RISCV64_DEFAULT_LMUL
57#define EIGEN_RISCV64_DEFAULT_LMUL 1
64#ifndef EIGEN_ALIGN_TO_AVOID_FALSE_SHARING
65#define EIGEN_ALIGN_TO_AVOID_FALSE_SHARING EIGEN_ALIGN_TO_BOUNDARY(128)
69#if defined(EIGEN_DONT_VECTORIZE)
70#if defined(EIGEN_GPUCC)
73#define EIGEN_IDEAL_MAX_ALIGN_BYTES 16
75#define EIGEN_IDEAL_MAX_ALIGN_BYTES 0
77#elif defined(EIGEN_VECTORIZE_GENERIC)
79#define EIGEN_IDEAL_MAX_ALIGN_BYTES EIGEN_GENERIC_VECTOR_SIZE_BYTES
80#elif defined(__AVX512F__)
82#define EIGEN_IDEAL_MAX_ALIGN_BYTES 64
92#elif defined(EIGEN_ARM64_USE_SVE) && defined(__ARM_FEATURE_SVE_BITS) && (__ARM_FEATURE_SVE_BITS != 0)
96#if __ARM_FEATURE_SVE_BITS <= 1024
97#define EIGEN_IDEAL_MAX_ALIGN_BYTES (__ARM_FEATURE_SVE_BITS / 8)
99#define EIGEN_IDEAL_MAX_ALIGN_BYTES 128
101#elif defined(__AVX__)
103#define EIGEN_IDEAL_MAX_ALIGN_BYTES 32
104#elif defined __HVX__ && (__HVX_LENGTH__ == 128)
105#define EIGEN_IDEAL_MAX_ALIGN_BYTES 128
106#elif defined(EIGEN_RISCV64_USE_RVV10) && defined(__riscv_v_fixed_vlen)
111#if (__riscv_v_fixed_vlen * EIGEN_RISCV64_DEFAULT_LMUL / 8) <= 128
112#define EIGEN_IDEAL_MAX_ALIGN_BYTES (__riscv_v_fixed_vlen * EIGEN_RISCV64_DEFAULT_LMUL / 8)
114#define EIGEN_IDEAL_MAX_ALIGN_BYTES 128
117#define EIGEN_IDEAL_MAX_ALIGN_BYTES 16
121#define EIGEN_MIN_ALIGN_BYTES 16
127#if (defined(EIGEN_DONT_ALIGN_STATICALLY) || defined(EIGEN_DONT_ALIGN)) && defined(EIGEN_MAX_STATIC_ALIGN_BYTES) && \
128 EIGEN_MAX_STATIC_ALIGN_BYTES > 0
129#error EIGEN_MAX_STATIC_ALIGN_BYTES and EIGEN_DONT_ALIGN[_STATICALLY] are both defined with EIGEN_MAX_STATIC_ALIGN_BYTES!=0. Use EIGEN_MAX_STATIC_ALIGN_BYTES=0 as a synonym of EIGEN_DONT_ALIGN_STATICALLY.
134#if defined(EIGEN_DONT_ALIGN_STATICALLY) || defined(EIGEN_DONT_ALIGN)
135#ifdef EIGEN_MAX_STATIC_ALIGN_BYTES
136#undef EIGEN_MAX_STATIC_ALIGN_BYTES
138#define EIGEN_MAX_STATIC_ALIGN_BYTES 0
141#ifndef EIGEN_MAX_STATIC_ALIGN_BYTES
151#if EIGEN_COMP_GNUC && !(EIGEN_ARCH_i386_OR_x86_64 || EIGEN_ARCH_ARM_OR_ARM64 || EIGEN_ARCH_PPC || EIGEN_ARCH_IA64 || \
152 EIGEN_ARCH_MIPS || EIGEN_ARCH_LOONGARCH64 || EIGEN_ARCH_RISCV)
153#define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1
155#define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 0
159#if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT && !EIGEN_COMP_SUNCC && !EIGEN_OS_QNX
160#define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 1
162#define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 0
165#if EIGEN_ARCH_WANTS_STACK_ALIGNMENT
166#define EIGEN_MAX_STATIC_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES
168#define EIGEN_MAX_STATIC_ALIGN_BYTES 0
174#if defined(EIGEN_MAX_ALIGN_BYTES) && EIGEN_MAX_ALIGN_BYTES < EIGEN_MAX_STATIC_ALIGN_BYTES
175#undef EIGEN_MAX_STATIC_ALIGN_BYTES
176#define EIGEN_MAX_STATIC_ALIGN_BYTES EIGEN_MAX_ALIGN_BYTES
179#if EIGEN_MAX_STATIC_ALIGN_BYTES == 0 && !defined(EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT)
180#define EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT
189#define EIGEN_ALIGN8 EIGEN_ALIGN_TO_BOUNDARY(8)
190#define EIGEN_ALIGN16 EIGEN_ALIGN_TO_BOUNDARY(16)
191#define EIGEN_ALIGN32 EIGEN_ALIGN_TO_BOUNDARY(32)
192#define EIGEN_ALIGN64 EIGEN_ALIGN_TO_BOUNDARY(64)
193#if EIGEN_MAX_STATIC_ALIGN_BYTES > 0
194#define EIGEN_ALIGN_MAX EIGEN_ALIGN_TO_BOUNDARY(EIGEN_MAX_STATIC_ALIGN_BYTES)
196#define EIGEN_ALIGN_MAX
201#if defined(EIGEN_DONT_ALIGN) && defined(EIGEN_MAX_ALIGN_BYTES) && EIGEN_MAX_ALIGN_BYTES > 0
202#error EIGEN_MAX_ALIGN_BYTES and EIGEN_DONT_ALIGN are both defined with EIGEN_MAX_ALIGN_BYTES!=0. Use EIGEN_MAX_ALIGN_BYTES=0 as a synonym of EIGEN_DONT_ALIGN.
205#ifdef EIGEN_DONT_ALIGN
206#ifdef EIGEN_MAX_ALIGN_BYTES
207#undef EIGEN_MAX_ALIGN_BYTES
209#define EIGEN_MAX_ALIGN_BYTES 0
210#elif !defined(EIGEN_MAX_ALIGN_BYTES)
211#define EIGEN_MAX_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES
214#if EIGEN_IDEAL_MAX_ALIGN_BYTES > EIGEN_MAX_ALIGN_BYTES
215#define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES
217#define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_MAX_ALIGN_BYTES
220#ifndef EIGEN_UNALIGNED_VECTORIZE
221#define EIGEN_UNALIGNED_VECTORIZE 1
228#if EIGEN_MAX_ALIGN_BYTES == 0
229#ifndef EIGEN_DONT_VECTORIZE
230#define EIGEN_DONT_VECTORIZE
238#if !(defined(EIGEN_DONT_VECTORIZE) || defined(EIGEN_GPUCC) || defined(EIGEN_VECTORIZE_GENERIC))
247#if defined(__ARM_FEATURE_SME2) && !(defined(__ARM_FEATURE_SVE_BITS) && (__ARM_FEATURE_SVE_BITS != 0))
248#define EIGEN_ARM64_SME_USABLE
256#if defined(EIGEN_ARM64_SME_USABLE) && !defined(EIGEN_ARM64_NO_SME) && !defined(EIGEN_ARM64_USE_SVE)
257#define EIGEN_ARM64_SME_SELECTED
260#if defined(EIGEN_ARM64_USE_SME) && defined(EIGEN_ARM64_NO_SME)
261#error "EIGEN_ARM64_USE_SME and EIGEN_ARM64_NO_SME are mutually exclusive."
265#if (EIGEN_COMP_MSVC && ((defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || EIGEN_ARCH_x86_64)) || \
266 (!EIGEN_COMP_MSVC && defined(__SSE2__))
271#define EIGEN_VECTORIZE
272#define EIGEN_VECTORIZE_SSE
273#define EIGEN_VECTORIZE_SSE2
280#define EIGEN_VECTORIZE_SSE3
283#define EIGEN_VECTORIZE_SSSE3
286#define EIGEN_VECTORIZE_SSE4_1
289#define EIGEN_VECTORIZE_SSE4_2
292#if !defined(EIGEN_USE_SYCL) && !EIGEN_COMP_EMSCRIPTEN
293#define EIGEN_VECTORIZE_AVX
295#define EIGEN_VECTORIZE_SSE3
296#define EIGEN_VECTORIZE_SSSE3
297#define EIGEN_VECTORIZE_SSE4_1
298#define EIGEN_VECTORIZE_SSE4_2
301#ifndef EIGEN_USE_SYCL
302#define EIGEN_VECTORIZE_AVX2
303#define EIGEN_VECTORIZE_AVX
305#define EIGEN_VECTORIZE_SSE3
306#define EIGEN_VECTORIZE_SSSE3
307#define EIGEN_VECTORIZE_SSE4_1
308#define EIGEN_VECTORIZE_SSE4_2
310#if defined(__FMA__) || (EIGEN_COMP_MSVC && defined(__AVX2__))
313#define EIGEN_VECTORIZE_FMA
315#if defined(__AVX512F__)
316#ifndef EIGEN_VECTORIZE_FMA
318#error Please add -mfma to your compiler flags: compiling with -mavx512f alone without SSE/AVX FMA is not supported (bug 1638).
320#error Please enable FMA in your compiler flags (e.g. -mfma): compiling with AVX512 alone without SSE/AVX FMA is not supported (bug 1638).
323#ifndef EIGEN_USE_SYCL
324#define EIGEN_VECTORIZE_AVX512
325#define EIGEN_VECTORIZE_AVX2
326#define EIGEN_VECTORIZE_AVX
328#define EIGEN_VECTORIZE_FMA
329#define EIGEN_VECTORIZE_SSE3
330#define EIGEN_VECTORIZE_SSSE3
331#define EIGEN_VECTORIZE_SSE4_1
332#define EIGEN_VECTORIZE_SSE4_2
333#ifndef EIGEN_USE_SYCL
335#define EIGEN_VECTORIZE_AVX512DQ
338#define EIGEN_VECTORIZE_AVX512ER
341#define EIGEN_VECTORIZE_AVX512BF16
344#define EIGEN_VECTORIZE_AVX512VL
350#elif defined(__AVX512VL__)
351#define EIGEN_VECTORIZE_AVX512FP16
353#define EIGEN_HAS_BUILTIN_FLOAT16 1
356#error Please add -mavx512vl to your compiler flags: compiling with -mavx512fp16 alone without AVX512-VL is not supported.
358#error Please enable AVX512-VL in your compiler flags (e.g. -mavx512vl): compiling with AVX512-FP16 alone without AVX512-VL is not supported.
366#if (EIGEN_COMP_CLANGAPPLE == 11000033) && (__MAC_OS_X_VERSION_MIN_REQUIRED == 101500)
369#ifdef EIGEN_VECTORIZE_AVX
370#undef EIGEN_VECTORIZE_AVX
372 "Disabling AVX support: clang compiler shipped with XCode 11.[012] generates broken assembly with -macosx-version-min=10.15 and AVX enabled. "
373#ifdef EIGEN_VECTORIZE_AVX2
374#undef EIGEN_VECTORIZE_AVX2
376#ifdef EIGEN_VECTORIZE_FMA
377#undef EIGEN_VECTORIZE_FMA
379#ifdef EIGEN_VECTORIZE_AVX512
380#undef EIGEN_VECTORIZE_AVX512
382#ifdef EIGEN_VECTORIZE_AVX512DQ
383#undef EIGEN_VECTORIZE_AVX512DQ
385#ifdef EIGEN_VECTORIZE_AVX512ER
386#undef EIGEN_VECTORIZE_AVX512ER
410#if EIGEN_COMP_ICC || EIGEN_COMP_EMSCRIPTEN
411#include <immintrin.h>
414#include <emmintrin.h>
415#include <xmmintrin.h>
416#ifdef EIGEN_VECTORIZE_SSE3
417#include <pmmintrin.h>
419#ifdef EIGEN_VECTORIZE_SSSE3
420#include <tmmintrin.h>
422#ifdef EIGEN_VECTORIZE_SSE4_1
423#include <smmintrin.h>
425#ifdef EIGEN_VECTORIZE_SSE4_2
426#include <nmmintrin.h>
428#if defined(EIGEN_VECTORIZE_AVX) || defined(EIGEN_VECTORIZE_AVX512)
429#include <immintrin.h>
434#elif defined(__VSX__) && !defined(__APPLE__)
436#define EIGEN_VECTORIZE
437#define EIGEN_VECTORIZE_VSX 1
440#if defined(__POWER8_VECTOR__)
441#define EIGEN_VECTORIZE_POWER8_VECTOR
443#define EIGEN_VECTORIZE_FMA
451#elif defined __ALTIVEC__
453#define EIGEN_VECTORIZE
454#define EIGEN_VECTORIZE_ALTIVEC
455#define EIGEN_VECTORIZE_FMA
463#elif defined(EIGEN_ARM64_USE_SME) && !defined(EIGEN_ARM64_SME_USABLE)
466 "EIGEN_ARM64_USE_SME requires a compiler targeting SME2 in scalable (SVE VLA) mode: build with e.g. -march=armv9.2-a+sme2 and without -msve-vector-bits."
468#elif ((defined __ARM_NEON) || (defined __ARM_NEON__)) && !(defined EIGEN_ARM64_USE_SVE) && \
469 !(defined EIGEN_ARM64_SME_SELECTED)
471#define EIGEN_VECTORIZE
472#define EIGEN_VECTORIZE_NEON
477#elif (defined __ARM_FEATURE_SVE) && (defined EIGEN_ARM64_USE_SVE)
479#define EIGEN_VECTORIZE
480#define EIGEN_VECTORIZE_SVE
485#if defined __ARM_FEATURE_SVE_BITS
486#define EIGEN_ARM64_SVE_VL __ARM_FEATURE_SVE_BITS
489static_assert((EIGEN_ARM64_SVE_VL >= 128) && (EIGEN_ARM64_SVE_VL <= 2048) &&
490 ((EIGEN_ARM64_SVE_VL & (EIGEN_ARM64_SVE_VL - 1)) == 0),
491 "SVE vector length must be 2^n for some n in [7, 11]");
493#error "Eigen requires a fixed SVE vector length but EIGEN_ARM64_SVE_VL is not set."
502#if defined(EIGEN_STACK_ALLOCATION_LIMIT_WAS_DEFAULTED) && \
503 EIGEN_STACK_ALLOCATION_LIMIT < (9 * EIGEN_ARM64_SVE_VL * EIGEN_ARM64_SVE_VL / 64)
504#undef EIGEN_STACK_ALLOCATION_LIMIT
505#define EIGEN_STACK_ALLOCATION_LIMIT (9 * EIGEN_ARM64_SVE_VL * EIGEN_ARM64_SVE_VL / 64)
510#elif defined(EIGEN_ARM64_SME_SELECTED)
512#define EIGEN_VECTORIZE
513#define EIGEN_VECTORIZE_SME
522#if !defined(EIGEN_ARM64_NO_SME_F64F64) && \
523 (defined(__ARM_FEATURE_SME_F64F64) || EIGEN_HAS_BUILTIN(__builtin_sme_svmopa_za64_f64_m))
524#define EIGEN_VECTORIZE_SME_F64F64
527#elif EIGEN_ARCH_RISCV
529#if defined(__riscv_zfh)
530#define EIGEN_HAS_BUILTIN_FLOAT16
535#if (defined EIGEN_RISCV64_USE_RVV10)
537#define EIGEN_VECTORIZE
538#define EIGEN_VECTORIZE_RVV10
540#define EIGEN_VECTORIZE_FMA
541#include <riscv_vector.h>
545#if defined(__riscv_v_fixed_vlen)
546#define EIGEN_RISCV64_RVV_VL __riscv_v_fixed_vlen
547#if __riscv_v_fixed_vlen >= 256
548#undef EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT
549#define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1
552#error "Eigen requires a fixed RVV vector length but -mrvv-vector-bits=zvl is not set."
559#ifdef EIGEN_STACK_ALLOCATION_LIMIT_WAS_DEFAULTED
560#undef EIGEN_STACK_ALLOCATION_LIMIT
561#if __riscv_v_fixed_vlen <= 512
562#define EIGEN_STACK_ALLOCATION_LIMIT 196608
564#define EIGEN_STACK_ALLOCATION_LIMIT 393216
568#if defined(__riscv_zvfh) && defined(__riscv_zfh)
569#define EIGEN_VECTORIZE_RVV10FP16
570#elif defined(__riscv_zvfh)
571#if defined(__GNUC__) || defined(__clang__)
572#warning "The Eigen::Half vectorization requires Zfh and Zvfh extensions."
573#elif defined(_MSC_VER)
574#pragma message("The Eigen::Half vectorization requires Zfh and Zvfh extensions.")
578#if defined(__riscv_zvfbfwma)
579#define EIGEN_VECTORIZE_RVV10BF16
585#elif (defined __s390x__ && defined __VEC__ && (!defined(__ARCH__) || __ARCH__ >= 12))
587#define EIGEN_VECTORIZE
588#define EIGEN_VECTORIZE_ZVECTOR
589#include <vecintrin.h>
591#elif defined __mips_msa
595#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
601#define EIGEN_VECTORIZE
602#define EIGEN_VECTORIZE_MSA
606#elif (defined __loongarch64 && defined __loongarch_sx)
608#define EIGEN_VECTORIZE
609#define EIGEN_VECTORIZE_LSX
611#define EIGEN_VECTORIZE_FMA
612#include <lsxintrin.h>
614#elif defined __HVX__ && (__HVX_LENGTH__ == 128)
616#define EIGEN_VECTORIZE
617#define EIGEN_VECTORIZE_HVX
618#include <hexagon_types.h>
624#undef EIGEN_STACK_ALLOCATION_LIMIT_WAS_DEFAULTED
629#if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC)
634#if defined(__ARM_FEATURE_FMA)
635#define EIGEN_VECTORIZE_FMA
640#if defined(EIGEN_VECTORIZE_GENERIC) && !defined(EIGEN_DONT_VECTORIZE) && defined(__FMA__) && \
641 !defined(EIGEN_VECTORIZE_FMA)
642#define EIGEN_VECTORIZE_FMA
645#if defined(__F16C__) && !defined(EIGEN_GPUCC)
647#define EIGEN_HAS_FP16_C
655#include <immintrin.h>
659#if defined EIGEN_CUDACC
660#define EIGEN_VECTORIZE_GPU
661#include <vector_types.h>
662#include <cuda_runtime_api.h>
663#if defined(EIGEN_HAS_CUDA_FP16)
664#include <cuda_fp16.h>
668#if defined(EIGEN_HIPCC)
669#define EIGEN_VECTORIZE_GPU
670#include <hip/hip_vector_types.h>
671#include <hip/hip_fp16.h>
672#define EIGEN_HAS_HIP_BF16
673#include <hip/hip_bfloat16.h>
678#include "../InternalHeaderCheck.h"
689#ifndef EIGEN_SCALAR_MADD_USE_FMA
690#ifdef EIGEN_VECTORIZE_FMA
691#define EIGEN_SCALAR_MADD_USE_FMA 1
693#define EIGEN_SCALAR_MADD_USE_FMA 0
699inline static const char* SimdInstructionSetsInUse(
void) {
700#if defined(EIGEN_VECTORIZE_GENERIC) && !defined(EIGEN_DONT_VECTORIZE)
701 return "Clang vector extensions";
702#elif defined(EIGEN_VECTORIZE_AVX512)
703 return "AVX512, FMA, AVX2, AVX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
704#elif defined(EIGEN_VECTORIZE_AVX)
705 return "AVX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
706#elif defined(EIGEN_VECTORIZE_SSE4_2)
707 return "SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
708#elif defined(EIGEN_VECTORIZE_SSE4_1)
709 return "SSE, SSE2, SSE3, SSSE3, SSE4.1";
710#elif defined(EIGEN_VECTORIZE_SSSE3)
711 return "SSE, SSE2, SSE3, SSSE3";
712#elif defined(EIGEN_VECTORIZE_SSE3)
713 return "SSE, SSE2, SSE3";
714#elif defined(EIGEN_VECTORIZE_SSE2)
716#elif defined(EIGEN_VECTORIZE_ALTIVEC)
718#elif defined(EIGEN_VECTORIZE_POWER8_VECTOR)
719 return "VSX, POWER8 vector";
720#elif defined(EIGEN_VECTORIZE_VSX)
722#elif defined(EIGEN_VECTORIZE_NEON)
724#elif defined(EIGEN_VECTORIZE_SME)
726#elif defined(EIGEN_VECTORIZE_SVE)
728#elif defined(EIGEN_VECTORIZE_ZVECTOR)
729 return "S390X ZVECTOR";
730#elif defined(EIGEN_VECTORIZE_MSA)
732#elif defined(EIGEN_VECTORIZE_LSX)
733 return "LOONGARCH64 LSX";
734#elif defined(EIGEN_VECTORIZE_RVV10)
736#elif defined(EIGEN_VECTORIZE_HVX)
737 return "Hexagon HVX";