ConfigureVectorization.h

// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
//
// Copyright (C) 2008-2018 Gael Guennebaud <gael.guennebaud@inria.fr>
// Copyright (C) 2020, Arm Limited and Contributors
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
// SPDX-License-Identifier: MPL-2.0
 
#ifndef EIGEN_CONFIGURE_VECTORIZATION_H
#define EIGEN_CONFIGURE_VECTORIZATION_H
 
// Prepare for using the generic clang backend if requested.
#if defined(EIGEN_VECTORIZE_GENERIC) && !defined(EIGEN_GENERIC_VECTOR_SIZE_BYTES)
#define EIGEN_GENERIC_VECTOR_SIZE_BYTES 64
#endif
#if defined(EIGEN_VECTORIZE_GENERIC) && !defined(EIGEN_DONT_VECTORIZE) && !defined(EIGEN_DONT_ALIGN)
#if !EIGEN_ARCH_VECTOR_EXTENSIONS
#error "The compiler does not support clang vector extensions."
#endif
#define EIGEN_VECTORIZE
#define EIGEN_MAX_ALIGN_BYTES EIGEN_GENERIC_VECTOR_SIZE_BYTES
#endif
 
//------------------------------------------------------------------------------------------
// Static and dynamic alignment control
//
// The main purpose of this section is to define EIGEN_MAX_ALIGN_BYTES and EIGEN_MAX_STATIC_ALIGN_BYTES
// as the maximal boundary in bytes on which dynamically and statically allocated data may be alignment respectively.
// The values of EIGEN_MAX_ALIGN_BYTES and EIGEN_MAX_STATIC_ALIGN_BYTES can be specified by the user. If not,
// a default value is automatically computed based on architecture, compiler, and OS.
//
// This section also defines macros EIGEN_ALIGN_TO_BOUNDARY(N) and the shortcuts EIGEN_ALIGN{8,16,32,_MAX}
// to be used to declare statically aligned buffers.
//------------------------------------------------------------------------------------------
 
/* EIGEN_ALIGN_TO_BOUNDARY(n) forces data to be n-byte aligned. This is used to satisfy SIMD requirements.
 * However, we do that EVEN if vectorization (EIGEN_VECTORIZE) is disabled,
 * so that vectorization doesn't affect binary compatibility.
 *
 * If we made alignment depend on whether or not EIGEN_VECTORIZE is defined, it would be impossible to link
 * vectorized and non-vectorized code.
 */
#if (defined EIGEN_CUDACC)
#define EIGEN_ALIGN_TO_BOUNDARY(n) __align__(n)
#define EIGEN_ALIGNOF(x) __alignof(x)
#else
#define EIGEN_ALIGN_TO_BOUNDARY(n) alignas(n)
#define EIGEN_ALIGNOF(x) alignof(x)
#endif
 
// Align to the boundary that avoids false sharing.
// Pinned to 128 bytes to preserve a stable ABI across architectures and standard libraries
// and avoid GCC -Winterference-size warnings.
#ifndef EIGEN_ALIGN_TO_AVOID_FALSE_SHARING
#define EIGEN_ALIGN_TO_AVOID_FALSE_SHARING EIGEN_ALIGN_TO_BOUNDARY(128)
#endif
 
// If the user explicitly disable vectorization, then we also disable alignment
#if defined(EIGEN_DONT_VECTORIZE)
#if defined(EIGEN_GPUCC)
// GPU code is always vectorized and requires memory alignment for
// statically allocated buffers.
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 16
#else
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 0
#endif
#elif defined(EIGEN_VECTORIZE_GENERIC)
// Generic clang backend overrides native SIMD; align to the generic vector size.
#define EIGEN_IDEAL_MAX_ALIGN_BYTES EIGEN_GENERIC_VECTOR_SIZE_BYTES
#elif defined(__AVX512F__)
// 64 bytes static alignment is preferred only if really required
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 64
// Deliberately no SME case: this block runs long before EIGEN_VECTORIZE_SME is
// defined, so a test for it here is unreachable -- and moving it would be wrong
// rather than merely late. Raising the value changes the alignment, and so the
// ABI, of every fixed-size object relative to a NEON build of the same headers;
// past 16 bytes aligned_malloc also switches to the prefixed
// handmade_aligned_malloc, so a matrix allocated in an SME translation unit and
// freed in a non-SME one corrupts the heap. Keeping 16 is not free: the SME GEMM
// kernel is up to 3.4x faster at small sizes when the result matrix starts on a
// 64-byte boundary, which is what 64 here would give every Eigen-owned matrix.
#elif defined(EIGEN_ARM64_USE_SVE) && defined(__ARM_FEATURE_SVE_BITS) && (__ARM_FEATURE_SVE_BITS != 0)
// A fixed-length SVE packet is __ARM_FEATURE_SVE_BITS/8 bytes wide and asks for
// exactly that much alignment; a fixed-size object has to be able to offer it or
// it never reaches the vectorized path.  The Alignment enum stops at 128.
#if __ARM_FEATURE_SVE_BITS <= 1024
#define EIGEN_IDEAL_MAX_ALIGN_BYTES (__ARM_FEATURE_SVE_BITS / 8)
#else
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 128
#endif
#elif defined(__AVX__)
// 32 bytes static alignment is preferred only if really required
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 32
#elif defined __HVX__ && (__HVX_LENGTH__ == 128)
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 128
#elif defined(EIGEN_RISCV64_USE_RVV10)
#if __riscv_v_fixed_vlen <= 512
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 64
#else
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 128
#endif
#else
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 16
#endif
 
// EIGEN_MIN_ALIGN_BYTES defines the minimal value for which the notion of explicit alignment makes sense
#define EIGEN_MIN_ALIGN_BYTES 16
 
// Defined the boundary (in bytes) on which the data needs to be aligned. Note
// that unless EIGEN_ALIGN is defined and not equal to 0, the data may not be
// aligned at all regardless of the value of this #define.
 
#if (defined(EIGEN_DONT_ALIGN_STATICALLY) || defined(EIGEN_DONT_ALIGN)) && defined(EIGEN_MAX_STATIC_ALIGN_BYTES) && \
    EIGEN_MAX_STATIC_ALIGN_BYTES > 0
#error EIGEN_MAX_STATIC_ALIGN_BYTES and EIGEN_DONT_ALIGN[_STATICALLY] are both defined with EIGEN_MAX_STATIC_ALIGN_BYTES!=0. Use EIGEN_MAX_STATIC_ALIGN_BYTES=0 as a synonym of EIGEN_DONT_ALIGN_STATICALLY.
#endif
 
// EIGEN_DONT_ALIGN_STATICALLY and EIGEN_DONT_ALIGN are deprecated
// They imply EIGEN_MAX_STATIC_ALIGN_BYTES=0
#if defined(EIGEN_DONT_ALIGN_STATICALLY) || defined(EIGEN_DONT_ALIGN)
#ifdef EIGEN_MAX_STATIC_ALIGN_BYTES
#undef EIGEN_MAX_STATIC_ALIGN_BYTES
#endif
#define EIGEN_MAX_STATIC_ALIGN_BYTES 0
#endif
 
#ifndef EIGEN_MAX_STATIC_ALIGN_BYTES
 
// Try to automatically guess what is the best default value for EIGEN_MAX_STATIC_ALIGN_BYTES
 
// 16 byte alignment is only useful for vectorization. Since it affects the ABI, we need to enable
// 16 byte alignment on all platforms where vectorization might be enabled. In theory we could always
// enable alignment, but it can be a cause of problems on some platforms, so we just disable it in
// certain common platform (compiler+architecture combinations) to avoid these problems.
// Only static alignment is really problematic (relies on nonstandard compiler extensions),
// try to keep heap alignment even when we have to disable static alignment.
#if EIGEN_COMP_GNUC && !(EIGEN_ARCH_i386_OR_x86_64 || EIGEN_ARCH_ARM_OR_ARM64 || EIGEN_ARCH_PPC || EIGEN_ARCH_IA64 || \
                         EIGEN_ARCH_MIPS || EIGEN_ARCH_LOONGARCH64 || EIGEN_ARCH_RISCV)
#define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1
#else
#define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 0
#endif
 
// static alignment is completely disabled with GCC 3, Sun Studio, and QCC/QNX
#if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT && !EIGEN_COMP_SUNCC && !EIGEN_OS_QNX
#define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 1
#else
#define EIGEN_ARCH_WANTS_STACK_ALIGNMENT 0
#endif
 
#if EIGEN_ARCH_WANTS_STACK_ALIGNMENT
#define EIGEN_MAX_STATIC_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES
#else
#define EIGEN_MAX_STATIC_ALIGN_BYTES 0
#endif
 
#endif
 
// If EIGEN_MAX_ALIGN_BYTES is defined, then it is considered as an upper bound for EIGEN_MAX_STATIC_ALIGN_BYTES
#if defined(EIGEN_MAX_ALIGN_BYTES) && EIGEN_MAX_ALIGN_BYTES < EIGEN_MAX_STATIC_ALIGN_BYTES
#undef EIGEN_MAX_STATIC_ALIGN_BYTES
#define EIGEN_MAX_STATIC_ALIGN_BYTES EIGEN_MAX_ALIGN_BYTES
#endif
 
#if EIGEN_MAX_STATIC_ALIGN_BYTES == 0 && !defined(EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT)
#define EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT
#endif
 
// At this stage, EIGEN_MAX_STATIC_ALIGN_BYTES>0 is the true test whether we want to align arrays on the stack or not.
// It takes into account both the user choice to explicitly enable/disable alignment (by setting
// EIGEN_MAX_STATIC_ALIGN_BYTES) and the architecture config (EIGEN_ARCH_WANTS_STACK_ALIGNMENT). Henceforth, only
// EIGEN_MAX_STATIC_ALIGN_BYTES should be used.
 
// Shortcuts to EIGEN_ALIGN_TO_BOUNDARY
#define EIGEN_ALIGN8 EIGEN_ALIGN_TO_BOUNDARY(8)
#define EIGEN_ALIGN16 EIGEN_ALIGN_TO_BOUNDARY(16)
#define EIGEN_ALIGN32 EIGEN_ALIGN_TO_BOUNDARY(32)
#define EIGEN_ALIGN64 EIGEN_ALIGN_TO_BOUNDARY(64)
#if EIGEN_MAX_STATIC_ALIGN_BYTES > 0
#define EIGEN_ALIGN_MAX EIGEN_ALIGN_TO_BOUNDARY(EIGEN_MAX_STATIC_ALIGN_BYTES)
#else
#define EIGEN_ALIGN_MAX
#endif
 
// Dynamic alignment control
 
#if defined(EIGEN_DONT_ALIGN) && defined(EIGEN_MAX_ALIGN_BYTES) && EIGEN_MAX_ALIGN_BYTES > 0
#error EIGEN_MAX_ALIGN_BYTES and EIGEN_DONT_ALIGN are both defined with EIGEN_MAX_ALIGN_BYTES!=0. Use EIGEN_MAX_ALIGN_BYTES=0 as a synonym of EIGEN_DONT_ALIGN.
#endif
 
#ifdef EIGEN_DONT_ALIGN
#ifdef EIGEN_MAX_ALIGN_BYTES
#undef EIGEN_MAX_ALIGN_BYTES
#endif
#define EIGEN_MAX_ALIGN_BYTES 0
#elif !defined(EIGEN_MAX_ALIGN_BYTES)
#define EIGEN_MAX_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES
#endif
 
#if EIGEN_IDEAL_MAX_ALIGN_BYTES > EIGEN_MAX_ALIGN_BYTES
#define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES
#else
#define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_MAX_ALIGN_BYTES
#endif
 
#ifndef EIGEN_UNALIGNED_VECTORIZE
#define EIGEN_UNALIGNED_VECTORIZE 1
#endif
 
//----------------------------------------------------------------------
 
// if alignment is disabled, then disable vectorization. Note: EIGEN_MAX_ALIGN_BYTES is the proper check, it takes into
// account both the user's will (EIGEN_MAX_ALIGN_BYTES,EIGEN_DONT_ALIGN) and our own platform checks
#if EIGEN_MAX_ALIGN_BYTES == 0
#ifndef EIGEN_DONT_VECTORIZE
#define EIGEN_DONT_VECTORIZE
#endif
#endif
 
// MSVC needs <malloc.h> for aligned allocation helpers, and 32-bit x86 uses
// _M_IX86_FP to decide whether SSE2 is enabled.
#if EIGEN_COMP_MSVC
#include <malloc.h>  // for _aligned_malloc -- need it regardless of whether vectorization is enabled
// a user reported that in 64-bit mode, MSVC doesn't care to define _M_IX86_FP.
#if (defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || EIGEN_ARCH_x86_64
#define EIGEN_SSE2_ON_MSVC_2008_OR_LATER
#endif
#else
#if defined(__SSE2__)
#define EIGEN_SSE2_ON_NON_MSVC
#endif
#endif
 
#if !(defined(EIGEN_DONT_VECTORIZE) || defined(EIGEN_GPUCC) || defined(EIGEN_VECTORIZE_GENERIC))
 
// Whether the ARM SME backend can be built at all. Two hard requirements, both
// properties of the translation unit rather than of the CPU:
//   - SME2, not just SME: the micro-kernel's multi-vector loads (svld1_*_x2/x4)
//     and svcount_t predicates are SME2 instructions.
//   - scalable (VLA) SVE mode: the kernel derives its ZA-tile geometry from the
//     runtime streaming vector length, so -msve-vector-bits=N would pin it to
//     one SVL and silently miscompute at any other.
#if defined(__ARM_FEATURE_SME2) && !(defined(__ARM_FEATURE_SVE_BITS) && (__ARM_FEATURE_SVE_BITS != 0))
#define EIGEN_ARM64_SME_USABLE
#endif
 
// ... and whether it is the backend to use. SME is selected automatically when
// it is usable: unlike SVE it needs no fixed vector length, and it only
// replaces the GEMM kernels -- everything else keeps the NEON packet path. Two
// escapes: EIGEN_ARM64_NO_SME turns it off, and EIGEN_ARM64_USE_SVE takes
// precedence for callers who asked for SVE specifically.
#if defined(EIGEN_ARM64_SME_USABLE) && !defined(EIGEN_ARM64_NO_SME) && !defined(EIGEN_ARM64_USE_SVE)
#define EIGEN_ARM64_SME_SELECTED
#endif
 
#if defined(EIGEN_ARM64_USE_SME) && defined(EIGEN_ARM64_NO_SME)
#error "EIGEN_ARM64_USE_SME and EIGEN_ARM64_NO_SME are mutually exclusive."
#endif
 
#if defined(EIGEN_SSE2_ON_NON_MSVC) || defined(EIGEN_SSE2_ON_MSVC_2008_OR_LATER)
 
// Defines symbols for compile-time detection of which instructions are
// used.
// EIGEN_VECTORIZE_YY is defined if and only if the instruction set YY is used
#define EIGEN_VECTORIZE
#define EIGEN_VECTORIZE_SSE
#define EIGEN_VECTORIZE_SSE2
 
// Detect sse3/ssse3/sse4:
// gcc and icc defines __SSE3__, ...
// there is no way to know about this on msvc. You can define EIGEN_VECTORIZE_SSE* if you
// want to force the use of those instructions with msvc.
#ifdef __SSE3__
#define EIGEN_VECTORIZE_SSE3
#endif
#ifdef __SSSE3__
#define EIGEN_VECTORIZE_SSSE3
#endif
#ifdef __SSE4_1__
#define EIGEN_VECTORIZE_SSE4_1
#endif
#ifdef __SSE4_2__
#define EIGEN_VECTORIZE_SSE4_2
#endif
#ifdef __AVX__
#if !defined(EIGEN_USE_SYCL) && !EIGEN_COMP_EMSCRIPTEN
#define EIGEN_VECTORIZE_AVX
#endif
#define EIGEN_VECTORIZE_SSE3
#define EIGEN_VECTORIZE_SSSE3
#define EIGEN_VECTORIZE_SSE4_1
#define EIGEN_VECTORIZE_SSE4_2
#endif
#ifdef __AVX2__
#ifndef EIGEN_USE_SYCL
#define EIGEN_VECTORIZE_AVX2
#define EIGEN_VECTORIZE_AVX
#endif
#define EIGEN_VECTORIZE_SSE3
#define EIGEN_VECTORIZE_SSSE3
#define EIGEN_VECTORIZE_SSE4_1
#define EIGEN_VECTORIZE_SSE4_2
#endif
#if defined(__FMA__) || (EIGEN_COMP_MSVC && defined(__AVX2__))
// MSVC does not expose a switch dedicated for FMA
// For MSVC, AVX2 => FMA
#define EIGEN_VECTORIZE_FMA
#endif
#if defined(__AVX512F__)
#ifndef EIGEN_VECTORIZE_FMA
#if EIGEN_COMP_GNUC
#error Please add -mfma to your compiler flags: compiling with -mavx512f alone without SSE/AVX FMA is not supported (bug 1638).
#else
#error Please enable FMA in your compiler flags (e.g. -mfma): compiling with AVX512 alone without SSE/AVX FMA is not supported (bug 1638).
#endif
#endif
#ifndef EIGEN_USE_SYCL
#define EIGEN_VECTORIZE_AVX512
#define EIGEN_VECTORIZE_AVX2
#define EIGEN_VECTORIZE_AVX
#endif
#define EIGEN_VECTORIZE_FMA
#define EIGEN_VECTORIZE_SSE3
#define EIGEN_VECTORIZE_SSSE3
#define EIGEN_VECTORIZE_SSE4_1
#define EIGEN_VECTORIZE_SSE4_2
#ifndef EIGEN_USE_SYCL
#ifdef __AVX512DQ__
#define EIGEN_VECTORIZE_AVX512DQ
#endif
#ifdef __AVX512ER__
#define EIGEN_VECTORIZE_AVX512ER
#endif
#ifdef __AVX512BF16__
#define EIGEN_VECTORIZE_AVX512BF16
#endif
#ifdef __AVX512VL__
#define EIGEN_VECTORIZE_AVX512VL
#endif
#ifdef __AVX512FP16__
#if EIGEN_COMP_NVHPC
// NVC++ exposes AVX512-FP16 inconsistently: older releases define the feature without _Float16/__m512h,
// and 24.11-26.3 lower compare/blend intrinsics to unresolved __builtin_ia32_*ph* references.
#elif defined(__AVX512VL__)
#define EIGEN_VECTORIZE_AVX512FP16
// Built-in _Float16.
#define EIGEN_HAS_BUILTIN_FLOAT16 1
#else
#if EIGEN_COMP_GNUC
#error Please add -mavx512vl to your compiler flags: compiling with -mavx512fp16 alone without AVX512-VL is not supported.
#else
#error Please enable AVX512-VL in your compiler flags (e.g. -mavx512vl): compiling with AVX512-FP16 alone without AVX512-VL is not supported.
#endif
#endif
#endif
#endif
#endif
 
// Disable AVX support on broken xcode versions
#if (EIGEN_COMP_CLANGAPPLE == 11000033) && (__MAC_OS_X_VERSION_MIN_REQUIRED == 101500)
// A nasty bug in the clang compiler shipped with xcode in a common compilation situation
// when XCode 11.0 and Mac deployment target macOS 10.15 is https://trac.macports.org/ticket/58776#no1
#ifdef EIGEN_VECTORIZE_AVX
#undef EIGEN_VECTORIZE_AVX
#warning \
    "Disabling AVX support: clang compiler shipped with XCode 11.[012] generates broken assembly with -macosx-version-min=10.15 and AVX enabled. "
#ifdef EIGEN_VECTORIZE_AVX2
#undef EIGEN_VECTORIZE_AVX2
#endif
#ifdef EIGEN_VECTORIZE_FMA
#undef EIGEN_VECTORIZE_FMA
#endif
#ifdef EIGEN_VECTORIZE_AVX512
#undef EIGEN_VECTORIZE_AVX512
#endif
#ifdef EIGEN_VECTORIZE_AVX512DQ
#undef EIGEN_VECTORIZE_AVX512DQ
#endif
#ifdef EIGEN_VECTORIZE_AVX512ER
#undef EIGEN_VECTORIZE_AVX512ER
#endif
#endif
// NOTE: Confirmed test failures in XCode 11.0, and XCode 11.2 with  -macosx-version-min=10.15 and AVX
// NOTE using -macosx-version-min=10.15 with Xcode 11.0 results in runtime segmentation faults in many tests, 11.2
// produce core dumps in 3 tests NOTE using -macosx-version-min=10.14 produces functioning and passing tests in all
// cases NOTE __clang_version__ "11.0.0 (clang-1100.0.33.8)"  XCode 11.0 <- Produces many segfault and core dumping
// tests
//                                                                    with  -macosx-version-min=10.15 and AVX
// NOTE __clang_version__ "11.0.0 (clang-1100.0.33.12)" XCode 11.2 <- Produces 3 core dumping tests with
//                                                                    -macosx-version-min=10.15 and AVX
#endif
 
// include files
 
// This extern "C" works around a MINGW-w64 compilation issue
// https://sourceforge.net/tracker/index.php?func=detail&aid=3018394&group_id=202880&atid=983354
// In essence, intrin.h is included by windows.h and also declares intrinsics (just as emmintrin.h etc. below do).
// However, intrin.h uses an extern "C" declaration, and g++ thus complains of duplicate declarations
// with conflicting linkage.  The linkage for intrinsics doesn't matter, but at that stage the compiler doesn't know;
// so, to avoid compile errors when windows.h is included after Eigen/Core, ensure intrinsics are extern "C" here too.
// notice that since these are C headers, the extern "C" is theoretically needed anyways.
extern "C" {
// ICC and Emscripten need the umbrella header instead of direct *mmintrin.h includes.
#if EIGEN_COMP_ICC || EIGEN_COMP_EMSCRIPTEN
#include <immintrin.h>
#else
#include <mmintrin.h>
#include <emmintrin.h>
#include <xmmintrin.h>
#ifdef EIGEN_VECTORIZE_SSE3
#include <pmmintrin.h>
#endif
#ifdef EIGEN_VECTORIZE_SSSE3
#include <tmmintrin.h>
#endif
#ifdef EIGEN_VECTORIZE_SSE4_1
#include <smmintrin.h>
#endif
#ifdef EIGEN_VECTORIZE_SSE4_2
#include <nmmintrin.h>
#endif
#if defined(EIGEN_VECTORIZE_AVX) || defined(EIGEN_VECTORIZE_AVX512)
#include <immintrin.h>
#endif
#endif
}  // end extern "C"
 
#elif defined(__VSX__) && !defined(__APPLE__)
 
#define EIGEN_VECTORIZE
#define EIGEN_VECTORIZE_VSX 1
// The POWER8 (ISA 2.07) vector instructions, such as 64-bit integer lane arithmetic and compares and vec_neg; a
// POWER7 target has VSX without them.
#if defined(__POWER8_VECTOR__)
#define EIGEN_VECTORIZE_POWER8_VECTOR
#endif
#define EIGEN_VECTORIZE_FMA
#include <altivec.h>
// We need to #undef macros defined by <altivec.h> that conflict with standard C++ names.
// => use __vector instead of vector
#undef bool
#undef vector
#undef pixel
 
#elif defined __ALTIVEC__
 
#define EIGEN_VECTORIZE
#define EIGEN_VECTORIZE_ALTIVEC
#define EIGEN_VECTORIZE_FMA
#include <altivec.h>
// We need to #undef macros defined by <altivec.h> that conflict with standard C++ names.
// => use __vector instead of vector
#undef bool
#undef vector
#undef pixel
 
#elif defined(EIGEN_ARM64_USE_SME) && !defined(EIGEN_ARM64_SME_USABLE)
 
#error \
    "EIGEN_ARM64_USE_SME requires a compiler targeting SME2 in scalable (SVE VLA) mode: build with e.g. -march=armv9.2-a+sme2 and without -msve-vector-bits."
 
#elif ((defined __ARM_NEON) || (defined __ARM_NEON__)) && !(defined EIGEN_ARM64_USE_SVE) && \
    !(defined EIGEN_ARM64_SME_SELECTED)
 
#define EIGEN_VECTORIZE
#define EIGEN_VECTORIZE_NEON
#include <arm_neon.h>
 
// We currently require SVE to be enabled explicitly via EIGEN_ARM64_USE_SVE and
// will not select the backend automatically
#elif (defined __ARM_FEATURE_SVE) && (defined EIGEN_ARM64_USE_SVE)
 
#define EIGEN_VECTORIZE
#define EIGEN_VECTORIZE_SVE
#include <arm_sve.h>
 
// Since we depend on knowing SVE vector length at compile-time, we need
// to ensure a fixed length is set
#if defined __ARM_FEATURE_SVE_BITS
#define EIGEN_ARM64_SVE_VL __ARM_FEATURE_SVE_BITS
 
// Architecture-mandated length constraints.
static_assert((EIGEN_ARM64_SVE_VL >= 128) && (EIGEN_ARM64_SVE_VL <= 2048) &&
                  ((EIGEN_ARM64_SVE_VL & (EIGEN_ARM64_SVE_VL - 1)) == 0),
              "SVE vector length must be 2^n for some n in [7, 11]");
#else
#error "Eigen requires a fixed SVE vector length but EIGEN_ARM64_SVE_VL is not set."
#endif
 
// TriangularMatrixMatrix.h puts a (2 * max(mr, nr))^2 panel of Scalar on the
// stack, and mr is 3 * PacketSize, so for float -- the widest scalar this backend
// vectorizes -- the panel is 9 * VL^2 / 64 bytes
// -- 144 kB at VL=1024 and 576 kB at VL=2048, past the 128 kB default, and the
// backend does not compile at those lengths without more room. Only raise Eigen's
// default; an explicit user limit remains authoritative, including 0.
#if defined(EIGEN_STACK_ALLOCATION_LIMIT_WAS_DEFAULTED) && \
    EIGEN_STACK_ALLOCATION_LIMIT < (9 * EIGEN_ARM64_SVE_VL * EIGEN_ARM64_SVE_VL / 64)
#undef EIGEN_STACK_ALLOCATION_LIMIT
#define EIGEN_STACK_ALLOCATION_LIMIT (9 * EIGEN_ARM64_SVE_VL * EIGEN_ARM64_SVE_VL / 64)
#endif
 
// Selected automatically whenever the toolchain can provide it; see
// EIGEN_ARM64_SME_SELECTED above for the conditions and the opt-out.
#elif defined(EIGEN_ARM64_SME_SELECTED)
 
#define EIGEN_VECTORIZE
#define EIGEN_VECTORIZE_SME
#include <arm_neon.h>
#include <arm_sme.h>
 
// Double-precision outer products (FMOPA into a ZA.D tile) need the optional
// FEAT_SME_F64F64, which each compiler reports differently: GCC defines the ACLE
// macro, clang defines no macro but gates the builtin on the target feature.
// Both halves are needed -- clang otherwise accepts svmopa_za64_f64_m without the
// feature, so a missed gate faults at run time rather than at build time.
#if !defined(EIGEN_ARM64_NO_SME_F64F64) && \
    (defined(__ARM_FEATURE_SME_F64F64) || EIGEN_HAS_BUILTIN(__builtin_sme_svmopa_za64_f64_m))
#define EIGEN_VECTORIZE_SME_F64F64
#endif
 
#elif EIGEN_ARCH_RISCV
 
#if defined(__riscv_zfh)
#define EIGEN_HAS_BUILTIN_FLOAT16
#endif
 
// We currently require RVV to be enabled explicitly via EIGEN_RISCV64_USE_RVV and
// will not select the backend automatically
#if (defined EIGEN_RISCV64_USE_RVV10)
 
#define EIGEN_VECTORIZE
#define EIGEN_VECTORIZE_RVV10
#include <riscv_vector.h>
 
// Since we depend on knowing RVV vector length at compile-time, we need
// to ensure a fixed length is set
#if defined(__riscv_v_fixed_vlen)
#define EIGEN_RISCV64_RVV_VL __riscv_v_fixed_vlen
#if __riscv_v_fixed_vlen >= 256
#undef EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT
#define EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT 1
#endif
#else
#error "Eigen requires a fixed RVV vector length but -mrvv-vector-bits=zvl is not set."
#endif
 
// Raise Eigen's own default only, as the SVE block above does: an explicit limit is a caller
// policy on stack safety, and rewriting it defeats the policy. nomalloc, bdcsvd, jacobisvd,
// diagonalview and diagonalmatrices set 0 to switch the check off, and clobbering that re-enables
// alloca underneath the very tests written to catch it.
#ifdef EIGEN_STACK_ALLOCATION_LIMIT_WAS_DEFAULTED
#undef EIGEN_STACK_ALLOCATION_LIMIT
#if __riscv_v_fixed_vlen <= 512
#define EIGEN_STACK_ALLOCATION_LIMIT 196608
#else
#define EIGEN_STACK_ALLOCATION_LIMIT 393216
#endif
#endif
 
#if defined(__riscv_zvfh) && defined(__riscv_zfh)
#define EIGEN_VECTORIZE_RVV10FP16
#elif defined(__riscv_zvfh)
#if defined(__GNUC__) || defined(__clang__)
#warning "The Eigen::Half vectorization requires Zfh and Zvfh extensions."
#elif defined(_MSC_VER)
#pragma message("The Eigen::Half vectorization requires Zfh and Zvfh extensions.")
#endif
#endif
 
#if defined(__riscv_zvfbfwma)
#define EIGEN_VECTORIZE_RVV10BF16
#endif
 
#endif  // defined(EIGEN_ARCH_RISCV)
 
// ZVector needs the vector float instructions of z14 (__ARCH__ 12); below that, s390x stays scalar.
#elif (defined __s390x__ && defined __VEC__ && (!defined(__ARCH__) || __ARCH__ >= 12))
 
#define EIGEN_VECTORIZE
#define EIGEN_VECTORIZE_ZVECTOR
#include <vecintrin.h>
 
#elif defined __mips_msa
 
// Limit MSA optimizations to little-endian CPUs for now.
// TODO: Perhaps, eventually support MSA optimizations on big-endian CPUs?
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#if defined(__LP64__)
#define EIGEN_MIPS_64
#else
#define EIGEN_MIPS_32
#endif
#define EIGEN_VECTORIZE
#define EIGEN_VECTORIZE_MSA
#include <msa.h>
#endif
 
#elif (defined __loongarch64 && defined __loongarch_sx)
 
#define EIGEN_VECTORIZE
#define EIGEN_VECTORIZE_LSX
#include <lsxintrin.h>
 
#elif defined __HVX__ && (__HVX_LENGTH__ == 128)
 
#define EIGEN_VECTORIZE
#define EIGEN_VECTORIZE_HVX
#include <hexagon_types.h>
 
#endif
#endif
 
// The backend blocks above are the only consumers; do not leak it into user code.
#undef EIGEN_STACK_ALLOCATION_LIMIT_WAS_DEFAULTED
 
// Following the Arm ACLE arm_neon.h should also include arm_fp16.h but not all
// compilers seem to follow this. We therefore include it explicitly.
// See also: https://bugs.llvm.org/show_bug.cgi?id=47955
#if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC)
#include <arm_fp16.h>
#endif
 
// Enable FMA for ARM.
#if defined(__ARM_FEATURE_FMA)
#define EIGEN_VECTORIZE_FMA
#endif
 
#if defined(__F16C__) && !defined(EIGEN_GPUCC)
// We can use the optimized fp16 to float and float to fp16 conversion routines
#define EIGEN_HAS_FP16_C
 
#if EIGEN_COMP_GNUC
// Make sure immintrin.h is included, even if e.g. vectorization is
// explicitly disabled (see also issue #2395).
// Note that FP16C intrinsics for gcc and clang are included by immintrin.h,
// as opposed to emmintrin.h as suggested by Intel:
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#othertechs=FP16C&expand=1711
#include <immintrin.h>
#endif
#endif
 
#if defined EIGEN_CUDACC
#define EIGEN_VECTORIZE_GPU
#include <vector_types.h>
#include <cuda_runtime_api.h>
#if defined(EIGEN_HAS_CUDA_FP16)
#include <cuda_fp16.h>
#endif
#endif
 
#if defined(EIGEN_HIPCC)
#define EIGEN_VECTORIZE_GPU
#include <hip/hip_vector_types.h>
#include <hip/hip_fp16.h>
#define EIGEN_HAS_HIP_BF16
#include <hip/hip_bfloat16.h>
#endif
 
#if defined(__riscv)
// Defines the default LMUL for RISC-V
#ifndef EIGEN_RISCV64_DEFAULT_LMUL
#define EIGEN_RISCV64_DEFAULT_LMUL 1
#endif
#endif

// IWYU pragma: private
#include "../InternalHeaderCheck.h"

#ifndef EIGEN_SCALAR_MADD_USE_FMA
#ifdef EIGEN_VECTORIZE_FMA
#define EIGEN_SCALAR_MADD_USE_FMA 1
#else
#define EIGEN_SCALAR_MADD_USE_FMA 0
#endif
#endif
 
namespace Eigen {
 
inline static const char* SimdInstructionSetsInUse(void) {
#if defined(EIGEN_VECTORIZE_AVX512)
  return "AVX512, FMA, AVX2, AVX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
#elif defined(EIGEN_VECTORIZE_AVX)
  return "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
#elif defined(EIGEN_VECTORIZE_SSE4_2)
  return "SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
#elif defined(EIGEN_VECTORIZE_SSE4_1)
  return "SSE, SSE2, SSE3, SSSE3, SSE4.1";
#elif defined(EIGEN_VECTORIZE_SSSE3)
  return "SSE, SSE2, SSE3, SSSE3";
#elif defined(EIGEN_VECTORIZE_SSE3)
  return "SSE, SSE2, SSE3";
#elif defined(EIGEN_VECTORIZE_SSE2)
  return "SSE, SSE2";
#elif defined(EIGEN_VECTORIZE_ALTIVEC)
  return "AltiVec";
#elif defined(EIGEN_VECTORIZE_POWER8_VECTOR)
  return "VSX, POWER8 vector";
#elif defined(EIGEN_VECTORIZE_VSX)
  return "VSX";
#elif defined(EIGEN_VECTORIZE_NEON)
  return "ARM NEON";
#elif defined(EIGEN_VECTORIZE_SME)
  return "ARM SME";
#elif defined(EIGEN_VECTORIZE_SVE)
  return "ARM SVE";
#elif defined(EIGEN_VECTORIZE_ZVECTOR)
  return "S390X ZVECTOR";
#elif defined(EIGEN_VECTORIZE_MSA)
  return "MIPS MSA";
#elif defined(EIGEN_VECTORIZE_LSX)
  return "LOONGARCH64 LSX";
#elif defined(EIGEN_VECTORIZE_RVV10)
  return "RVV";
#else
  return "None";
#endif
}
 
}  // end namespace Eigen
 
#endif  // EIGEN_CONFIGURE_VECTORIZATION_H