Add support for AVX-512 family of instruction sets
This patch adds compiler and runtime tests (user and kernel) for following instruction sets: avx512f, avx512cd, avx512er, avx512pf, avx512bw, avx512dq, avx512vl, avx512ifma, avx512vbmi. note: Linux support for AVX-512F (Foundation) instruction set started with linux v3.15 Signed-off-by: Gvozden Neskovic <neskovic@gmail.com> Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov> Issue #4952
This commit is contained in:
Gvozden Neskovic
Brian Behlendorf
parent
6d836e6f8b
commit
32ffaa3de5
|
|
@ -12,6 +12,15 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [
|
||||||
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE4_2
|
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE4_2
|
||||||
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX
|
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX
|
||||||
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX2
|
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX2
|
||||||
|
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512F
|
||||||
|
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512CD
|
||||||
|
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512DQ
|
||||||
|
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512BW
|
||||||
|
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512IFMA
|
||||||
|
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512VBMI
|
||||||
|
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512PF
|
||||||
|
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512ER
|
||||||
|
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512VL
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
])
|
])
|
||||||
|
|
@ -170,3 +179,183 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX2], [
|
||||||
AC_MSG_RESULT([no])
|
AC_MSG_RESULT([no])
|
||||||
])
|
])
|
||||||
])
|
])
|
||||||
|
|
||||||
|
dnl #
|
||||||
|
dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512F
|
||||||
|
dnl #
|
||||||
|
AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512F], [
|
||||||
|
AC_MSG_CHECKING([whether host toolchain supports AVX512F])
|
||||||
|
|
||||||
|
AC_LINK_IFELSE([AC_LANG_SOURCE([
|
||||||
|
[
|
||||||
|
void main()
|
||||||
|
{
|
||||||
|
__asm__ __volatile__("vpandd %zmm0,%zmm1,%zmm2");
|
||||||
|
}
|
||||||
|
]])], [
|
||||||
|
AC_MSG_RESULT([yes])
|
||||||
|
AC_DEFINE([HAVE_AVX512F], 1, [Define if host toolchain supports AVX512F])
|
||||||
|
], [
|
||||||
|
AC_MSG_RESULT([no])
|
||||||
|
])
|
||||||
|
])
|
||||||
|
|
||||||
|
dnl #
|
||||||
|
dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512CD
|
||||||
|
dnl #
|
||||||
|
AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512CD], [
|
||||||
|
AC_MSG_CHECKING([whether host toolchain supports AVX512CD])
|
||||||
|
|
||||||
|
AC_LINK_IFELSE([AC_LANG_SOURCE([
|
||||||
|
[
|
||||||
|
void main()
|
||||||
|
{
|
||||||
|
__asm__ __volatile__("vplzcntd %zmm0,%zmm1");
|
||||||
|
}
|
||||||
|
]])], [
|
||||||
|
AC_MSG_RESULT([yes])
|
||||||
|
AC_DEFINE([HAVE_AVX512CD], 1, [Define if host toolchain supports AVX512CD])
|
||||||
|
], [
|
||||||
|
AC_MSG_RESULT([no])
|
||||||
|
])
|
||||||
|
])
|
||||||
|
|
||||||
|
dnl #
|
||||||
|
dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512DQ
|
||||||
|
dnl #
|
||||||
|
AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512DQ], [
|
||||||
|
AC_MSG_CHECKING([whether host toolchain supports AVX512DQ])
|
||||||
|
|
||||||
|
AC_LINK_IFELSE([AC_LANG_SOURCE([
|
||||||
|
[
|
||||||
|
void main()
|
||||||
|
{
|
||||||
|
__asm__ __volatile__("vandpd %zmm0,%zmm1,%zmm2");
|
||||||
|
}
|
||||||
|
]])], [
|
||||||
|
AC_MSG_RESULT([yes])
|
||||||
|
AC_DEFINE([HAVE_AVX512DQ], 1, [Define if host toolchain supports AVX512DQ])
|
||||||
|
], [
|
||||||
|
AC_MSG_RESULT([no])
|
||||||
|
])
|
||||||
|
])
|
||||||
|
|
||||||
|
dnl #
|
||||||
|
dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512BW
|
||||||
|
dnl #
|
||||||
|
AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512BW], [
|
||||||
|
AC_MSG_CHECKING([whether host toolchain supports AVX512BW])
|
||||||
|
|
||||||
|
AC_LINK_IFELSE([AC_LANG_SOURCE([
|
||||||
|
[
|
||||||
|
void main()
|
||||||
|
{
|
||||||
|
__asm__ __volatile__("vpshufb %zmm0,%zmm1,%zmm2");
|
||||||
|
}
|
||||||
|
]])], [
|
||||||
|
AC_MSG_RESULT([yes])
|
||||||
|
AC_DEFINE([HAVE_AVX512BW], 1, [Define if host toolchain supports AVX512BW])
|
||||||
|
], [
|
||||||
|
AC_MSG_RESULT([no])
|
||||||
|
])
|
||||||
|
])
|
||||||
|
|
||||||
|
dnl #
|
||||||
|
dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512IFMA
|
||||||
|
dnl #
|
||||||
|
AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512IFMA], [
|
||||||
|
AC_MSG_CHECKING([whether host toolchain supports AVX512IFMA])
|
||||||
|
|
||||||
|
AC_LINK_IFELSE([AC_LANG_SOURCE([
|
||||||
|
[
|
||||||
|
void main()
|
||||||
|
{
|
||||||
|
__asm__ __volatile__("vpmadd52luq %zmm0,%zmm1,%zmm2");
|
||||||
|
}
|
||||||
|
]])], [
|
||||||
|
AC_MSG_RESULT([yes])
|
||||||
|
AC_DEFINE([HAVE_AVX512IFMA], 1, [Define if host toolchain supports AVX512IFMA])
|
||||||
|
], [
|
||||||
|
AC_MSG_RESULT([no])
|
||||||
|
])
|
||||||
|
])
|
||||||
|
|
||||||
|
dnl #
|
||||||
|
dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512VBMI
|
||||||
|
dnl #
|
||||||
|
AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512VBMI], [
|
||||||
|
AC_MSG_CHECKING([whether host toolchain supports AVX512VBMI])
|
||||||
|
|
||||||
|
AC_LINK_IFELSE([AC_LANG_SOURCE([
|
||||||
|
[
|
||||||
|
void main()
|
||||||
|
{
|
||||||
|
__asm__ __volatile__("vpermb %zmm0,%zmm1,%zmm2");
|
||||||
|
}
|
||||||
|
]])], [
|
||||||
|
AC_MSG_RESULT([yes])
|
||||||
|
AC_DEFINE([HAVE_AVX512VBMI], 1, [Define if host toolchain supports AVX512VBMI])
|
||||||
|
], [
|
||||||
|
AC_MSG_RESULT([no])
|
||||||
|
])
|
||||||
|
])
|
||||||
|
|
||||||
|
dnl #
|
||||||
|
dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512PF
|
||||||
|
dnl #
|
||||||
|
AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512PF], [
|
||||||
|
AC_MSG_CHECKING([whether host toolchain supports AVX512PF])
|
||||||
|
|
||||||
|
AC_LINK_IFELSE([AC_LANG_SOURCE([
|
||||||
|
[
|
||||||
|
void main()
|
||||||
|
{
|
||||||
|
__asm__ __volatile__("vgatherpf0dps (%rsi,%zmm0,4){%k1}");
|
||||||
|
}
|
||||||
|
]])], [
|
||||||
|
AC_MSG_RESULT([yes])
|
||||||
|
AC_DEFINE([HAVE_AVX512PF], 1, [Define if host toolchain supports AVX512PF])
|
||||||
|
], [
|
||||||
|
AC_MSG_RESULT([no])
|
||||||
|
])
|
||||||
|
])
|
||||||
|
|
||||||
|
dnl #
|
||||||
|
dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512ER
|
||||||
|
dnl #
|
||||||
|
AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512ER], [
|
||||||
|
AC_MSG_CHECKING([whether host toolchain supports AVX512ER])
|
||||||
|
|
||||||
|
AC_LINK_IFELSE([AC_LANG_SOURCE([
|
||||||
|
[
|
||||||
|
void main()
|
||||||
|
{
|
||||||
|
__asm__ __volatile__("vexp2pd %zmm0,%zmm1");
|
||||||
|
}
|
||||||
|
]])], [
|
||||||
|
AC_MSG_RESULT([yes])
|
||||||
|
AC_DEFINE([HAVE_AVX512ER], 1, [Define if host toolchain supports AVX512ER])
|
||||||
|
], [
|
||||||
|
AC_MSG_RESULT([no])
|
||||||
|
])
|
||||||
|
])
|
||||||
|
|
||||||
|
dnl #
|
||||||
|
dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512VL
|
||||||
|
dnl #
|
||||||
|
AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512VL], [
|
||||||
|
AC_MSG_CHECKING([whether host toolchain supports AVX512VL])
|
||||||
|
|
||||||
|
AC_LINK_IFELSE([AC_LANG_SOURCE([
|
||||||
|
[
|
||||||
|
void main()
|
||||||
|
{
|
||||||
|
__asm__ __volatile__("vpabsq %zmm0,%zmm1");
|
||||||
|
}
|
||||||
|
]])], [
|
||||||
|
AC_MSG_RESULT([yes])
|
||||||
|
AC_DEFINE([HAVE_AVX512VL], 1, [Define if host toolchain supports AVX512VL])
|
||||||
|
], [
|
||||||
|
AC_MSG_RESULT([no])
|
||||||
|
])
|
||||||
|
])
|
||||||
|
|
|
||||||
|
|
@ -43,10 +43,25 @@
|
||||||
* zfs_ssse3_available()
|
* zfs_ssse3_available()
|
||||||
* zfs_sse4_1_available()
|
* zfs_sse4_1_available()
|
||||||
* zfs_sse4_2_available()
|
* zfs_sse4_2_available()
|
||||||
|
*
|
||||||
* zfs_avx_available()
|
* zfs_avx_available()
|
||||||
* zfs_avx2_available()
|
* zfs_avx2_available()
|
||||||
|
*
|
||||||
* zfs_bmi1_available()
|
* zfs_bmi1_available()
|
||||||
* zfs_bmi2_available()
|
* zfs_bmi2_available()
|
||||||
|
*
|
||||||
|
* zfs_avx512f_available()
|
||||||
|
* zfs_avx512cd_available()
|
||||||
|
* zfs_avx512er_available()
|
||||||
|
* zfs_avx512pf_available()
|
||||||
|
* zfs_avx512bw_available()
|
||||||
|
* zfs_avx512dq_available()
|
||||||
|
* zfs_avx512vl_available()
|
||||||
|
* zfs_avx512ifma_available()
|
||||||
|
* zfs_avx512vbmi_available()
|
||||||
|
*
|
||||||
|
* NOTE(AVX-512VL): If using AVX-512 instructions with 128Bit registers
|
||||||
|
* also add zfs_avx512vl_available() to feature check.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#ifndef _SIMD_X86_H
|
#ifndef _SIMD_X86_H
|
||||||
|
|
@ -124,7 +139,16 @@ typedef enum cpuid_inst_sets {
|
||||||
AVX,
|
AVX,
|
||||||
AVX2,
|
AVX2,
|
||||||
BMI1,
|
BMI1,
|
||||||
BMI2
|
BMI2,
|
||||||
|
AVX512F,
|
||||||
|
AVX512CD,
|
||||||
|
AVX512DQ,
|
||||||
|
AVX512BW,
|
||||||
|
AVX512IFMA,
|
||||||
|
AVX512VBMI,
|
||||||
|
AVX512PF,
|
||||||
|
AVX512ER,
|
||||||
|
AVX512VL
|
||||||
} cpuid_inst_sets_t;
|
} cpuid_inst_sets_t;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
@ -132,11 +156,21 @@ typedef enum cpuid_inst_sets {
|
||||||
*/
|
*/
|
||||||
typedef struct cpuid_feature_desc {
|
typedef struct cpuid_feature_desc {
|
||||||
uint32_t leaf; /* CPUID leaf */
|
uint32_t leaf; /* CPUID leaf */
|
||||||
uint32_t subleaf; /* CPUID subleaf */
|
uint32_t subleaf; /* CPUID sub-leaf */
|
||||||
uint32_t flag; /* bit mask of the feature */
|
uint32_t flag; /* bit mask of the feature */
|
||||||
cpuid_regs_t reg; /* which CPUID return register to test */
|
cpuid_regs_t reg; /* which CPUID return register to test */
|
||||||
} cpuid_feature_desc_t;
|
} cpuid_feature_desc_t;
|
||||||
|
|
||||||
|
#define _AVX512F_BIT (1U << 16)
|
||||||
|
#define _AVX512CD_BIT (_AVX512F_BIT | (1U << 28))
|
||||||
|
#define _AVX512DQ_BIT (_AVX512F_BIT | (1U << 17))
|
||||||
|
#define _AVX512BW_BIT (_AVX512F_BIT | (1U << 30))
|
||||||
|
#define _AVX512IFMA_BIT (_AVX512F_BIT | (1U << 21))
|
||||||
|
#define _AVX512VBMI_BIT (1U << 1) /* AVX512F_BIT is on another leaf */
|
||||||
|
#define _AVX512PF_BIT (_AVX512F_BIT | (1U << 26))
|
||||||
|
#define _AVX512ER_BIT (_AVX512F_BIT | (1U << 27))
|
||||||
|
#define _AVX512VL_BIT (1U << 31) /* if used also check other levels */
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Descriptions of supported instruction sets
|
* Descriptions of supported instruction sets
|
||||||
*/
|
*/
|
||||||
|
|
@ -151,7 +185,16 @@ static const cpuid_feature_desc_t cpuid_features[] = {
|
||||||
[AVX] = {1U, 0U, 1U << 28, ECX },
|
[AVX] = {1U, 0U, 1U << 28, ECX },
|
||||||
[AVX2] = {7U, 0U, 1U << 5, EBX },
|
[AVX2] = {7U, 0U, 1U << 5, EBX },
|
||||||
[BMI1] = {7U, 0U, 1U << 3, EBX },
|
[BMI1] = {7U, 0U, 1U << 3, EBX },
|
||||||
[BMI2] = {7U, 0U, 1U << 8, EBX }
|
[BMI2] = {7U, 0U, 1U << 8, EBX },
|
||||||
|
[AVX512F] = {7U, 0U, _AVX512F_BIT, EBX },
|
||||||
|
[AVX512CD] = {7U, 0U, _AVX512CD_BIT, EBX },
|
||||||
|
[AVX512DQ] = {7U, 0U, _AVX512DQ_BIT, EBX },
|
||||||
|
[AVX512BW] = {7U, 0U, _AVX512BW_BIT, EBX },
|
||||||
|
[AVX512IFMA] = {7U, 0U, _AVX512IFMA_BIT, EBX },
|
||||||
|
[AVX512VBMI] = {7U, 0U, _AVX512VBMI_BIT, ECX },
|
||||||
|
[AVX512PF] = {7U, 0U, _AVX512PF_BIT, EBX },
|
||||||
|
[AVX512ER] = {7U, 0U, _AVX512ER_BIT, EBX },
|
||||||
|
[AVX512VL] = {7U, 0U, _AVX512ER_BIT, EBX }
|
||||||
};
|
};
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
@ -187,14 +230,14 @@ __cpuid_check_feature(const cpuid_feature_desc_t *desc)
|
||||||
*/
|
*/
|
||||||
__cpuid_count(desc->leaf, desc->subleaf,
|
__cpuid_count(desc->leaf, desc->subleaf,
|
||||||
r[EAX], r[EBX], r[ECX], r[EDX]);
|
r[EAX], r[EBX], r[ECX], r[EDX]);
|
||||||
return (!!(r[desc->reg] & desc->flag));
|
return ((r[desc->reg] & desc->flag) == desc->flag);
|
||||||
}
|
}
|
||||||
return (B_FALSE);
|
return (B_FALSE);
|
||||||
}
|
}
|
||||||
|
|
||||||
#define CPUID_FEATURE_CHECK(name, id) \
|
#define CPUID_FEATURE_CHECK(name, id) \
|
||||||
static inline boolean_t \
|
static inline boolean_t \
|
||||||
__cpuid_has_ ## name(void)\
|
__cpuid_has_ ## name(void) \
|
||||||
{ \
|
{ \
|
||||||
return (__cpuid_check_feature(&cpuid_features[id])); \
|
return (__cpuid_check_feature(&cpuid_features[id])); \
|
||||||
}
|
}
|
||||||
|
|
@ -213,16 +256,25 @@ CPUID_FEATURE_CHECK(avx2, AVX2);
|
||||||
CPUID_FEATURE_CHECK(osxsave, OSXSAVE);
|
CPUID_FEATURE_CHECK(osxsave, OSXSAVE);
|
||||||
CPUID_FEATURE_CHECK(bmi1, BMI1);
|
CPUID_FEATURE_CHECK(bmi1, BMI1);
|
||||||
CPUID_FEATURE_CHECK(bmi2, BMI2);
|
CPUID_FEATURE_CHECK(bmi2, BMI2);
|
||||||
|
CPUID_FEATURE_CHECK(avx512f, AVX512F);
|
||||||
|
CPUID_FEATURE_CHECK(avx512cd, AVX512CD);
|
||||||
|
CPUID_FEATURE_CHECK(avx512dq, AVX512DQ);
|
||||||
|
CPUID_FEATURE_CHECK(avx512bw, AVX512BW);
|
||||||
|
CPUID_FEATURE_CHECK(avx512ifma, AVX512IFMA);
|
||||||
|
CPUID_FEATURE_CHECK(avx512vbmi, AVX512VBMI);
|
||||||
|
CPUID_FEATURE_CHECK(avx512pf, AVX512PF);
|
||||||
|
CPUID_FEATURE_CHECK(avx512er, AVX512ER);
|
||||||
|
CPUID_FEATURE_CHECK(avx512vl, AVX512VL);
|
||||||
|
|
||||||
#endif /* !defined(_KERNEL) */
|
#endif /* !defined(_KERNEL) */
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Detect ymm register set support
|
* Detect register set support
|
||||||
*/
|
*/
|
||||||
static inline boolean_t
|
static inline boolean_t
|
||||||
__ymm_enabled(void)
|
__simd_state_enabled(const uint64_t state)
|
||||||
{
|
{
|
||||||
static const uint64_t XSTATE_SSE_AVX = 0x2 | 0x4;
|
|
||||||
boolean_t has_osxsave;
|
boolean_t has_osxsave;
|
||||||
uint64_t xcr0;
|
uint64_t xcr0;
|
||||||
|
|
||||||
|
|
@ -238,9 +290,16 @@ __ymm_enabled(void)
|
||||||
return (B_FALSE);
|
return (B_FALSE);
|
||||||
|
|
||||||
xcr0 = xgetbv(0);
|
xcr0 = xgetbv(0);
|
||||||
return ((xcr0 & XSTATE_SSE_AVX) == XSTATE_SSE_AVX);
|
return ((xcr0 & state) == state);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define _XSTATE_SSE_AVX (0x2 | 0x4)
|
||||||
|
#define _XSTATE_AVX512 (0xE0 | _XSTATE_SSE_AVX)
|
||||||
|
|
||||||
|
#define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX)
|
||||||
|
#define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512)
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Check if SSE instruction set is available
|
* Check if SSE instruction set is available
|
||||||
*/
|
*/
|
||||||
|
|
@ -383,6 +442,168 @@ zfs_bmi2_available(void)
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
* AVX-512 family of instruction sets:
|
||||||
|
*
|
||||||
|
* AVX512F Foundation
|
||||||
|
* AVX512CD Conflict Detection Instructions
|
||||||
|
* AVX512ER Exponential and Reciprocal Instructions
|
||||||
|
* AVX512PF Prefetch Instructions
|
||||||
|
*
|
||||||
|
* AVX512BW Byte and Word Instructions
|
||||||
|
* AVX512DQ Double-word and Quadword Instructions
|
||||||
|
* AVX512VL Vector Length Extensions
|
||||||
|
*
|
||||||
|
* AVX512IFMA Integer Fused Multiply Add (Not supported by kernel 4.4)
|
||||||
|
* AVX512VBMI Vector Byte Manipulation Instructions
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
/* Check if AVX512F instruction set is available */
|
||||||
|
static inline boolean_t
|
||||||
|
zfs_avx512f_available(void)
|
||||||
|
{
|
||||||
|
boolean_t has_avx512 = B_FALSE;
|
||||||
|
|
||||||
|
#if defined(_KERNEL) && defined(X86_FEATURE_AVX512F)
|
||||||
|
has_avx512 = !!boot_cpu_has(X86_FEATURE_AVX512F);
|
||||||
|
#elif !defined(_KERNEL)
|
||||||
|
has_avx512 = __cpuid_has_avx512f();
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return (has_avx512 && __zmm_enabled());
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Check if AVX512CD instruction set is available */
|
||||||
|
static inline boolean_t
|
||||||
|
zfs_avx512cd_available(void)
|
||||||
|
{
|
||||||
|
boolean_t has_avx512 = B_FALSE;
|
||||||
|
|
||||||
|
#if defined(_KERNEL) && defined(X86_FEATURE_AVX512CD)
|
||||||
|
has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
|
||||||
|
boot_cpu_has(X86_FEATURE_AVX512CD);
|
||||||
|
#elif !defined(_KERNEL)
|
||||||
|
has_avx512 = __cpuid_has_avx512cd();
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return (has_avx512 && __zmm_enabled());
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Check if AVX512ER instruction set is available */
|
||||||
|
static inline boolean_t
|
||||||
|
zfs_avx512er_available(void)
|
||||||
|
{
|
||||||
|
boolean_t has_avx512 = B_FALSE;
|
||||||
|
|
||||||
|
#if defined(_KERNEL) && defined(X86_FEATURE_AVX512ER)
|
||||||
|
has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
|
||||||
|
boot_cpu_has(X86_FEATURE_AVX512ER);
|
||||||
|
#elif !defined(_KERNEL)
|
||||||
|
has_avx512 = __cpuid_has_avx512er();
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return (has_avx512 && __zmm_enabled());
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Check if AVX512PF instruction set is available */
|
||||||
|
static inline boolean_t
|
||||||
|
zfs_avx512pf_available(void)
|
||||||
|
{
|
||||||
|
boolean_t has_avx512 = B_FALSE;
|
||||||
|
|
||||||
|
#if defined(_KERNEL) && defined(X86_FEATURE_AVX512PF)
|
||||||
|
has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
|
||||||
|
boot_cpu_has(X86_FEATURE_AVX512PF);
|
||||||
|
#elif !defined(_KERNEL)
|
||||||
|
has_avx512 = __cpuid_has_avx512pf();
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return (has_avx512 && __zmm_enabled());
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Check if AVX512BW instruction set is available */
|
||||||
|
static inline boolean_t
|
||||||
|
zfs_avx512bw_available(void)
|
||||||
|
{
|
||||||
|
boolean_t has_avx512 = B_FALSE;
|
||||||
|
|
||||||
|
#if defined(_KERNEL) && defined(X86_FEATURE_AVX512BW)
|
||||||
|
has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
|
||||||
|
boot_cpu_has(X86_FEATURE_AVX512BW);
|
||||||
|
#elif !defined(_KERNEL)
|
||||||
|
has_avx512 = __cpuid_has_avx512bw();
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return (has_avx512 && __zmm_enabled());
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Check if AVX512DQ instruction set is available */
|
||||||
|
static inline boolean_t
|
||||||
|
zfs_avx512dq_available(void)
|
||||||
|
{
|
||||||
|
boolean_t has_avx512 = B_FALSE;
|
||||||
|
|
||||||
|
#if defined(_KERNEL) && defined(X86_FEATURE_AVX512DQ)
|
||||||
|
has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
|
||||||
|
boot_cpu_has(X86_FEATURE_AVX512DQ);
|
||||||
|
#elif !defined(_KERNEL)
|
||||||
|
has_avx512 = __cpuid_has_avx512dq();
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return (has_avx512 && __zmm_enabled());
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Check if AVX512VL instruction set is available */
|
||||||
|
static inline boolean_t
|
||||||
|
zfs_avx512vl_available(void)
|
||||||
|
{
|
||||||
|
boolean_t has_avx512 = B_FALSE;
|
||||||
|
|
||||||
|
#if defined(_KERNEL) && defined(X86_FEATURE_AVX512VL)
|
||||||
|
has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
|
||||||
|
boot_cpu_has(X86_FEATURE_AVX512VL);
|
||||||
|
#elif !defined(_KERNEL)
|
||||||
|
has_avx512 = __cpuid_has_avx512vl();
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return (has_avx512 && __zmm_enabled());
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Check if AVX512IFMA instruction set is available */
|
||||||
|
static inline boolean_t
|
||||||
|
zfs_avx512ifma_available(void)
|
||||||
|
{
|
||||||
|
boolean_t has_avx512 = B_FALSE;
|
||||||
|
|
||||||
|
#if defined(_KERNEL) && defined(X86_FEATURE_AVX512IFMA)
|
||||||
|
has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
|
||||||
|
boot_cpu_has(X86_FEATURE_AVX512IFMA);
|
||||||
|
#elif !defined(_KERNEL)
|
||||||
|
has_avx512 = __cpuid_has_avx512ifma();
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return (has_avx512 && __zmm_enabled());
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Check if AVX512VBMI instruction set is available */
|
||||||
|
static inline boolean_t
|
||||||
|
zfs_avx512vbmi_available(void)
|
||||||
|
{
|
||||||
|
boolean_t has_avx512 = B_FALSE;
|
||||||
|
|
||||||
|
#if defined(_KERNEL) && defined(X86_FEATURE_AVX512VBMI)
|
||||||
|
has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
|
||||||
|
boot_cpu_has(X86_FEATURE_AVX512VBMI);
|
||||||
|
#elif !defined(_KERNEL)
|
||||||
|
has_avx512 = __cpuid_has_avx512f() &&
|
||||||
|
__cpuid_has_avx512vbmi();
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return (has_avx512 && __zmm_enabled());
|
||||||
|
}
|
||||||
|
|
||||||
#endif /* defined(__x86) */
|
#endif /* defined(__x86) */
|
||||||
|
|
||||||
#endif /* _SIMD_X86_H */
|
#endif /* _SIMD_X86_H */
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue