From 3b52ccd7d7f5e59a1306b7e904e0e244248f94d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20F=C3=BCl=C3=B6p?= Date: Wed, 9 Feb 2022 21:50:10 +0100 Subject: [PATCH] Linux 5.16 compat: don't use XSTATE_XSAVE to save FPU state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linux 5.16 moved XSTATE_XSAVE and XSTATE_XRESTORE out of our reach, so add our own XSAVE{,OPT,S} code and use it for Linux 5.16. Please note that this differs from previous behavior in that it won't handle exceptions created by XSAVE an XRSTOR. This is sensible for three reasons. - Exceptions during XSAVE and XRSTOR can only occur if the feature is not supported or enabled or the memory operand isn't aligned on a 64 byte boundary. If this happens something else went terribly wrong, and it may be better to stop execution. - Previously we just printed a warning and didn't handle the fault, this is arguable for the above reason. - All other *SAVE instruction also don't handle exceptions, so this at least aligns behavior. Finally add a test to catch such a regression in the future. Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf Signed-off-by: Attila Fülöp Closes #13042 Closes #13059 --- config/kernel-fpu.m4 | 43 +++++- config/toolchain-simd.m4 | 66 +++++++++ configure.ac | 1 + include/os/linux/kernel/linux/simd_x86.h | 125 +++++++++++++++++- module/zcommon/zfs_prop.c | 4 +- tests/runfiles/linux.run | 6 + tests/zfs-tests/tests/functional/Makefile.am | 1 + .../tests/functional/simd/Makefile.am | 2 + .../tests/functional/simd/simd_supported.ksh | 58 ++++++++ 9 files changed, 300 insertions(+), 6 deletions(-) create mode 100644 tests/zfs-tests/tests/functional/simd/Makefile.am create mode 100755 tests/zfs-tests/tests/functional/simd/simd_supported.ksh diff --git a/config/kernel-fpu.m4 b/config/kernel-fpu.m4 index faa64f1ec4..06280239dd 100644 --- a/config/kernel-fpu.m4 +++ b/config/kernel-fpu.m4 @@ -1,9 +1,12 @@ -dnl # +dnl # dnl # Handle differences in kernel FPU code. dnl # dnl # Kernel dnl # 5.16: XCR code put into asm/fpu/xcr.h -dnl # HAVE_KERNEL_FPU_XCR_HEADER +dnl # HAVE_KERNEL_FPU_XCR_HEADER +dnl # +dnl # XSTATE_XSAVE and XSTATE_XRESTORE aren't accessible any more +dnl # HAVE_KERNEL_FPU_XSAVE_INTERNAL dnl # dnl # 5.0: Wrappers have been introduced to save/restore the FPU state. dnl # This change was made to the 4.19.38 and 4.14.120 LTS kernels. @@ -107,6 +110,36 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FPU], [ struct fxregs_state *fxr __attribute__ ((unused)) = &st->fxsave; struct xregs_state *xr __attribute__ ((unused)) = &st->xsave; ]) + + ZFS_LINUX_TEST_SRC([fpu_xsave_internal], [ + #include + #if defined(__x86_64) || defined(__x86_64__) || \ + defined(__i386) || defined(__i386__) + #if !defined(__x86) + #define __x86 + #endif + #endif + + #if !defined(__x86) + #error Unsupported architecture + #endif + + #include + #ifdef HAVE_KERNEL_FPU_API_HEADER + #include + #include + #else + #include + #include + #endif + + ],[ + struct fpu *fpu = ¤t->thread.fpu; + union fpregs_state *st = &fpu->fpstate->regs; + struct fregs_state *fr __attribute__ ((unused)) = &st->fsave; + struct fxregs_state *fxr __attribute__ ((unused)) = &st->fxsave; + struct xregs_state *xr __attribute__ ((unused)) = &st->xsave; + ]) ]) AC_DEFUN([ZFS_AC_KERNEL_FPU], [ @@ -139,7 +172,13 @@ AC_DEFUN([ZFS_AC_KERNEL_FPU], [ AC_DEFINE(HAVE_KERNEL_FPU_INTERNAL, 1, [kernel fpu internal]) ],[ + ZFS_LINUX_TEST_RESULT([fpu_xsave_internal], [ + AC_MSG_RESULT(internal with internal XSAVE) + AC_DEFINE(HAVE_KERNEL_FPU_XSAVE_INTERNAL, 1, + [kernel fpu and XSAVE internal]) + ],[ AC_MSG_RESULT(unavailable) + ]) ]) ]) ]) diff --git a/config/toolchain-simd.m4 b/config/toolchain-simd.m4 index 1153cd6941..061576fd94 100644 --- a/config/toolchain-simd.m4 +++ b/config/toolchain-simd.m4 @@ -24,6 +24,9 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [ ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AES ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE + ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE + ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT + ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES ;; esac ]) @@ -422,3 +425,66 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE], [ AC_MSG_RESULT([no]) ]) ]) + +dnl # +dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE +dnl # +AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE], [ + AC_MSG_CHECKING([whether host toolchain supports XSAVE]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([ + [ + void main() + { + char b[4096] __attribute__ ((aligned (64))); + __asm__ __volatile__("xsave %[b]\n" : : [b] "m" (*b) : "memory"); + } + ]])], [ + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_XSAVE], 1, [Define if host toolchain supports XSAVE]) + ], [ + AC_MSG_RESULT([no]) + ]) +]) + +dnl # +dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT +dnl # +AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT], [ + AC_MSG_CHECKING([whether host toolchain supports XSAVEOPT]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([ + [ + void main() + { + char b[4096] __attribute__ ((aligned (64))); + __asm__ __volatile__("xsaveopt %[b]\n" : : [b] "m" (*b) : "memory"); + } + ]])], [ + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_XSAVEOPT], 1, [Define if host toolchain supports XSAVEOPT]) + ], [ + AC_MSG_RESULT([no]) + ]) +]) + +dnl # +dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES +dnl # +AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES], [ + AC_MSG_CHECKING([whether host toolchain supports XSAVES]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([ + [ + void main() + { + char b[4096] __attribute__ ((aligned (64))); + __asm__ __volatile__("xsaves %[b]\n" : : [b] "m" (*b) : "memory"); + } + ]])], [ + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_XSAVES], 1, [Define if host toolchain supports XSAVES]) + ], [ + AC_MSG_RESULT([no]) + ]) +]) diff --git a/configure.ac b/configure.ac index ebc7b276a6..2671434afc 100644 --- a/configure.ac +++ b/configure.ac @@ -383,6 +383,7 @@ AC_CONFIG_FILES([ tests/zfs-tests/tests/functional/rootpool/Makefile tests/zfs-tests/tests/functional/rsend/Makefile tests/zfs-tests/tests/functional/scrub_mirror/Makefile + tests/zfs-tests/tests/functional/simd/Makefile tests/zfs-tests/tests/functional/slog/Makefile tests/zfs-tests/tests/functional/snapshot/Makefile tests/zfs-tests/tests/functional/snapused/Makefile diff --git a/include/os/linux/kernel/linux/simd_x86.h b/include/os/linux/kernel/linux/simd_x86.h index f2ae0fcbc2..7c806bf2c6 100644 --- a/include/os/linux/kernel/linux/simd_x86.h +++ b/include/os/linux/kernel/linux/simd_x86.h @@ -136,9 +136,27 @@ * When the kernel_fpu_* symbols are unavailable then provide our own * versions which allow the FPU to be safely used. */ +#if defined(HAVE_KERNEL_FPU_INTERNAL) || defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) + +#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) +/* + * Some sanity checks. + * HAVE_KERNEL_FPU_INTERNAL and HAVE_KERNEL_FPU_XSAVE_INTERNAL are exclusive. + */ #if defined(HAVE_KERNEL_FPU_INTERNAL) +#error "HAVE_KERNEL_FPU_INTERNAL and HAVE_KERNEL_FPU_XSAVE_INTERNAL defined" +#endif +/* + * For kernels >= 5.16 we have to use inline assembly with the XSAVE{,OPT,S} + * instructions, so we need the toolchain to support at least XSAVE. + */ +#if !defined(HAVE_XSAVE) +#error "Toolchain needs to support the XSAVE assembler instruction" +#endif +#endif #include +#include extern union fpregs_state **zfs_kfpu_fpregs; @@ -191,7 +209,9 @@ kfpu_init(void) } #define kfpu_allowed() 1 +#if defined(HAVE_KERNEL_FPU_INTERNAL) #define ex_handler_fprestore ex_handler_default +#endif /* * FPU save and restore instructions. @@ -206,6 +226,7 @@ kfpu_init(void) #define kfpu_fxsr_clean(rval) __asm("fnclex; emms; fildl %P[addr]" \ : : [addr] "m" (rval)); +#if defined(HAVE_KERNEL_FPU_INTERNAL) static inline void kfpu_save_xsave(struct xregs_state *addr, uint64_t mask) { @@ -217,6 +238,21 @@ kfpu_save_xsave(struct xregs_state *addr, uint64_t mask) XSTATE_XSAVE(addr, low, hi, err); WARN_ON_ONCE(err); } +#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */ + +#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) +#define kfpu_do_xsave(instruction, addr, mask) \ +{ \ + uint32_t low, hi; \ + \ + low = mask; \ + hi = (uint64_t)(mask) >> 32; \ + __asm(instruction " %[dst]\n\t" \ + : \ + : [dst] "m" (*(addr)), "a" (low), "d" (hi) \ + : "memory"); \ +} +#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */ static inline void kfpu_save_fxsr(struct fxregs_state *addr) @@ -233,6 +269,7 @@ kfpu_save_fsave(struct fregs_state *addr) kfpu_fnsave(addr); } +#if defined(HAVE_KERNEL_FPU_INTERNAL) static inline void kfpu_begin(void) { @@ -250,7 +287,6 @@ kfpu_begin(void) * FPU state to be correctly preserved and restored. */ union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()]; - if (static_cpu_has(X86_FEATURE_XSAVE)) { kfpu_save_xsave(&state->xsave, ~0); } else if (static_cpu_has(X86_FEATURE_FXSR)) { @@ -259,7 +295,50 @@ kfpu_begin(void) kfpu_save_fsave(&state->fsave); } } +#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */ +#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) +static inline void +kfpu_begin(void) +{ + /* + * Preemption and interrupts must be disabled for the critical + * region where the FPU state is being modified. + */ + preempt_disable(); + local_irq_disable(); + + /* + * The current FPU registers need to be preserved by kfpu_begin() + * and restored by kfpu_end(). They are stored in a dedicated + * per-cpu variable, not in the task struct, this allows any user + * FPU state to be correctly preserved and restored. + */ + union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()]; +#if defined(HAVE_XSAVES) + if (static_cpu_has(X86_FEATURE_XSAVES)) { + kfpu_do_xsave("xsaves", &state->xsave, ~0); + goto out; + } +#endif +#if defined(HAVE_XSAVEOPT) + if (static_cpu_has(X86_FEATURE_XSAVEOPT)) { + kfpu_do_xsave("xsaveopt", &state->xsave, ~0); + goto out; + } +#endif + if (static_cpu_has(X86_FEATURE_XSAVE)) { + kfpu_do_xsave("xsave", &state->xsave, ~0); + } else if (static_cpu_has(X86_FEATURE_FXSR)) { + kfpu_save_fxsr(&state->fxsave); + } else { + kfpu_save_fsave(&state->fsave); + } +out: +} +#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */ + +#if defined(HAVE_KERNEL_FPU_INTERNAL) static inline void kfpu_restore_xsave(struct xregs_state *addr, uint64_t mask) { @@ -269,6 +348,21 @@ kfpu_restore_xsave(struct xregs_state *addr, uint64_t mask) hi = mask >> 32; XSTATE_XRESTORE(addr, low, hi); } +#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */ + +#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) +#define kfpu_do_xrstor(instruction, addr, mask) \ +{ \ + uint32_t low, hi; \ + \ + low = mask; \ + hi = (uint64_t)(mask) >> 32; \ + __asm(instruction " %[src]" \ + : \ + : [src] "m" (*(addr)), "a" (low), "d" (hi) \ + : "memory"); \ +} +#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */ static inline void kfpu_restore_fxsr(struct fxregs_state *addr) @@ -294,6 +388,7 @@ kfpu_restore_fsave(struct fregs_state *addr) kfpu_frstor(addr); } +#if defined(HAVE_KERNEL_FPU_INTERNAL) static inline void kfpu_end(void) { @@ -310,6 +405,32 @@ kfpu_end(void) local_irq_enable(); preempt_enable(); } +#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */ + +#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) +static inline void +kfpu_end(void) +{ + union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()]; +#if defined(HAVE_XSAVES) + if (static_cpu_has(X86_FEATURE_XSAVES)) { + kfpu_do_xrstor("xrstors", &state->xsave, ~0); + goto out; + } +#endif + if (static_cpu_has(X86_FEATURE_XSAVE)) { + kfpu_do_xrstor("xrstor", &state->xsave, ~0); + } else if (static_cpu_has(X86_FEATURE_FXSR)) { + kfpu_save_fxsr(&state->fxsave); + } else { + kfpu_save_fsave(&state->fsave); + } +out: + local_irq_enable(); + preempt_enable(); + +} +#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */ #else @@ -322,7 +443,7 @@ kfpu_end(void) #define kfpu_init() 0 #define kfpu_fini() ((void) 0) -#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */ +#endif /* defined(HAVE_KERNEL_FPU_INTERNAL || HAVE_KERNEL_FPU_XSAVE_INTERNAL) */ #endif /* defined(KERNEL_EXPORTS_X86_FPU) */ /* diff --git a/module/zcommon/zfs_prop.c b/module/zcommon/zfs_prop.c index d173219908..5f88bd0208 100644 --- a/module/zcommon/zfs_prop.c +++ b/module/zcommon/zfs_prop.c @@ -996,10 +996,10 @@ zfs_prop_align_right(zfs_prop_t prop) #include -#if defined(HAVE_KERNEL_FPU_INTERNAL) +#if defined(HAVE_KERNEL_FPU_INTERNAL) || defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) union fpregs_state **zfs_kfpu_fpregs; EXPORT_SYMBOL(zfs_kfpu_fpregs); -#endif /* HAVE_KERNEL_FPU_INTERNAL */ +#endif /* HAVE_KERNEL_FPU_INTERNAL || HAVE_KERNEL_FPU_XSAVE_INTERNAL */ static int __init zcommon_init(void) diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index b7831c3acf..5a9fbe9946 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -151,6 +151,12 @@ tags = ['functional', 'projectquota'] tests = ['send_realloc_dnode_size', 'send_encrypted_files'] tags = ['functional', 'rsend'] +[tests/functional/simd:Linux] +pre = +post = +tests = ['simd_supported'] +tags = ['functional', 'simd'] + [tests/functional/snapshot:Linux] tests = ['snapshot_015_pos', 'snapshot_016_pos'] tags = ['functional', 'snapshot'] diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am index 137cddd5f7..fd586ecee5 100644 --- a/tests/zfs-tests/tests/functional/Makefile.am +++ b/tests/zfs-tests/tests/functional/Makefile.am @@ -88,5 +88,6 @@ SUBDIRS = \ if BUILD_LINUX SUBDIRS += \ + simd \ tmpfile endif diff --git a/tests/zfs-tests/tests/functional/simd/Makefile.am b/tests/zfs-tests/tests/functional/simd/Makefile.am new file mode 100644 index 0000000000..bfc2886802 --- /dev/null +++ b/tests/zfs-tests/tests/functional/simd/Makefile.am @@ -0,0 +1,2 @@ +pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/simd +dist_pkgdata_SCRIPTS = simd_supported.ksh diff --git a/tests/zfs-tests/tests/functional/simd/simd_supported.ksh b/tests/zfs-tests/tests/functional/simd/simd_supported.ksh new file mode 100755 index 0000000000..d88bc582bf --- /dev/null +++ b/tests/zfs-tests/tests/functional/simd/simd_supported.ksh @@ -0,0 +1,58 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2022 by Attila Fülöp +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# Make sure we have SIMD support, so it will not go away without notice +# +# STRATEGY: +# 1. Test if we are running on a Linux x86 system with SSE support +# 2. If so, check if the zfs_fletcher_4_impl module parameter contains +# a sse implementation +# 3. If not fail the test, otherwise pass it + +log_note "Testing if we support SIMD instructions (Linux x86 only)" + +if !is_linux; then + log_unsupported "Not a Linux System" +fi + +case "$(uname -m)" in +i386|i686|x86_64) + typeset -R modparam="/sys/module/zcommon/parameters/zfs_fletcher_4_impl" + if cat /proc/cpuinfo | awk '/^flags/ {print; exit;}' | grep -q sse; then + log_must grep -q sse "$modparam" + log_pass "SIMD instructions supported" + else + log_unsupported "No FPU present" + fi + ;; +*) + log_unsupported "Not a x86 CPU" + ;; +esac