zfs/module/zcommon/zfs_fletcher_intel.c

176 lines
5.5 KiB
C
Raw Normal View History

/*
* Implement fast Fletcher4 with AVX2 instructions. (x86_64)
*
* Use the 256-bit AVX2 SIMD instructions and registers to compute
* Fletcher4 in four incremental 64-bit parallel accumulator streams,
* and then combine the streams to form the final four checksum words.
*
* Copyright (C) 2015 Intel Corporation.
*
* Authors:
* James Guilford <james.guilford@intel.com>
* Jinshan Xiong <jinshan.xiong@intel.com>
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#if defined(HAVE_AVX) && defined(HAVE_AVX2)
#include <sys/spa_checksum.h>
#include <sys/simd.h>
Update build system and packaging Minimal changes required to integrate the SPL sources in to the ZFS repository build infrastructure and packaging. Build system and packaging: * Renamed SPL_* autoconf m4 macros to ZFS_*. * Removed redundant SPL_* autoconf m4 macros. * Updated the RPM spec files to remove SPL package dependency. * The zfs package obsoletes the spl package, and the zfs-kmod package obsoletes the spl-kmod package. * The zfs-kmod-devel* packages were updated to add compatibility symlinks under /usr/src/spl-x.y.z until all dependent packages can be updated. They will be removed in a future release. * Updated copy-builtin script for in-kernel builds. * Updated DKMS package to include the spl.ko. * Updated stale AUTHORS file to include all contributors. * Updated stale COPYRIGHT and included the SPL as an exception. * Renamed README.markdown to README.md * Renamed OPENSOLARIS.LICENSE to LICENSE. * Renamed DISCLAIMER to NOTICE. Required code changes: * Removed redundant HAVE_SPL macro. * Removed _BOOT from nvpairs since it doesn't apply for Linux. * Initial header cleanup (removal of empty headers, refactoring). * Remove SPL repository clone/build from zimport.sh. * Use of DEFINE_RATELIMIT_STATE and DEFINE_SPINLOCK removed due to build issues when forcing C99 compilation. * Replaced legacy ACCESS_ONCE with READ_ONCE. * Include needed headers for `current` and `EXPORT_SYMBOL`. Reviewed-by: Tony Hutter <hutter2@llnl.gov> Reviewed-by: Olaf Faaland <faaland1@llnl.gov> Reviewed-by: Matthew Ahrens <mahrens@delphix.com> Reviewed-by: Pavel Zakharov <pavel.zakharov@delphix.com> Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov> TEST_ZIMPORT_SKIP="yes" Closes #7556
2018-02-16 01:53:18 +00:00
#include <sys/strings.h>
#include <zfs_fletcher.h>
ZFS_NO_SANITIZE_UNDEFINED
static void
fletcher_4_avx2_init(fletcher_4_ctx_t *ctx)
{
memset(ctx->avx, 0, 4 * sizeof (zfs_fletcher_avx_t));
}
ZFS_NO_SANITIZE_UNDEFINED
static void
fletcher_4_avx2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
{
uint64_t A, B, C, D;
A = ctx->avx[0].v[0] + ctx->avx[0].v[1] +
ctx->avx[0].v[2] + ctx->avx[0].v[3];
B = 0 - ctx->avx[0].v[1] - 2 * ctx->avx[0].v[2] - 3 * ctx->avx[0].v[3] +
4 * ctx->avx[1].v[0] + 4 * ctx->avx[1].v[1] + 4 * ctx->avx[1].v[2] +
4 * ctx->avx[1].v[3];
C = ctx->avx[0].v[2] + 3 * ctx->avx[0].v[3] - 6 * ctx->avx[1].v[0] -
10 * ctx->avx[1].v[1] - 14 * ctx->avx[1].v[2] -
18 * ctx->avx[1].v[3] + 16 * ctx->avx[2].v[0] +
16 * ctx->avx[2].v[1] + 16 * ctx->avx[2].v[2] +
16 * ctx->avx[2].v[3];
D = 0 - ctx->avx[0].v[3] + 4 * ctx->avx[1].v[0] +
10 * ctx->avx[1].v[1] + 20 * ctx->avx[1].v[2] +
34 * ctx->avx[1].v[3] - 48 * ctx->avx[2].v[0] -
64 * ctx->avx[2].v[1] - 80 * ctx->avx[2].v[2] -
96 * ctx->avx[2].v[3] + 64 * ctx->avx[3].v[0] +
64 * ctx->avx[3].v[1] + 64 * ctx->avx[3].v[2] +
64 * ctx->avx[3].v[3];
ZIO_SET_CHECKSUM(zcp, A, B, C, D);
}
#define FLETCHER_4_AVX2_RESTORE_CTX(ctx) \
{ \
asm volatile("vmovdqu %0, %%ymm0" :: "m" ((ctx)->avx[0])); \
asm volatile("vmovdqu %0, %%ymm1" :: "m" ((ctx)->avx[1])); \
asm volatile("vmovdqu %0, %%ymm2" :: "m" ((ctx)->avx[2])); \
asm volatile("vmovdqu %0, %%ymm3" :: "m" ((ctx)->avx[3])); \
}
#define FLETCHER_4_AVX2_SAVE_CTX(ctx) \
{ \
asm volatile("vmovdqu %%ymm0, %0" : "=m" ((ctx)->avx[0])); \
asm volatile("vmovdqu %%ymm1, %0" : "=m" ((ctx)->avx[1])); \
asm volatile("vmovdqu %%ymm2, %0" : "=m" ((ctx)->avx[2])); \
asm volatile("vmovdqu %%ymm3, %0" : "=m" ((ctx)->avx[3])); \
}
static void
fletcher_4_avx2_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size)
{
const uint64_t *ip = buf;
const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size);
kfpu_begin();
FLETCHER_4_AVX2_RESTORE_CTX(ctx);
for (; ip < ipend; ip += 2) {
asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip));
asm volatile("vpaddq %ymm4, %ymm0, %ymm0");
asm volatile("vpaddq %ymm0, %ymm1, %ymm1");
asm volatile("vpaddq %ymm1, %ymm2, %ymm2");
asm volatile("vpaddq %ymm2, %ymm3, %ymm3");
}
FLETCHER_4_AVX2_SAVE_CTX(ctx);
asm volatile("vzeroupper");
kfpu_end();
}
static void
fletcher_4_avx2_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size)
{
static const zfs_fletcher_avx_t mask = {
.v = { 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B,
0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B }
};
const uint64_t *ip = buf;
const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size);
kfpu_begin();
FLETCHER_4_AVX2_RESTORE_CTX(ctx);
asm volatile("vmovdqu %0, %%ymm5" :: "m" (mask));
for (; ip < ipend; ip += 2) {
asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip));
asm volatile("vpshufb %ymm5, %ymm4, %ymm4");
asm volatile("vpaddq %ymm4, %ymm0, %ymm0");
asm volatile("vpaddq %ymm0, %ymm1, %ymm1");
asm volatile("vpaddq %ymm1, %ymm2, %ymm2");
asm volatile("vpaddq %ymm2, %ymm3, %ymm3");
}
FLETCHER_4_AVX2_SAVE_CTX(ctx);
asm volatile("vzeroupper");
kfpu_end();
}
static boolean_t fletcher_4_avx2_valid(void)
{
Linux 5.0 compat: SIMD compatibility Restore the SIMD optimization for 4.19.38 LTS, 4.14.120 LTS, and 5.0 and newer kernels. This is accomplished by leveraging the fact that by definition dedicated kernel threads never need to concern themselves with saving and restoring the user FPU state. Therefore, they may use the FPU as long as we can guarantee user tasks always restore their FPU state before context switching back to user space. For the 5.0 and 5.1 kernels disabling preemption and local interrupts is sufficient to allow the FPU to be used. All non-kernel threads will restore the preserved user FPU state. For 5.2 and latter kernels the user FPU state restoration will be skipped if the kernel determines the registers have not changed. Therefore, for these kernels we need to perform the additional step of saving and restoring the FPU registers. Invalidating the per-cpu global tracking the FPU state would force a restore but that functionality is private to the core x86 FPU implementation and unavailable. In practice, restricting SIMD to kernel threads is not a major restriction for ZFS. The vast majority of SIMD operations are already performed by the IO pipeline. The remaining cases are relatively infrequent and can be handled by the generic code without significant impact. The two most noteworthy cases are: 1) Decrypting the wrapping key for an encrypted dataset, i.e. `zfs load-key`. All other encryption and decryption operations will use the SIMD optimized implementations. 2) Generating the payload checksums for a `zfs send` stream. In order to avoid making any changes to the higher layers of ZFS all of the `*_get_ops()` functions were updated to take in to consideration the calling context. This allows for the fastest implementation to be used as appropriate (see kfpu_allowed()). The only other notable instance of SIMD operations being used outside a kernel thread was at module load time. This code was moved in to a taskq in order to accommodate the new kernel thread restriction. Finally, a few other modifications were made in order to further harden this code and facilitate testing. They include updating each implementations operations structure to be declared as a constant. And allowing "cycle" to be set when selecting the preferred ops in the kernel as well as user space. Reviewed-by: Tony Hutter <hutter2@llnl.gov> Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov> Closes #8754 Closes #8793 Closes #8965
2019-07-12 16:31:20 +00:00
return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available());
}
const fletcher_4_ops_t fletcher_4_avx2_ops = {
Rework of fletcher_4 module - Benchmark memory block is increased to 128kiB to reflect real block sizes more accurately. Measurements include all three stages needed for checksum generation, i.e. `init()/compute()/fini()`. The inner loop is repeated multiple times to offset overhead of time function. - Fastest implementation selects native and byteswap methods independently in benchmark. To support this new function pointers `init_byteswap()/fini_byteswap()` are introduced. - Implementation mutex lock is replaced by atomic variable. - To save time, benchmark is not executed in userspace. Instead, highest supported implementation is used for fastest. Default userspace selector is still 'cycle'. - `fletcher_4_native/byteswap()` methods use incremental methods to finish calculation if data size is not multiple of vector stride (currently 64B). - Added `fletcher_4_native_varsize()` special purpose method for use when buffer size is not known in advance. The method does not enforce 4B alignment on buffer size, and will ignore last (size % 4) bytes of the data buffer. - Benchmark `kstat` is changed to match the one of vdev_raidz. It now shows throughput for all supported implementations (in B/s), native and byteswap, as well as the code [fastest] is running. Example of `fletcher_4_bench` running on `Intel(R) Xeon(R) CPU E5-2660 v3 @ 2.60GHz`: implementation native byteswap scalar 4768120823 3426105750 sse2 7947841777 4318964249 ssse3 7951922722 6112191941 avx2 13269714358 11043200912 fastest avx2 avx2 Example of `fletcher_4_bench` running on `Intel(R) Xeon Phi(TM) CPU 7210 @ 1.30GHz`: implementation native byteswap scalar 1291115967 1031555336 sse2 2539571138 1280970926 ssse3 2537778746 1080016762 avx2 4950749767 1078493449 avx512f 9581379998 4010029046 fastest avx512f avx512f Signed-off-by: Gvozden Neskovic <neskovic@gmail.com> Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov> Closes #4952
2016-07-12 15:50:54 +00:00
.init_native = fletcher_4_avx2_init,
.fini_native = fletcher_4_avx2_fini,
.compute_native = fletcher_4_avx2_native,
.init_byteswap = fletcher_4_avx2_init,
.fini_byteswap = fletcher_4_avx2_fini,
.compute_byteswap = fletcher_4_avx2_byteswap,
.valid = fletcher_4_avx2_valid,
.name = "avx2"
};
#endif /* defined(HAVE_AVX) && defined(HAVE_AVX2) */