Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity

Implementation of AVX2 optimized Fletcher-4 

New functionality:
- Preserves existing scalar implementation.
- Adds AVX2 optimized Fletcher-4 computation.
- Fastest routines selected on module load (benchmark).
- Test case for Fletcher-4 added to ztest.

New zcommon module parameters:
-  zfs_fletcher_4_impl (str): selects the implementation to use.
    "fastest" - use the fastest version available
    "cycle"   - cycle trough all available impl for ztest
    "scalar"  - use the original version
    "avx2"    - new AVX2 implementation if available

Performance comparison (Intel i7 CPU, 1MB data buffers):
- Scalar:  4216 MB/s
- AVX2:   14499 MB/s

See contents of `/sys/module/zcommon/parameters/zfs_fletcher_4_impl`
to get list of supported values. If an implementation is not supported
on the system, it will not be shown. Currently selected option is
enclosed in `[]`.

Signed-off-by: Jinshan Xiong <jinshan.xiong@intel.com>
Signed-off-by: Andreas Dilger <andreas.dilger@intel.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #4330
This commit is contained in:
Jinshan Xiong 2015-12-09 15:34:16 -08:00 committed by Brian Behlendorf
parent 8fbbc6b4cf
commit 1eeb4562a7
12 changed files with 589 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
cmd/ztest/ztest.c
View File

@ -123,6 +123,7 @@
#include <ctype.h>
#include <math.h>
#include <sys/fs/zfs.h>
#include <zfs_fletcher.h>
#include <libnvpair.h>
#ifdef __GLIBC__
#include <execinfo.h> /* for backtrace() */
@ -327,6 +328,7 @@ ztest_func_t ztest_vdev_aux_add_remove;
ztest_func_t ztest_split_pool;
ztest_func_t ztest_reguid;
ztest_func_t ztest_spa_upgrade;
ztest_func_t ztest_fletcher;
uint64_t zopt_always = 0ULL * NANOSEC; /* all the time */
uint64_t zopt_incessant = 1ULL * NANOSEC / 10; /* every 1/10 second */
@ -372,6 +374,7 @@ ztest_info_t ztest_info[] = {
ZTI_INIT(ztest_vdev_LUN_growth, 1, &zopt_rarely),
ZTI_INIT(ztest_vdev_add_remove, 1, &ztest_opts.zo_vdevtime),
ZTI_INIT(ztest_vdev_aux_add_remove, 1, &ztest_opts.zo_vdevtime),
ZTI_INIT(ztest_fletcher, 1, &zopt_rarely),
};
#define ZTEST_FUNCS (sizeof (ztest_info) / sizeof (ztest_info_t))
@ -5496,6 +5499,47 @@ ztest_spa_rename(ztest_ds_t *zd, uint64_t id)
(void) rw_unlock(&ztest_name_lock);
}
void
ztest_fletcher(ztest_ds_t *zd, uint64_t id)
{
hrtime_t end = gethrtime() + NANOSEC;
while (gethrtime() <= end) {
int run_count = 100;
void *buf;
uint32_t size;
int *ptr;
int i;
zio_cksum_t zc_ref;
zio_cksum_t zc_ref_byteswap;
size = ztest_random_blocksize();
buf = umem_alloc(size, UMEM_NOFAIL);
for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
*ptr = ztest_random(UINT_MAX);
VERIFY0(fletcher_4_impl_set("scalar"));
fletcher_4_native(buf, size, &zc_ref);
fletcher_4_byteswap(buf, size, &zc_ref_byteswap);
VERIFY0(fletcher_4_impl_set("cycle"));
while (run_count-- > 0) {
zio_cksum_t zc;
zio_cksum_t zc_byteswap;
fletcher_4_byteswap(buf, size, &zc_byteswap);
fletcher_4_native(buf, size, &zc);
VERIFY0(bcmp(&zc, &zc_ref, sizeof (zc)));
VERIFY0(bcmp(&zc_byteswap, &zc_ref_byteswap,
sizeof (zc_byteswap)));
}
umem_free(buf, size);
}
}
static int
ztest_check_path(char *path)
{

1
include/sys/Makefile.am
View File

@ -50,6 +50,7 @@ COMMON_H = \
$(top_srcdir)/include/sys/space_reftree.h \
$(top_srcdir)/include/sys/spa.h \
$(top_srcdir)/include/sys/spa_impl.h \
$(top_srcdir)/include/sys/spa_checksum.h \
$(top_srcdir)/include/sys/trace.h \
$(top_srcdir)/include/sys/trace_acl.h \
$(top_srcdir)/include/sys/trace_arc.h \

33
include/sys/spa.h
View File

@ -35,6 +35,7 @@
#include <sys/sysmacros.h>
#include <sys/types.h>
#include <sys/fs/zfs.h>
#include <sys/spa_checksum.h>
#ifdef __cplusplus
extern "C" {
@ -142,12 +143,6 @@ typedef struct dva {
uint64_t dva_word[2];
} dva_t;
/*
* Each block has a 256-bit checksum -- strong enough for cryptographic hashes.
*/
typedef struct zio_cksum {
uint64_t zc_word[4];
} zio_cksum_t;
/*
* Each block is described by its DVAs, time of birth, checksum, etc.
@ -440,35 +435,9 @@ _NOTE(CONSTCOND) } while (0)
DVA_EQUAL(&(bp1)->blk_dva[1], &(bp2)->blk_dva[1]) && \
DVA_EQUAL(&(bp1)->blk_dva[2], &(bp2)->blk_dva[2]))
#define ZIO_CHECKSUM_EQUAL(zc1, zc2) \
(0 == (((zc1).zc_word[0] - (zc2).zc_word[0]) | \
((zc1).zc_word[1] - (zc2).zc_word[1]) | \
((zc1).zc_word[2] - (zc2).zc_word[2]) | \
((zc1).zc_word[3] - (zc2).zc_word[3])))
#define ZIO_CHECKSUM_IS_ZERO(zc) \
(0 == ((zc)->zc_word[0] | (zc)->zc_word[1] | \
(zc)->zc_word[2] | (zc)->zc_word[3]))
#define ZIO_CHECKSUM_BSWAP(zcp) \
{ \
(zcp)->zc_word[0] = BSWAP_64((zcp)->zc_word[0]); \
(zcp)->zc_word[1] = BSWAP_64((zcp)->zc_word[1]); \
(zcp)->zc_word[2] = BSWAP_64((zcp)->zc_word[2]); \
(zcp)->zc_word[3] = BSWAP_64((zcp)->zc_word[3]); \
}
#define DVA_IS_VALID(dva) (DVA_GET_ASIZE(dva) != 0)
#define ZIO_SET_CHECKSUM(zcp, w0, w1, w2, w3) \
{ \
(zcp)->zc_word[0] = w0; \
(zcp)->zc_word[1] = w1; \
(zcp)->zc_word[2] = w2; \
(zcp)->zc_word[3] = w3; \
}
#define BP_IDENTITY(bp) (ASSERT(!BP_IS_EMBEDDED(bp)), &(bp)->blk_dva[0])
#define BP_IS_GANG(bp) \
(BP_IS_EMBEDDED(bp) ? B_FALSE : DVA_GET_GANG(BP_IDENTITY(bp)))

72
include/sys/spa_checksum.h Normal file
View File

@ -0,0 +1,72 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _SPA_CHECKSUM_H
#define _SPA_CHECKSUM_H
#include <sys/types.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Each block has a 256-bit checksum -- strong enough for cryptographic hashes.
*/
typedef struct zio_cksum {
uint64_t zc_word[4];
} zio_cksum_t;
#define ZIO_SET_CHECKSUM(zcp, w0, w1, w2, w3) \
{ \
(zcp)->zc_word[0] = w0; \
(zcp)->zc_word[1] = w1; \
(zcp)->zc_word[2] = w2; \
(zcp)->zc_word[3] = w3; \
}
#define ZIO_CHECKSUM_EQUAL(zc1, zc2) \
(0 == (((zc1).zc_word[0] - (zc2).zc_word[0]) | \
((zc1).zc_word[1] - (zc2).zc_word[1]) | \
((zc1).zc_word[2] - (zc2).zc_word[2]) | \
((zc1).zc_word[3] - (zc2).zc_word[3])))
#define ZIO_CHECKSUM_IS_ZERO(zc) \
(0 == ((zc)->zc_word[0] | (zc)->zc_word[1] | \
(zc)->zc_word[2] | (zc)->zc_word[3]))
#define ZIO_CHECKSUM_BSWAP(zcp) \
{ \
(zcp)->zc_word[0] = BSWAP_64((zcp)->zc_word[0]); \
(zcp)->zc_word[1] = BSWAP_64((zcp)->zc_word[1]); \
(zcp)->zc_word[2] = BSWAP_64((zcp)->zc_word[2]); \
(zcp)->zc_word[3] = BSWAP_64((zcp)->zc_word[3]); \
}
#ifdef __cplusplus
}
#endif
#endif

21
include/zfs_fletcher.h
View File

@ -27,7 +27,7 @@
#define _ZFS_FLETCHER_H
#include <sys/types.h>
#include <sys/spa.h>
#include <sys/spa_checksum.h>
#ifdef __cplusplus
extern "C" {
@ -45,6 +45,25 @@ void fletcher_4_incremental_native(const void *, uint64_t,
zio_cksum_t *);
void fletcher_4_incremental_byteswap(const void *, uint64_t,
zio_cksum_t *);
int fletcher_4_impl_set(const char *selector);
void fletcher_4_init(void);
void fletcher_4_fini(void);
/*
* fletcher checksum struct
*/
typedef struct fletcher_4_func {
void (*init)(zio_cksum_t *);
void (*fini)(zio_cksum_t *);
void (*compute)(const void *, uint64_t, zio_cksum_t *);
void (*compute_byteswap)(const void *, uint64_t, zio_cksum_t *);
boolean_t (*valid)(void);
const char *name;
} fletcher_4_ops_t;
#if defined(HAVE_AVX) && defined(HAVE_AVX2)
extern const fletcher_4_ops_t fletcher_4_avx2_ops;
#endif
#ifdef __cplusplus
}

1
lib/libzpool/Makefile.am
View File

@ -22,6 +22,7 @@ KERNEL_C = \
zfs_comutil.c \
zfs_deleg.c \
zfs_fletcher.c \
zfs_fletcher_intel.c \
zfs_namecheck.c \
zfs_prop.c \
zfs_uio.c \

4
lib/libzpool/kernel.c
View File

@ -40,6 +40,7 @@
#include <sys/utsname.h>
#include <sys/time.h>
#include <sys/systeminfo.h>
#include <zfs_fletcher.h>
/*
* Emulation of kernel services in userland.
@ -1236,12 +1237,15 @@ kernel_init(int mode)
spa_init(mode);
fletcher_4_init();
tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
}
void
kernel_fini(void)
{
fletcher_4_fini();
spa_fini();
system_taskq_fini();

17
man/man5/zfs-module-parameters.5
View File

@ -830,6 +830,23 @@ Start syncing out a transaction group if there is at least this much dirty data.
Default value: \fB67,108,864\fR.
.RE
.sp
.ne 2
.na
\fBzfs_fletcher_4_impl\fR (string)
.ad
.RS 12n
Select a fletcher 4 implementation.
.sp
Supported selectors are: \fBfastest\fR, \fBscalar\fR, and \fBavx2\fR when
AVX2 is supported by the processor. If multiple implementations of fletcher 4
are available the \fBfastest\fR will be chosen using a micro benchmark.
Selecting \fBscalar\fR results in the original CPU based calculation being
used, \fBavx2\fR uses the AVX2 vector instructions to compute a fletcher 4.
.sp
Default value: \fBfastest\fR.
.RE
.sp
.ne 2
.na

2
module/zcommon/Makefile.in
View File

@ -15,3 +15,5 @@ $(MODULE)-objs += zfs_comutil.o
$(MODULE)-objs += zfs_fletcher.o
$(MODULE)-objs += zfs_uio.o
$(MODULE)-objs += zpool_prop.o
$(MODULE)-$(CONFIG_X86) += zfs_fletcher_intel.o

313
module/zcommon/zfs_fletcher.c
View File

@ -128,8 +128,60 @@
#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/byteorder.h>
#include <sys/zio.h>
#include <sys/spa.h>
#include <sys/zfs_context.h>
#include <zfs_fletcher.h>
static void fletcher_4_scalar_init(zio_cksum_t *zcp);
static void fletcher_4_scalar(const void *buf, uint64_t size,
zio_cksum_t *zcp);
static void fletcher_4_scalar_byteswap(const void *buf, uint64_t size,
zio_cksum_t *zcp);
static boolean_t fletcher_4_scalar_valid(void);
static const fletcher_4_ops_t fletcher_4_scalar_ops = {
.init = fletcher_4_scalar_init,
.compute = fletcher_4_scalar,
.compute_byteswap = fletcher_4_scalar_byteswap,
.valid = fletcher_4_scalar_valid,
.name = "scalar"
};
static const fletcher_4_ops_t *fletcher_4_algos[] = {
&fletcher_4_scalar_ops,
#if defined(HAVE_AVX) && defined(HAVE_AVX2)
&fletcher_4_avx2_ops,
#endif
};
static enum fletcher_selector {
FLETCHER_FASTEST = 0,
FLETCHER_SCALAR,
#if defined(HAVE_AVX) && defined(HAVE_AVX2)
FLETCHER_AVX2,
#endif
FLETCHER_CYCLE
} fletcher_4_impl_chosen = FLETCHER_SCALAR;
static struct fletcher_4_impl_selector {
const char *fis_name;
const fletcher_4_ops_t *fis_ops;
} fletcher_4_impl_selectors[] = {
[ FLETCHER_FASTEST ] = { "fastest", NULL },
[ FLETCHER_SCALAR ] = { "scalar", &fletcher_4_scalar_ops },
#if defined(HAVE_AVX) && defined(HAVE_AVX2)
[ FLETCHER_AVX2 ] = { "avx2", &fletcher_4_avx2_ops },
#endif
#if !defined(_KERNEL)
[ FLETCHER_CYCLE ] = { "cycle", &fletcher_4_scalar_ops }
#endif
};
static kmutex_t fletcher_4_impl_lock;
static kstat_t *fletcher_4_kstat;
static kstat_named_t fletcher_4_kstat_data[ARRAY_SIZE(fletcher_4_algos)];
void
fletcher_2_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
@ -165,14 +217,24 @@ fletcher_2_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
}
void
fletcher_4_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
static void fletcher_4_scalar_init(zio_cksum_t *zcp)
{
ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
}
static void
fletcher_4_scalar(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
const uint32_t *ip = buf;
const uint32_t *ipend = ip + (size / sizeof (uint32_t));
uint64_t a, b, c, d;
for (a = b = c = d = 0; ip < ipend; ip++) {
a = zcp->zc_word[0];
b = zcp->zc_word[1];
c = zcp->zc_word[2];
d = zcp->zc_word[3];
for (; ip < ipend; ip++) {
a += ip[0];
b += a;
c += b;
@ -182,14 +244,19 @@ fletcher_4_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
ZIO_SET_CHECKSUM(zcp, a, b, c, d);
}
void
fletcher_4_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
static void
fletcher_4_scalar_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
const uint32_t *ip = buf;
const uint32_t *ipend = ip + (size / sizeof (uint32_t));
uint64_t a, b, c, d;
for (a = b = c = d = 0; ip < ipend; ip++) {
a = zcp->zc_word[0];
b = zcp->zc_word[1];
c = zcp->zc_word[2];
d = zcp->zc_word[3];
for (; ip < ipend; ip++) {
a += BSWAP_32(ip[0]);
b += a;
c += b;
@ -199,53 +266,225 @@ fletcher_4_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
ZIO_SET_CHECKSUM(zcp, a, b, c, d);
}
static boolean_t
fletcher_4_scalar_valid(void)
{
return (B_TRUE);
}
int
fletcher_4_impl_set(const char *val)
{
const fletcher_4_ops_t *ops;
enum fletcher_selector idx;
size_t val_len;
unsigned i;
val_len = strlen(val);
while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */
val_len--;
for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) {
const char *name = fletcher_4_impl_selectors[i].fis_name;
if (val_len == strlen(name) &&
strncmp(val, name, val_len) == 0) {
idx = i;
break;
}
}
if (i >= ARRAY_SIZE(fletcher_4_impl_selectors))
return (-EINVAL);
ops = fletcher_4_impl_selectors[idx].fis_ops;
if (ops == NULL || !ops->valid())
return (-ENOTSUP);
mutex_enter(&fletcher_4_impl_lock);
if (fletcher_4_impl_chosen != idx)
fletcher_4_impl_chosen = idx;
mutex_exit(&fletcher_4_impl_lock);
return (0);
}
static inline const fletcher_4_ops_t *
fletcher_4_impl_get(void)
{
#if !defined(_KERNEL)
if (fletcher_4_impl_chosen == FLETCHER_CYCLE) {
static volatile unsigned int cycle_count = 0;
const fletcher_4_ops_t *ops = NULL;
unsigned int index;
while (1) {
index = atomic_inc_uint_nv(&cycle_count);
ops = fletcher_4_algos[
index % ARRAY_SIZE(fletcher_4_algos)];
if (ops->valid())
break;
}
return (ops);
}
#endif
membar_producer();
return (fletcher_4_impl_selectors[fletcher_4_impl_chosen].fis_ops);
}
void
fletcher_4_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
const fletcher_4_ops_t *ops = fletcher_4_impl_get();
ops->init(zcp);
ops->compute(buf, size, zcp);
if (ops->fini != NULL)
ops->fini(zcp);
}
void
fletcher_4_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
const fletcher_4_ops_t *ops = fletcher_4_impl_get();
ops->init(zcp);
ops->compute_byteswap(buf, size, zcp);
if (ops->fini != NULL)
ops->fini(zcp);
}
void
fletcher_4_incremental_native(const void *buf, uint64_t size,
zio_cksum_t *zcp)
{
const uint32_t *ip = buf;
const uint32_t *ipend = ip + (size / sizeof (uint32_t));
uint64_t a, b, c, d;
a = zcp->zc_word[0];
b = zcp->zc_word[1];
c = zcp->zc_word[2];
d = zcp->zc_word[3];
for (; ip < ipend; ip++) {
a += ip[0];
b += a;
c += b;
d += c;
}
ZIO_SET_CHECKSUM(zcp, a, b, c, d);
fletcher_4_scalar(buf, size, zcp);
}
void
fletcher_4_incremental_byteswap(const void *buf, uint64_t size,
zio_cksum_t *zcp)
{
const uint32_t *ip = buf;
const uint32_t *ipend = ip + (size / sizeof (uint32_t));
uint64_t a, b, c, d;
fletcher_4_scalar_byteswap(buf, size, zcp);
}
a = zcp->zc_word[0];
b = zcp->zc_word[1];
c = zcp->zc_word[2];
d = zcp->zc_word[3];
void
fletcher_4_init(void)
{
const uint64_t const bench_ns = (50 * MICROSEC); /* 50ms */
unsigned long best_run_count = 0;
unsigned long best_run_index = 0;
const unsigned data_size = 4096;
char *databuf;
int i;
for (; ip < ipend; ip++) {
a += BSWAP_32(ip[0]);
b += a;
c += b;
d += c;
databuf = kmem_alloc(data_size, KM_SLEEP);
for (i = 0; i < ARRAY_SIZE(fletcher_4_algos); i++) {
const fletcher_4_ops_t *ops = fletcher_4_algos[i];
kstat_named_t *stat = &fletcher_4_kstat_data[i];
unsigned long run_count = 0;
hrtime_t start;
zio_cksum_t zc;
strncpy(stat->name, ops->name, sizeof (stat->name) - 1);
stat->data_type = KSTAT_DATA_UINT64;
stat->value.ui64 = 0;
if (!ops->valid())
continue;
kpreempt_disable();
start = gethrtime();
ops->init(&zc);
do {
ops->compute(databuf, data_size, &zc);
run_count++;
} while (gethrtime() < start + bench_ns);
if (ops->fini != NULL)
ops->fini(&zc);
kpreempt_enable();
if (run_count > best_run_count) {
best_run_count = run_count;
best_run_index = i;
}
/*
* Due to high overhead of gethrtime(), the performance data
* here is inaccurate and much slower than it could be.
* It's fine for our use though because only relative speed
* is important.
*/
stat->value.ui64 = data_size * run_count *
(NANOSEC / bench_ns) >> 20; /* by MB/s */
}
kmem_free(databuf, data_size);
ZIO_SET_CHECKSUM(zcp, a, b, c, d);
fletcher_4_impl_selectors[FLETCHER_FASTEST].fis_ops =
fletcher_4_algos[best_run_index];
mutex_init(&fletcher_4_impl_lock, NULL, MUTEX_DEFAULT, NULL);
fletcher_4_impl_set("fastest");
fletcher_4_kstat = kstat_create("zfs", 0, "fletcher_4_bench",
"misc", KSTAT_TYPE_NAMED, ARRAY_SIZE(fletcher_4_algos),
KSTAT_FLAG_VIRTUAL);
if (fletcher_4_kstat != NULL) {
fletcher_4_kstat->ks_data = fletcher_4_kstat_data;
kstat_install(fletcher_4_kstat);
}
}
void
fletcher_4_fini(void)
{
mutex_destroy(&fletcher_4_impl_lock);
if (fletcher_4_kstat != NULL) {
kstat_delete(fletcher_4_kstat);
fletcher_4_kstat = NULL;
}
}
#if defined(_KERNEL) && defined(HAVE_SPL)
static int
fletcher_4_param_get(char *buffer, struct kernel_param *unused)
{
int i, cnt = 0;
for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) {
const fletcher_4_ops_t *ops;
ops = fletcher_4_impl_selectors[i].fis_ops;
if (!ops->valid())
continue;
cnt += sprintf(buffer + cnt,
fletcher_4_impl_chosen == i ? "[%s] " : "%s ",
fletcher_4_impl_selectors[i].fis_name);
}
return (cnt);
}
static int
fletcher_4_param_set(const char *val, struct kernel_param *unused)
{
return (fletcher_4_impl_set(val));
}
/*
* Choose a fletcher 4 implementation in ZFS.
* Users can choose the "fastest" algorithm, or "scalar" and "avx2" which means
* to compute fletcher 4 by CPU or vector instructions respectively.
* Users can also choose "cycle" to exercise all implementions, but this is
* for testing purpose therefore it can only be set in user space.
*/
module_param_call(zfs_fletcher_4_impl,
fletcher_4_param_set, fletcher_4_param_get, NULL, 0644);
MODULE_PARM_DESC(zfs_fletcher_4_impl, "Select fletcher 4 algorithm");
EXPORT_SYMBOL(fletcher_4_init);
EXPORT_SYMBOL(fletcher_4_fini);
EXPORT_SYMBOL(fletcher_2_native);
EXPORT_SYMBOL(fletcher_2_byteswap);
EXPORT_SYMBOL(fletcher_4_native);

148
module/zcommon/zfs_fletcher_intel.c Normal file
View File

@ -0,0 +1,148 @@
/*
* Implement fast Fletcher4 with AVX2 instructions. (x86_64)
*
* Use the 256-bit AVX2 SIMD instructions and registers to compute
* Fletcher4 in four incremental 64-bit parallel accumulator streams,
* and then combine the streams to form the final four checksum words.
*
* Copyright (C) 2015 Intel Corporation.
*
* Authors:
* James Guilford <james.guilford@intel.com>
* Jinshan Xiong <jinshan.xiong@intel.com>
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#if defined(HAVE_AVX) && defined(HAVE_AVX2)
#include <linux/simd_x86.h>
#include <sys/spa_checksum.h>
#include <zfs_fletcher.h>
static void
fletcher_4_avx2_init(zio_cksum_t *zcp)
{
kfpu_begin();
/* clear avx2 registers */
asm volatile("vpxor %ymm0, %ymm0, %ymm0");
asm volatile("vpxor %ymm1, %ymm1, %ymm1");
asm volatile("vpxor %ymm2, %ymm2, %ymm2");
asm volatile("vpxor %ymm3, %ymm3, %ymm3");
}
static void
fletcher_4_avx2_fini(zio_cksum_t *zcp)
{
uint64_t __attribute__((aligned(32))) a[4];
uint64_t __attribute__((aligned(32))) b[4];
uint64_t __attribute__((aligned(32))) c[4];
uint64_t __attribute__((aligned(32))) d[4];
uint64_t A, B, C, D;
asm volatile("vmovdqu %%ymm0, %0":"=m" (a));
asm volatile("vmovdqu %%ymm1, %0":"=m" (b));
asm volatile("vmovdqu %%ymm2, %0":"=m" (c));
asm volatile("vmovdqu %%ymm3, %0":"=m" (d));
asm volatile("vzeroupper");
kfpu_end();
A = a[0] + a[1] + a[2] + a[3];
B = 0 - a[1] - 2*a[2] - 3*a[3]
+ 4*b[0] + 4*b[1] + 4*b[2] + 4*b[3];
C = a[2] + 3*a[3]
- 6*b[0] - 10*b[1] - 14*b[2] - 18*b[3]
+ 16*c[0] + 16*c[1] + 16*c[2] + 16*c[3];
D = 0 - a[3]
+ 4*b[0] + 10*b[1] + 20*b[2] + 34*b[3]
- 48*c[0] - 64*c[1] - 80*c[2] - 96*c[3]
+ 64*d[0] + 64*d[1] + 64*d[2] + 64*d[3];
ZIO_SET_CHECKSUM(zcp, A, B, C, D);
}
static void
fletcher_4_avx2(const void *buf, uint64_t size, zio_cksum_t *unused)
{
const uint64_t *ip = buf;
const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size);
for (; ip < ipend; ip += 2) {
asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip));
asm volatile("vpaddq %ymm4, %ymm0, %ymm0");
asm volatile("vpaddq %ymm0, %ymm1, %ymm1");
asm volatile("vpaddq %ymm1, %ymm2, %ymm2");
asm volatile("vpaddq %ymm2, %ymm3, %ymm3");
}
}
static void
fletcher_4_avx2_byteswap(const void *buf, uint64_t size, zio_cksum_t *unused)
{
static const struct {
uint64_t v[4] __attribute__((aligned(32)));
} mask = {
.v = { 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B,
0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B }
};
const uint64_t *ip = buf;
const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size);
asm volatile("vmovdqa %0, %%ymm5"::"m"(mask));
for (; ip < ipend; ip += 2) {
asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip));
asm volatile("vpshufb %ymm5, %ymm4, %ymm4");
asm volatile("vpaddq %ymm4, %ymm0, %ymm0");
asm volatile("vpaddq %ymm0, %ymm1, %ymm1");
asm volatile("vpaddq %ymm1, %ymm2, %ymm2");
asm volatile("vpaddq %ymm2, %ymm3, %ymm3");
}
}
static boolean_t fletcher_4_avx2_valid(void)
{
return (zfs_avx_available() && zfs_avx2_available());
}
const fletcher_4_ops_t fletcher_4_avx2_ops = {
.init = fletcher_4_avx2_init,
.fini = fletcher_4_avx2_fini,
.compute = fletcher_4_avx2,
.compute_byteswap = fletcher_4_avx2_byteswap,
.valid = fletcher_4_avx2_valid,
.name = "avx2"
};
#endif /* defined(HAVE_AVX) && defined(HAVE_AVX2) */

3
module/zcommon/zfs_prop.c
View File

@ -35,6 +35,7 @@
#include "zfs_prop.h"
#include "zfs_deleg.h"
#include "zfs_fletcher.h"
#if defined(_KERNEL)
#include <sys/systm.h>
@ -695,12 +696,14 @@ zfs_prop_align_right(zfs_prop_t prop)
static int __init
zcommon_init(void)
{
fletcher_4_init();
return (0);
}
static void __exit
zcommon_fini(void)
{
fletcher_4_fini();
}
module_init(zcommon_init);