Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity

Code improvement and bug fixes for QAT support 

1. Support QAT when ZFS is root file-system:
   When ZFS module is loaded before QAT started, the QAT can
   be started again in post-process, e.g.:
   echo 0 > /sys/module/zfs/parameters/zfs_qat_compress_disable
   echo 0 > /sys/module/zfs/parameters/zfs_qat_encrypt_disable
   echo 0 > /sys/module/zfs/parameters/zfs_qat_checksum_disable
2. Verify alder checksum of the de-compress result
3. Allocate Digest, IV and AAD buffer in physical contiguous
   memory by QAT_PHYS_CONTIG_ALLOC.
4. Update the documentation for zfs_qat_compress_disable,
   zfs_qat_checksum_disable, zfs_qat_encrypt_disable.

Reviewed-by: Tom Caputi <tcaputi@datto.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Weigang Li <weigang.li@intel.com>
Signed-off-by: Chengfeix Zhu <chengfeix.zhu@intel.com>
Closes #8323 
Closes #8610
This commit is contained in:
cfzhu 2019-04-17 03:38:36 +08:00 committed by Brian Behlendorf
parent 59f6594cf6
commit 5090f72743
5 changed files with 182 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
man/man5/zfs-module-parameters.5
View File

@ -2042,6 +2042,45 @@ can't hurt performance.
Use \fB1\fR for yes and \fB0\fR for no (default).
.RE
.sp
.ne 2
.na
\fBzfs_qat_checksum_disable\fR (int)
.ad
.RS 12n
This tunable disables qat hardware acceleration for sha256 checksums. It
may be set after the zfs modules have been loaded to initialize the qat
hardware as long as support is compiled in and the qat driver is present.
.sp
Use \fB1\fR for yes and \fB0\fR for no (default).
.RE
.sp
.ne 2
.na
\fBzfs_qat_compress_disable\fR (int)
.ad
.RS 12n
This tunable disables qat hardware acceleration for gzip compression. It
may be set after the zfs modules have been loaded to initialize the qat
hardware as long as support is compiled in and the qat driver is present.
.sp
Use \fB1\fR for yes and \fB0\fR for no (default).
.RE
.sp
.ne 2
.na
\fBzfs_qat_encrypt_disable\fR (int)
.ad
.RS 12n
This tunable disables qat hardware acceleration for AES-GCM encryption. It
may be set after the zfs modules have been loaded to initialize the qat
hardware as long as support is compiled in and the qat driver is present.
.sp
Use \fB1\fR for yes and \fB0\fR for no (default).
.RE
.sp
.ne 2
.na
@ -2961,19 +3000,6 @@ Valid values are \fB1\fR (full), \fB2\fR (dev) and \fB3\fR (none).
Default value: \fB1\fR.
.RE
.sp
.ne 2
.na
\fBzfs_qat_disable\fR (int)
.ad
.RS 12n
This tunable disables qat hardware acceleration for gzip compression and.
AES-GCM encryption. It is available only if qat acceleration is compiled in
and the qat driver is present.
.sp
Use \fB1\fR for yes and \fB0\fR for no (default).
.RE
.SH ZFS I/O SCHEDULER
ZFS issues I/O operations to leaf vdevs to satisfy and complete I/Os.
The I/O scheduler determines when and in what order those operations are

27
module/zfs/qat.c
View File

@ -66,18 +66,6 @@ qat_mem_free_contig(void **pp_mem_addr)
int
qat_init(void)
{
int ret;
ret = qat_dc_init();
if (ret != 0)
return (ret);
ret = qat_crypt_init();
if (ret != 0) {
qat_dc_fini();
return (ret);
}
qat_ksp = kstat_create("zfs", 0, "qat", "misc",
KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t),
KSTAT_FLAG_VIRTUAL);
@ -86,6 +74,19 @@ qat_init(void)
kstat_install(qat_ksp);
}
/*
* Just set the disable flag when qat init failed, qat can be
* turned on again in post-process after zfs module is loaded, e.g.:
* echo 0 > /sys/module/zfs/parameters/zfs_qat_compress_disable
*/
if (qat_dc_init() != 0)
zfs_qat_compress_disable = 1;
if (qat_cy_init() != 0) {
zfs_qat_checksum_disable = 1;
zfs_qat_encrypt_disable = 1;
}
return (0);
}
@ -97,7 +98,7 @@ qat_fini(void)
qat_ksp = NULL;
}
qat_crypt_fini();
qat_cy_fini();
qat_dc_fini();
}

7
module/zfs/qat.h
View File

@ -147,6 +147,9 @@ typedef struct qat_stats {
QAT_STAT_INCR(stat, 1)
extern qat_stats_t qat_stats;
extern int zfs_qat_compress_disable;
extern int zfs_qat_checksum_disable;
extern int zfs_qat_encrypt_disable;
/* inlined for performance */
static inline struct page *
@ -167,8 +170,8 @@ void qat_mem_free_contig(void **pp_mem_addr);
extern int qat_dc_init(void);
extern void qat_dc_fini(void);
extern int qat_crypt_init(void);
extern void qat_crypt_fini(void);
extern int qat_cy_init(void);
extern void qat_cy_fini(void);
extern int qat_init(void);
extern void qat_fini(void);

39
module/zfs/qat_compress.c
View File

@ -24,7 +24,9 @@
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/completion.h>
#include <linux/mod_compat.h>
#include <sys/zfs_context.h>
#include <sys/byteorder.h>
#include <sys/zio.h>
#include "qat.h"
@ -111,6 +113,9 @@ qat_dc_init(void)
Cpa32U buff_meta_size = 0;
CpaDcSessionSetupData sd = {0};
if (qat_dc_init_done)
return (0);
status = cpaDcGetNumInstances(&num_inst);
if (status != CPA_STATUS_SUCCESS)
return (-1);
@ -252,6 +257,7 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2;
Cpa32U bytes_left;
Cpa32U dst_pages = 0;
Cpa32U adler32 = 0;
char *data;
struct page *page;
struct page **in_pages = NULL;
@ -468,6 +474,12 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
goto fail;
}
/* verify adler checksum */
adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ);
if (adler32 != BSWAP_32(dc_results.checksum)) {
status = CPA_STATUS_FAIL;
goto fail;
}
*c_len = dc_results.produced;
QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
}
@ -534,7 +546,30 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
return (ret);
}
module_param(zfs_qat_compress_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_compress_disable, "Disable QAT compression");
static int
param_set_qat_compress(const char *val, struct kernel_param *kp)
{
int ret;
int *pvalue = kp->arg;
ret = param_set_int(val, kp);
if (ret)
return (ret);
/*
* zfs_qat_compress_disable = 0: enable qat compress
* try to initialize qat instance if it has not been done
*/
if (*pvalue == 0 && !qat_dc_init_done) {
ret = qat_dc_init();
if (ret != 0) {
zfs_qat_compress_disable = 1;
return (ret);
}
}
return (ret);
}
module_param_call(zfs_qat_compress_disable, param_set_qat_compress,
param_get_int, &zfs_qat_compress_disable, 0644);
MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression");
#endif

108
module/zfs/qat_crypt.c
View File

@ -31,6 +31,7 @@
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/completion.h>
#include <linux/mod_compat.h>
#include <sys/zfs_context.h>
#include <sys/zio_crypt.h>
#include "lac/cpa_cy_im.h"
@ -50,7 +51,7 @@
static Cpa32U inst_num = 0;
static Cpa16U num_inst = 0;
static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
static boolean_t qat_crypt_init_done = B_FALSE;
static boolean_t qat_cy_init_done = B_FALSE;
int zfs_qat_encrypt_disable = 0;
int zfs_qat_checksum_disable = 0;
@ -76,7 +77,7 @@ boolean_t
qat_crypt_use_accel(size_t s_len)
{
return (!zfs_qat_encrypt_disable &&
qat_crypt_init_done &&
qat_cy_init_done &&
s_len >= QAT_MIN_BUF_SIZE &&
s_len <= QAT_MAX_BUF_SIZE);
}
@ -85,26 +86,29 @@ boolean_t
qat_checksum_use_accel(size_t s_len)
{
return (!zfs_qat_checksum_disable &&
qat_crypt_init_done &&
qat_cy_init_done &&
s_len >= QAT_MIN_BUF_SIZE &&
s_len <= QAT_MAX_BUF_SIZE);
}
void
qat_crypt_clean(void)
qat_cy_clean(void)
{
for (Cpa16U i = 0; i < num_inst; i++)
cpaCyStopInstance(cy_inst_handles[i]);
num_inst = 0;
qat_crypt_init_done = B_FALSE;
qat_cy_init_done = B_FALSE;
}
int
qat_crypt_init(void)
qat_cy_init(void)
{
CpaStatus status = CPA_STATUS_FAIL;
if (qat_cy_init_done)
return (0);
status = cpaCyGetNumInstances(&num_inst);
if (status != CPA_STATUS_SUCCESS)
return (-1);
@ -131,21 +135,21 @@ qat_crypt_init(void)
goto error;
}
qat_crypt_init_done = B_TRUE;
qat_cy_init_done = B_TRUE;
return (0);
error:
qat_crypt_clean();
qat_cy_clean();
return (-1);
}
void
qat_crypt_fini(void)
qat_cy_fini(void)
{
if (!qat_crypt_init_done)
if (!qat_cy_init_done)
return;
qat_crypt_clean();
qat_cy_clean();
}
static CpaStatus
@ -351,6 +355,21 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
nr_bufs * sizeof (CpaFlatBuffer));
if (status != CPA_STATUS_SUCCESS)
goto fail;
status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult,
ZIO_DATA_MAC_LEN);
if (status != CPA_STATUS_SUCCESS)
goto fail;
status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv,
ZIO_DATA_IV_LEN);
if (status != CPA_STATUS_SUCCESS)
goto fail;
if (aad_len > 0) {
status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData,
aad_len);
if (status != CPA_STATUS_SUCCESS)
goto fail;
bcopy(aad_buf, op_data.pAdditionalAuthData, aad_len);
}
bytes_left = enc_len;
data = src_buf;
@ -389,18 +408,13 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
op_data.sessionCtx = cy_session_ctx;
op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
op_data.pIv = NULL; /* set this later as the J0 block */
op_data.ivLenInBytes = 0;
op_data.cryptoStartSrcOffsetInBytes = 0;
op_data.messageLenToCipherInBytes = 0;
op_data.hashStartSrcOffsetInBytes = 0;
op_data.messageLenToHashInBytes = 0;
op_data.pDigestResult = 0;
op_data.messageLenToCipherInBytes = enc_len;
op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
op_data.pDigestResult = digest_buf;
op_data.pAdditionalAuthData = aad_buf;
op_data.pIv = iv_buf;
bcopy(iv_buf, op_data.pIv, ZIO_DATA_IV_LEN);
cb.verify_result = CPA_FALSE;
init_completion(&cb.complete);
@ -420,6 +434,8 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
goto fail;
}
/* save digest result to digest_buf */
bcopy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
if (dir == QAT_ENCRYPT)
QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
else
@ -435,6 +451,10 @@ fail:
kunmap(out_pages[i]);
cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
if (aad_len > 0)
QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData);
QAT_PHYS_CONTIG_FREE(op_data.pIv);
QAT_PHYS_CONTIG_FREE(op_data.pDigestResult);
QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
QAT_PHYS_CONTIG_FREE(cy_session_ctx);
@ -557,10 +577,56 @@ fail:
return (status);
}
module_param(zfs_qat_encrypt_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption");
static int
param_set_qat_encrypt(const char *val, struct kernel_param *kp)
{
int ret;
int *pvalue = kp->arg;
ret = param_set_int(val, kp);
if (ret)
return (ret);
/*
* zfs_qat_encrypt_disable = 0: enable qat encrypt
* try to initialize qat instance if it has not been done
*/
if (*pvalue == 0 && !qat_cy_init_done) {
ret = qat_cy_init();
if (ret != 0) {
zfs_qat_encrypt_disable = 1;
return (ret);
}
}
return (ret);
}
module_param(zfs_qat_checksum_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming");
static int
param_set_qat_checksum(const char *val, struct kernel_param *kp)
{
int ret;
int *pvalue = kp->arg;
ret = param_set_int(val, kp);
if (ret)
return (ret);
/*
* set_checksum_param_ops = 0: enable qat checksum
* try to initialize qat instance if it has not been done
*/
if (*pvalue == 0 && !qat_cy_init_done) {
ret = qat_cy_init();
if (ret != 0) {
zfs_qat_checksum_disable = 1;
return (ret);
}
}
return (ret);
}
module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt,
param_get_int, &zfs_qat_encrypt_disable, 0644);
MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption");
module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum,
param_get_int, &zfs_qat_checksum_disable, 0644);
MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming");
#endif