Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity

Enable SIMD for encryption 

When adding the SIMD compatibility code in e5db313 the decryption of a
dataset wrapping key was left in a user thread context.  This was done
intentionally since it's a relatively infrequent operation.  However,
this also meant that the encryption context templates were initialized
using the generic operations.  Therefore, subsequent encryption and
decryption operations would use the generic implementation even when
executed by an I/O pipeline thread.

Resolve the issue by initializing the context templates in an I/O
pipeline thread.  And by updating zio_do_crypt_uio() to dispatch any
encryption operations to a pipeline thread when called from the user
context.  For example, when performing a read from the ARC.

Tested-by: Attila Fülöp <attila@fueloep.org>
Reviewed-by: Tom Caputi <tcaputi@datto.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #9215
Closes #9296
This commit is contained in:
Brian Behlendorf 2019-09-10 10:45:46 -07:00 committed by GitHub
parent b63e2d881f
commit b88ca2acf5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 132 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
include/sys/zio_crypt.h
View File

@ -107,11 +107,11 @@ void zio_crypt_key_destroy(zio_crypt_key_t *key);
int zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key); int zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key);
int zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt_out); int zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt_out);
int zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv, int zio_crypt_key_wrap(spa_t *spa, crypto_key_t *cwkey, zio_crypt_key_t *key,
uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out); uint8_t *iv, uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out);
int zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, int zio_crypt_key_unwrap(spa_t *spa, crypto_key_t *cwkey, uint64_t crypt,
uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv, uint64_t version, uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata,
uint8_t *mac, zio_crypt_key_t *key); uint8_t *iv, uint8_t *mac, zio_crypt_key_t *key);
int zio_crypt_generate_iv(uint8_t *ivbuf); int zio_crypt_generate_iv(uint8_t *ivbuf);
int zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data, int zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data,
uint_t datalen, uint8_t *ivbuf, uint8_t *salt); uint_t datalen, uint8_t *ivbuf, uint8_t *salt);
@ -132,11 +132,11 @@ int zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen,
uint8_t *digestbuf, uint_t digestlen); uint8_t *digestbuf, uint_t digestlen);
int zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, int zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen,
boolean_t byteswap, uint8_t *portable_mac, uint8_t *local_mac); boolean_t byteswap, uint8_t *portable_mac, uint8_t *local_mac);
int zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, int zio_do_crypt_data(spa_t *spa, boolean_t encrypt, zio_crypt_key_t *key,
dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv, dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv,
uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf, uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf,
boolean_t *no_crypt); boolean_t *no_crypt);
int zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, int zio_do_crypt_abd(spa_t *spa, boolean_t encrypt, zio_crypt_key_t *key,
dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv, dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv,
uint8_t *mac, uint_t datalen, abd_t *pabd, abd_t *cabd, uint8_t *mac, uint_t datalen, abd_t *pabd, abd_t *cabd,
boolean_t *no_crypt); boolean_t *no_crypt);

147
module/os/linux/zfs/zio_crypt.c
View File

@ -25,6 +25,8 @@
#include <sys/zio.h> #include <sys/zio.h>
#include <sys/zil.h> #include <sys/zil.h>
#include <sys/sha2.h> #include <sys/sha2.h>
#include <sys/simd.h>
#include <sys/spa_impl.h>
#include <sys/hkdf.h> #include <sys/hkdf.h>
#include <sys/qat.h> #include <sys/qat.h>
@ -374,7 +376,7 @@ error:
* plaintext / ciphertext alone. * plaintext / ciphertext alone.
*/ */
static int static int
zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key, zio_do_crypt_uio_impl(boolean_t encrypt, uint64_t crypt, crypto_key_t *key,
crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen, crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen,
uio_t *puio, uio_t *cuio, uint8_t *authbuf, uint_t auth_len) uio_t *puio, uio_t *cuio, uint8_t *authbuf, uint_t auth_len)
{ {
@ -474,9 +476,75 @@ error:
return (ret); return (ret);
} }
typedef struct crypt_uio_arg {
boolean_t cu_encrypt;
uint64_t cu_crypt;
crypto_key_t *cu_key;
crypto_ctx_template_t cu_tmpl;
uint8_t *cu_ivbuf;
uint_t cu_datalen;
uio_t *cu_puio;
uio_t *cu_cuio;
uint8_t *cu_authbuf;
uint_t cu_auth_len;
int cu_error;
} crypt_uio_arg_t;
static void
zio_do_crypt_uio_func(void *arg)
{
crypt_uio_arg_t *cu = (crypt_uio_arg_t *)arg;
cu->cu_error = zio_do_crypt_uio_impl(cu->cu_encrypt, cu->cu_crypt,
cu->cu_key, cu->cu_tmpl, cu->cu_ivbuf, cu->cu_datalen,
cu->cu_puio, cu->cu_cuio, cu->cu_authbuf, cu->cu_auth_len);
}
static int
zio_do_crypt_uio(spa_t *spa, boolean_t encrypt, uint64_t crypt,
crypto_key_t *key, crypto_ctx_template_t tmpl, uint8_t *ivbuf,
uint_t datalen, uio_t *puio, uio_t *cuio, uint8_t *authbuf,
uint_t auth_len)
{
int error;
/*
* Dispatch to the I/O pipeline as required by the context in order
* to take advantage of the SIMD optimization when available.
*/
if (kfpu_allowed()) {
error = zio_do_crypt_uio_impl(encrypt, crypt, key, tmpl,
ivbuf, datalen, puio, cuio, authbuf, auth_len);
} else {
crypt_uio_arg_t *cu;
cu = kmem_alloc(sizeof (*cu), KM_SLEEP);
cu->cu_encrypt = encrypt;
cu->cu_crypt = crypt;
cu->cu_key = key;
cu->cu_tmpl = tmpl;
cu->cu_ivbuf = ivbuf;
cu->cu_datalen = datalen;
cu->cu_puio = puio;
cu->cu_cuio = cuio;
cu->cu_authbuf = authbuf;
cu->cu_auth_len = auth_len;
cu->cu_error = 0;
spa_taskq_dispatch_sync(spa,
encrypt ? ZIO_TYPE_WRITE : ZIO_TYPE_READ,
ZIO_TASKQ_ISSUE, zio_do_crypt_uio_func, cu, TQ_SLEEP);
error = cu->cu_error;
kmem_free(cu, sizeof (*cu));
}
return (error);
}
int int
zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv, zio_crypt_key_wrap(spa_t *spa, crypto_key_t *cwkey, zio_crypt_key_t *key,
uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out) uint8_t *iv, uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out)
{ {
int ret; int ret;
uio_t puio, cuio; uio_t puio, cuio;
@ -533,7 +601,7 @@ zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv,
cuio.uio_segflg = UIO_SYSSPACE; cuio.uio_segflg = UIO_SYSSPACE;
/* encrypt the keys and store the resulting ciphertext and mac */ /* encrypt the keys and store the resulting ciphertext and mac */
ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len, ret = zio_do_crypt_uio(spa, B_TRUE, crypt, cwkey, NULL, iv, enc_len,
&puio, &cuio, (uint8_t *)aad, aad_len); &puio, &cuio, (uint8_t *)aad, aad_len);
if (ret != 0) if (ret != 0)
goto error; goto error;
@ -544,17 +612,38 @@ error:
return (ret); return (ret);
} }
int static void
zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, zio_crypt_create_ctx_templates(void *arg)
uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv,
uint8_t *mac, zio_crypt_key_t *key)
{ {
int ret; zio_crypt_key_t *key = (zio_crypt_key_t *)arg;
crypto_mechanism_t mech; crypto_mechanism_t mech;
int ret;
mech.cm_type = crypto_mech2id(
zio_crypt_table[key->zk_crypt].ci_mechname);
ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
&key->zk_current_tmpl, KM_SLEEP);
if (ret != CRYPTO_SUCCESS)
key->zk_current_tmpl = NULL;
mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
&key->zk_hmac_tmpl, KM_SLEEP);
if (ret != CRYPTO_SUCCESS)
key->zk_hmac_tmpl = NULL;
}
int
zio_crypt_key_unwrap(spa_t *spa, crypto_key_t *cwkey, uint64_t crypt,
uint64_t version, uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata,
uint8_t *iv, uint8_t *mac, zio_crypt_key_t *key)
{
uio_t puio, cuio; uio_t puio, cuio;
uint64_t aad[3]; uint64_t aad[3];
iovec_t plain_iovecs[2], cipher_iovecs[3]; iovec_t plain_iovecs[2], cipher_iovecs[3];
uint_t enc_len, keydata_len, aad_len; uint_t enc_len, keydata_len, aad_len;
int ret;
ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW);
@ -596,7 +685,7 @@ zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version,
cuio.uio_segflg = UIO_SYSSPACE; cuio.uio_segflg = UIO_SYSSPACE;
/* decrypt the keys and store the result in the output buffers */ /* decrypt the keys and store the result in the output buffers */
ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len, ret = zio_do_crypt_uio(spa, B_FALSE, crypt, cwkey, NULL, iv, enc_len,
&puio, &cuio, (uint8_t *)aad, aad_len); &puio, &cuio, (uint8_t *)aad, aad_len);
if (ret != 0) if (ret != 0)
goto error; goto error;
@ -622,27 +711,18 @@ zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version,
key->zk_hmac_key.ck_data = key->zk_hmac_keydata; key->zk_hmac_key.ck_data = key->zk_hmac_keydata;
key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
/*
* Initialize the crypto templates. It's ok if this fails because
* this is just an optimization.
*/
mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname);
ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
&key->zk_current_tmpl, KM_SLEEP);
if (ret != CRYPTO_SUCCESS)
key->zk_current_tmpl = NULL;
mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
&key->zk_hmac_tmpl, KM_SLEEP);
if (ret != CRYPTO_SUCCESS)
key->zk_hmac_tmpl = NULL;
key->zk_crypt = crypt; key->zk_crypt = crypt;
key->zk_version = version; key->zk_version = version;
key->zk_guid = guid; key->zk_guid = guid;
key->zk_salt_count = 0; key->zk_salt_count = 0;
/*
* Initialize the crypto templates in the context they will be
* primarily used. It's ok if this fails, it's just an optimization.
*/
spa_taskq_dispatch_sync(spa, ZIO_TYPE_READ, ZIO_TASKQ_ISSUE,
zio_crypt_create_ctx_templates, key, TQ_SLEEP);
return (0); return (0);
error: error:
@ -1861,7 +1941,7 @@ error:
* Primary encryption / decryption entrypoint for zio data. * Primary encryption / decryption entrypoint for zio data.
*/ */
int int
zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, zio_do_crypt_data(spa_t *spa, boolean_t encrypt, zio_crypt_key_t *key,
dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv, dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv,
uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf, uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf,
boolean_t *no_crypt) boolean_t *no_crypt)
@ -1948,8 +2028,8 @@ zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key,
goto error; goto error;
/* perform the encryption / decryption in software */ /* perform the encryption / decryption in software */
ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len, ret = zio_do_crypt_uio(spa, encrypt, key->zk_crypt, ckey, tmpl, iv,
&puio, &cuio, authbuf, auth_len); enc_len, &puio, &cuio, authbuf, auth_len);
if (ret != 0) if (ret != 0)
goto error; goto error;
@ -1985,9 +2065,10 @@ error:
* linear buffers. * linear buffers.
*/ */
int int
zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot, zio_do_crypt_abd(spa_t *spa, boolean_t encrypt, zio_crypt_key_t *key,
boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac, dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv,
uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt) uint8_t *mac, uint_t datalen, abd_t *pabd, abd_t *cabd,
boolean_t *no_crypt)
{ {
int ret; int ret;
void *ptmp, *ctmp; void *ptmp, *ctmp;
@ -2000,7 +2081,7 @@ zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot,
ctmp = abd_borrow_buf_copy(cabd, datalen); ctmp = abd_borrow_buf_copy(cabd, datalen);
} }
ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac, ret = zio_do_crypt_data(spa, encrypt, key, ot, byteswap, salt, iv, mac,
datalen, ptmp, ctmp, no_crypt); datalen, ptmp, ctmp, no_crypt);
if (ret != 0) if (ret != 0)
goto error; goto error;

2
module/zfs/arc.c
View File

@ -8828,7 +8828,7 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
if (ret != 0) if (ret != 0)
goto error; goto error;
ret = zio_do_crypt_abd(B_TRUE, &dck->dck_key, ret = zio_do_crypt_abd(spa, B_TRUE, &dck->dck_key,
hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt,
hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd, hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd,
&no_crypt); &no_crypt);

20
module/zfs/dsl_crypt.c
View File

@ -601,8 +601,8 @@ dsl_crypto_key_open(objset_t *mos, dsl_wrapping_key_t *wkey,
* Unwrap the keys. If there is an error return EACCES to indicate * Unwrap the keys. If there is an error return EACCES to indicate
* an authentication failure. * an authentication failure.
*/ */
ret = zio_crypt_key_unwrap(&wkey->wk_key, crypt, version, guid, ret = zio_crypt_key_unwrap(mos->os_spa, &wkey->wk_key, crypt, version,
raw_keydata, raw_hmac_keydata, iv, mac, &dck->dck_key); guid, raw_keydata, raw_hmac_keydata, iv, mac, &dck->dck_key);
if (ret != 0) { if (ret != 0) {
ret = SET_ERROR(EACCES); ret = SET_ERROR(EACCES);
goto error; goto error;
@ -1221,6 +1221,7 @@ dsl_crypto_key_sync(dsl_crypto_key_t *dck, dmu_tx_t *tx)
{ {
zio_crypt_key_t *key = &dck->dck_key; zio_crypt_key_t *key = &dck->dck_key;
dsl_wrapping_key_t *wkey = dck->dck_wkey; dsl_wrapping_key_t *wkey = dck->dck_wkey;
objset_t *mos = tx->tx_pool->dp_meta_objset;
uint8_t keydata[MASTER_KEY_MAX_LEN]; uint8_t keydata[MASTER_KEY_MAX_LEN];
uint8_t hmac_keydata[SHA512_HMAC_KEYLEN]; uint8_t hmac_keydata[SHA512_HMAC_KEYLEN];
uint8_t iv[WRAPPING_IV_LEN]; uint8_t iv[WRAPPING_IV_LEN];
@ -1230,14 +1231,13 @@ dsl_crypto_key_sync(dsl_crypto_key_t *dck, dmu_tx_t *tx)
ASSERT3U(key->zk_crypt, <, ZIO_CRYPT_FUNCTIONS); ASSERT3U(key->zk_crypt, <, ZIO_CRYPT_FUNCTIONS);
/* encrypt and store the keys along with the IV and MAC */ /* encrypt and store the keys along with the IV and MAC */
VERIFY0(zio_crypt_key_wrap(&dck->dck_wkey->wk_key, key, iv, mac, VERIFY0(zio_crypt_key_wrap(mos->os_spa, &dck->dck_wkey->wk_key, key,
keydata, hmac_keydata)); iv, mac, keydata, hmac_keydata));
/* update the ZAP with the obtained values */ /* update the ZAP with the obtained values */
dsl_crypto_key_sync_impl(tx->tx_pool->dp_meta_objset, dck->dck_obj, dsl_crypto_key_sync_impl(mos, dck->dck_obj, key->zk_crypt,
key->zk_crypt, wkey->wk_ddobj, key->zk_guid, iv, mac, keydata, wkey->wk_ddobj, key->zk_guid, iv, mac, keydata, hmac_keydata,
hmac_keydata, wkey->wk_keyformat, wkey->wk_salt, wkey->wk_iters, wkey->wk_keyformat, wkey->wk_salt, wkey->wk_iters, tx);
tx);
} }
typedef struct spa_keystore_change_key_args { typedef struct spa_keystore_change_key_args {
@ -2815,8 +2815,8 @@ spa_do_crypt_abd(boolean_t encrypt, spa_t *spa, const zbookmark_phys_t *zb,
} }
/* call lower level function to perform encryption / decryption */ /* call lower level function to perform encryption / decryption */
ret = zio_do_crypt_data(encrypt, &dck->dck_key, ot, bswap, salt, iv, ret = zio_do_crypt_data(spa, encrypt, &dck->dck_key, ot, bswap, salt,
mac, datalen, plainbuf, cipherbuf, no_crypt); iv, mac, datalen, plainbuf, cipherbuf, no_crypt);
/* /*
* Handle injected decryption faults. Unfortunately, we cannot inject * Handle injected decryption faults. Unfortunately, we cannot inject