Post-Encryption Followup

This PR includes fixes for bugs and documentation issues found 
after the encryption patch was merged and general code improvements 
for long-term maintainability.

Reviewed-by: Jorgen Lundman <lundman@lundman.net>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Tom Caputi <tcaputi@datto.com>
Issue #6526
Closes #6639
Closes #6703
Cloese #6706
Closes #6714
Closes #6595

COPYRIGHT

@@ -23,7 +23,17 @@ approved for release under LLNL-CODE-403049.
 
 Unless otherwise noted, all files in this distribution are released
 under the Common Development and Distribution License (CDDL).
-Exceptions are noted within the associated source files.  See the file
+Exceptions are noted within the associated source files.  A few notable
-OPENSOLARIS.LICENSE for more information.
+exceptions and their respective licenses include:
+
+Skein Checksum Implementation: module/icp/algs/skein/THIRDPARTYLICENSE
+AES Implementation: module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.gladman
+AES Implementation: module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.openssl
+PBKDF2 Implementation: lib/libzfs/THIRDPARTYLICENSE.openssl
+
+This product includes software developed by the OpenSSL Project for use
+in the OpenSSL Toolkit (http://www.openssl.org/)
+
+See the file OPENSOLARIS.LICENSE for more information.
 
 Refer to the git commit log for authoritative copyright attribution.

cmd/zfs/zfs_main.c

@@ -7267,28 +7267,27 @@ zfs_do_change_key(int argc, char **argv)
 		keystatus = zfs_prop_get_int(zhp, ZFS_PROP_KEYSTATUS);
 		if (keystatus != ZFS_KEYSTATUS_AVAILABLE) {
 			ret = zfs_crypto_load_key(zhp, B_FALSE, NULL);
-			if (ret != 0)
+			if (ret != 0) {
-				goto error;
+				nvlist_free(props);
+				zfs_close(zhp);
+				return (-1);
+			}
 		}
 
-		/* refresh the properties so the new keystatus is visable */
+		/* refresh the properties so the new keystatus is visible */
 		zfs_refresh_properties(zhp);
 	}
 
 	ret = zfs_crypto_rewrap(zhp, props, inheritkey);
-	if (ret != 0)
+	if (ret != 0) {
-		goto error;
+		nvlist_free(props);
+		zfs_close(zhp);
+		return (-1);
+	}
 
 	nvlist_free(props);
 	zfs_close(zhp);
 	return (0);
-
-error:
-	if (props != NULL)
-		nvlist_free(props);
-	if (zhp != NULL)
-		zfs_close(zhp);
-	return (-1);
 }
 
 int

cmd/zpool/zpool_main.c

@@ -8049,10 +8049,17 @@ main(int argc, char **argv)
 		 * 'freeze' is a vile debugging abomination, so we treat
 		 * it as such.
 		 */
-		char buf[16384];
+		zfs_cmd_t zc = {"\0"};
-		int fd = open(ZFS_DEV, O_RDWR);
+
-		(void) strlcpy((void *)buf, argv[2], sizeof (buf));
+		(void) strlcpy(zc.zc_name, argv[2], sizeof (zc.zc_name));
-		return (!!ioctl(fd, ZFS_IOC_POOL_FREEZE, buf));
+		ret = zfs_ioctl(g_zfs, ZFS_IOC_POOL_FREEZE, &zc);
+		if (ret != 0) {
+			(void) fprintf(stderr,
+			gettext("failed to freeze pool: %d\n"), errno);
+			ret = 1;
+		}
+
+		log_history = 0;
 	} else {
 		(void) fprintf(stderr, gettext("unrecognized "
 		    "command '%s'\n"), cmdname);

cmd/ztest/ztest.c

@@ -201,6 +201,7 @@ extern int  zfs_abd_scatter_enabled;
 
 static ztest_shared_opts_t *ztest_shared_opts;
 static ztest_shared_opts_t ztest_opts;
+static char *ztest_wkeydata = "abcdefghijklmnopqrstuvwxyz012345";
 
 typedef struct ztest_shared_ds {
 	uint64_t	zd_seq;
@@ -1180,6 +1181,42 @@ ztest_spa_prop_set_uint64(zpool_prop_t prop, uint64_t value)
 	return (error);
 }
 
+static int
+ztest_dmu_objset_own(const char *name, dmu_objset_type_t type,
+    boolean_t readonly, boolean_t decrypt, void *tag, objset_t **osp)
+{
+	int err;
+
+	err = dmu_objset_own(name, type, readonly, decrypt, tag, osp);
+	if (decrypt && err == EACCES) {
+		char ddname[ZFS_MAX_DATASET_NAME_LEN];
+		dsl_crypto_params_t *dcp;
+		nvlist_t *crypto_args = fnvlist_alloc();
+		char *cp = NULL;
+
+		/* spa_keystore_load_wkey() expects a dsl dir name */
+		strcpy(ddname, name);
+		cp = strchr(ddname, '@');
+		if (cp != NULL)
+			*cp = '\0';
+
+		fnvlist_add_uint8_array(crypto_args, "wkeydata",
+		    (uint8_t *)ztest_wkeydata, WRAPPING_KEY_LEN);
+		VERIFY0(dsl_crypto_params_create_nvlist(DCP_CMD_NONE, NULL,
+		    crypto_args, &dcp));
+		err = spa_keystore_load_wkey(ddname, dcp, B_FALSE);
+		dsl_crypto_params_free(dcp, B_FALSE);
+		fnvlist_free(crypto_args);
+
+		if (err != 0)
+			return (err);
+
+		err = dmu_objset_own(name, type, readonly, decrypt, tag, osp);
+	}
+
+	return (err);
+}
+
 
 /*
  * Object and range lock mechanics
@@ -1921,7 +1958,7 @@ ztest_replay_write(ztest_ds_t *zd, lr_write_t *lr, boolean_t byteswap)
 		dmu_write(os, lr->lr_foid, offset, length, data, tx);
 	} else {
 		bcopy(data, abuf->b_data, length);
-		dmu_assign_arcbuf(db, offset, abuf, tx);
+		dmu_assign_arcbuf_by_dbuf(db, offset, abuf, tx);
 	}
 
 	(void) ztest_log_write(zd, tx, lr);
@@ -3532,11 +3569,57 @@ ztest_objset_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx)
 static int
 ztest_dataset_create(char *dsname)
 {
-	uint64_t zilset = ztest_random(100);
+	int err;
-	int err = dmu_objset_create(dsname, DMU_OST_OTHER, 0, NULL,
+	uint64_t rand;
-	    ztest_objset_create_cb, NULL);
+	dsl_crypto_params_t *dcp = NULL;
 
-	if (err || zilset < 80)
+	/*
+	 * 50% of the time, we create encrypted datasets
+	 * using a random cipher suite and a hard-coded
+	 * wrapping key.
+	 */
+	rand = ztest_random(2);
+	if (rand != 0) {
+		nvlist_t *crypto_args = fnvlist_alloc();
+		nvlist_t *props = fnvlist_alloc();
+
+		/* slight bias towards the default cipher suite */
+		rand = ztest_random(ZIO_CRYPT_FUNCTIONS);
+		if (rand < ZIO_CRYPT_AES_128_CCM)
+			rand = ZIO_CRYPT_ON;
+
+		fnvlist_add_uint64(props,
+		    zfs_prop_to_name(ZFS_PROP_ENCRYPTION), rand);
+		fnvlist_add_uint8_array(crypto_args, "wkeydata",
+		    (uint8_t *)ztest_wkeydata, WRAPPING_KEY_LEN);
+
+		/*
+		 * These parameters aren't really used by the kernel. They
+		 * are simply stored so that userspace knows how to load
+		 * the wrapping key.
+		 */
+		fnvlist_add_uint64(props,
+		    zfs_prop_to_name(ZFS_PROP_KEYFORMAT), ZFS_KEYFORMAT_RAW);
+		fnvlist_add_string(props,
+		    zfs_prop_to_name(ZFS_PROP_KEYLOCATION), "prompt");
+		fnvlist_add_uint64(props,
+		    zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 0ULL);
+		fnvlist_add_uint64(props,
+		    zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 0ULL);
+
+		VERIFY0(dsl_crypto_params_create_nvlist(DCP_CMD_NONE, props,
+		    crypto_args, &dcp));
+
+		fnvlist_free(crypto_args);
+		fnvlist_free(props);
+	}
+
+	err = dmu_objset_create(dsname, DMU_OST_OTHER, 0, dcp,
+	    ztest_objset_create_cb, NULL);
+	dsl_crypto_params_free(dcp, !!err);
+
+	rand = ztest_random(100);
+	if (err || rand < 80)
 		return (err);
 
 	if (ztest_opts.zo_verbose >= 5)
@@ -3556,7 +3639,8 @@ ztest_objset_destroy_cb(const char *name, void *arg)
 	/*
 	 * Verify that the dataset contains a directory object.
 	 */
-	VERIFY0(dmu_objset_own(name, DMU_OST_OTHER, B_TRUE, B_TRUE, FTAG, &os));
+	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
+	    B_TRUE, FTAG, &os));
 	error = dmu_object_info(os, ZTEST_DIROBJ, &doi);
 	if (error != ENOENT) {
 		/* We could have crashed in the middle of destroying it */
@@ -3640,11 +3724,12 @@ ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
 	 * (invoked from ztest_objset_destroy_cb()) should just throw it away.
 	 */
 	if (ztest_random(2) == 0 &&
-	    dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
+	    ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
 	    B_TRUE, FTAG, &os) == 0) {
 		ztest_zd_init(zdtmp, NULL, os);
 		zil_replay(os, zdtmp, ztest_replay_vector);
 		ztest_zd_fini(zdtmp);
+		txg_wait_synced(dmu_objset_pool(os), 0);
 		dmu_objset_disown(os, B_TRUE, FTAG);
 	}
 
@@ -3659,8 +3744,8 @@ ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
 	/*
 	 * Verify that the destroyed dataset is no longer in the namespace.
 	 */
-	VERIFY3U(ENOENT, ==, dmu_objset_own(name, DMU_OST_OTHER, B_TRUE, B_TRUE,
+	VERIFY3U(ENOENT, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
-	    FTAG, &os));
+	    B_TRUE, FTAG, &os));
 
 	/*
 	 * Verify that we can create a new dataset.
@@ -3674,7 +3759,7 @@ ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
 		fatal(0, "dmu_objset_create(%s) = %d", name, error);
 	}
 
-	VERIFY0(dmu_objset_own(name, DMU_OST_OTHER, B_FALSE, B_TRUE,
+	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE, B_TRUE,
 	    FTAG, &os));
 
 	ztest_zd_init(zdtmp, NULL, os);
@@ -3710,10 +3795,11 @@ ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
 	/*
 	 * Verify that we cannot own an objset that is already owned.
 	 */
-	VERIFY3U(EBUSY, ==,
+	VERIFY3U(EBUSY, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER,
-	    dmu_objset_own(name, DMU_OST_OTHER, B_FALSE, B_TRUE, FTAG, &os2));
+	    B_FALSE, B_TRUE, FTAG, &os2));
 
 	zil_close(zilog);
+	txg_wait_synced(spa_get_dsl(os->os_spa), 0);
 	dmu_objset_disown(os, B_TRUE, FTAG);
 	ztest_zd_fini(zdtmp);
 out:
@@ -3868,7 +3954,7 @@ ztest_dsl_dataset_promote_busy(ztest_ds_t *zd, uint64_t id)
 		fatal(0, "dmu_objset_create(%s) = %d", clone2name, error);
 	}
 
-	error = dmu_objset_own(snap2name, DMU_OST_ANY, B_TRUE, B_TRUE,
+	error = ztest_dmu_objset_own(snap2name, DMU_OST_ANY, B_TRUE, B_TRUE,
 	    FTAG, &os);
 	if (error)
 		fatal(0, "dmu_objset_own(%s) = %d", snap2name, error);
@@ -4260,7 +4346,7 @@ ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
 	 *	bigobj, at the tail of the nth chunk
 	 *
 	 * The chunk size is set equal to bigobj block size so that
-	 * dmu_assign_arcbuf() can be tested for object updates.
+	 * dmu_assign_arcbuf_by_dbuf() can be tested for object updates.
 	 */
 
 	/*
@@ -4322,7 +4408,7 @@ ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
 		/*
 		 * In iteration 5 (i == 5) use arcbufs
 		 * that don't match bigobj blksz to test
-		 * dmu_assign_arcbuf() when it can't directly
+		 * dmu_assign_arcbuf_by_dbuf() when it can't directly
 		 * assign an arcbuf to a dbuf.
 		 */
 		for (j = 0; j < s; j++) {
@@ -4368,8 +4454,8 @@ ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
 
 		/*
 		 * 50% of the time don't read objects in the 1st iteration to
-		 * test dmu_assign_arcbuf() for the case when there're no
+		 * test dmu_assign_arcbuf_by_dbuf() for the case when there are
-		 * existing dbufs for the specified offsets.
+		 * no existing dbufs for the specified offsets.
 		 */
 		if (i != 0 || ztest_random(2) != 0) {
 			error = dmu_read(os, packobj, packoff,
@@ -4414,12 +4500,12 @@ ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
 				    FTAG, &dbt, DMU_READ_NO_PREFETCH) == 0);
 			}
 			if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
-				dmu_assign_arcbuf(bonus_db, off,
+				dmu_assign_arcbuf_by_dbuf(bonus_db, off,
 				    bigbuf_arcbufs[j], tx);
 			} else {
-				dmu_assign_arcbuf(bonus_db, off,
+				dmu_assign_arcbuf_by_dbuf(bonus_db, off,
 				    bigbuf_arcbufs[2 * j], tx);
-				dmu_assign_arcbuf(bonus_db,
+				dmu_assign_arcbuf_by_dbuf(bonus_db,
 				    off + chunksize / 2,
 				    bigbuf_arcbufs[2 * j + 1], tx);
 			}
@@ -6262,7 +6348,8 @@ ztest_dataset_open(int d)
 	}
 	ASSERT(error == 0 || error == EEXIST);
 
-	VERIFY0(dmu_objset_own(name, DMU_OST_OTHER, B_FALSE, B_TRUE, zd, &os));
+	VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
+	    B_TRUE, zd, &os));
 	(void) rw_unlock(&ztest_name_lock);
 
 	ztest_zd_init(zd, ZTEST_GET_SHARED_DS(d), os);
@@ -6303,6 +6390,7 @@ ztest_dataset_close(int d)
 	ztest_ds_t *zd = &ztest_ds[d];
 
 	zil_close(zd->zd_zilog);
+	txg_wait_synced(spa_get_dsl(zd->zd_os->os_spa), 0);
 	dmu_objset_disown(zd->zd_os, B_TRUE, zd);
 
 	ztest_zd_fini(zd);
@@ -6355,7 +6443,7 @@ ztest_run(ztest_shared_t *zs)
 	ztest_spa = spa;
 
 	dmu_objset_stats_t dds;
-	VERIFY0(dmu_objset_own(ztest_opts.zo_pool,
+	VERIFY0(ztest_dmu_objset_own(ztest_opts.zo_pool,
 	    DMU_OST_ANY, B_TRUE, B_TRUE, FTAG, &os));
 	dsl_pool_config_enter(dmu_objset_pool(os), FTAG);
 	dmu_objset_fast_stat(os, &dds);
@@ -6582,11 +6670,10 @@ ztest_freeze(void)
 	VERIFY3U(0, ==, spa_open(ztest_opts.zo_pool, &spa, FTAG));
 	ASSERT(spa_freeze_txg(spa) == UINT64_MAX);
 	VERIFY3U(0, ==, ztest_dataset_open(0));
-	ztest_dataset_close(0);
-
 	spa->spa_debug = B_TRUE;
 	ztest_spa = spa;
 	txg_wait_synced(spa_get_dsl(spa), 0);
+	ztest_dataset_close(0);
 	ztest_reguid(NULL, 0);
 
 	spa_close(spa, FTAG);

config/user-libssl.m4

@@ -0,0 +1,12 @@
+dnl #
+dnl # Check for libssl. Used for userspace password derivation via PBKDF2.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_USER_LIBSSL], [
+	LIBSSL=
+
+	AC_CHECK_HEADER([openssl/evp.h], [], [AC_MSG_FAILURE([
+        *** evp.h missing, libssl-devel package required])])
+
+	AC_SUBST([LIBSSL], ["-lssl -lcrypto"])
+	AC_DEFINE([HAVE_LIBSSL], 1, [Define if you have libssl])
+])

config/user.m4

@@ -13,6 +13,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER], [
 	ZFS_AC_CONFIG_USER_LIBBLKID
 	ZFS_AC_CONFIG_USER_LIBATTR
 	ZFS_AC_CONFIG_USER_LIBUDEV
+	ZFS_AC_CONFIG_USER_LIBSSL
 	ZFS_AC_CONFIG_USER_FRAME_LARGER_THAN
 	ZFS_AC_CONFIG_USER_RUNSTATEDIR
 	ZFS_AC_CONFIG_USER_MAKEDEV_IN_SYSMACROS

configure.ac

@@ -248,6 +248,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/tests/functional/grow_pool/Makefile
 	tests/zfs-tests/tests/functional/grow_replicas/Makefile
 	tests/zfs-tests/tests/functional/history/Makefile
+	tests/zfs-tests/tests/functional/hkdf/Makefile
 	tests/zfs-tests/tests/functional/inheritance/Makefile
 	tests/zfs-tests/tests/functional/inuse/Makefile
 	tests/zfs-tests/tests/functional/large_files/Makefile

contrib/dracut/90zfs/module-setup.sh.in

@@ -25,6 +25,7 @@ installkernel() {
 	instmods znvpair
 	instmods zavl
 	instmods zunicode
+	instmods icp
 	instmods spl
 	instmods zlib_deflate
 	instmods zlib_inflate

include/sys/Makefile.am

@@ -35,6 +35,7 @@ COMMON_H = \
 	$(top_srcdir)/include/sys/dsl_userhold.h \
 	$(top_srcdir)/include/sys/edonr.h \
 	$(top_srcdir)/include/sys/efi_partition.h \
+	$(top_srcdir)/include/sys/hkdf.h \
 	$(top_srcdir)/include/sys/metaslab.h \
 	$(top_srcdir)/include/sys/metaslab_impl.h \
 	$(top_srcdir)/include/sys/mmp.h \

include/sys/arc_impl.h

@@ -96,6 +96,7 @@ struct arc_callback {
 	boolean_t		acb_encrypted;
 	boolean_t		acb_compressed;
 	boolean_t		acb_noauth;
+	uint64_t		acb_dsobj;
 	zio_t			*acb_zio_dummy;
 	arc_callback_t		*acb_next;
 };

include/sys/dmu.h

@@ -759,10 +759,13 @@ void dmu_tx_callback_register(dmu_tx_t *tx, dmu_tx_callback_func_t *dcb_func,
  * -1, the range from offset to end-of-file is freed.
  */
 int dmu_free_range(objset_t *os, uint64_t object, uint64_t offset,
-	uint64_t size, dmu_tx_t *tx);
+    uint64_t size, dmu_tx_t *tx);
 int dmu_free_long_range(objset_t *os, uint64_t object, uint64_t offset,
-	uint64_t size);
+    uint64_t size);
+int dmu_free_long_range_raw(objset_t *os, uint64_t object, uint64_t offset,
+    uint64_t size);
 int dmu_free_long_object(objset_t *os, uint64_t object);
+int dmu_free_long_object_raw(objset_t *os, uint64_t object);
 
 /*
  * Convenience functions.
@@ -797,10 +800,11 @@ int dmu_write_uio_dnode(dnode_t *dn, struct uio *uio, uint64_t size,
 #endif
 struct arc_buf *dmu_request_arcbuf(dmu_buf_t *handle, int size);
 void dmu_return_arcbuf(struct arc_buf *buf);
-void dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, struct arc_buf *buf,
+void dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset,
-    dmu_tx_t *tx);
+    struct arc_buf *buf, dmu_tx_t *tx);
-void dmu_assign_arcbuf_impl(dmu_buf_t *handle, struct arc_buf *buf,
+void dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset,
-    dmu_tx_t *tx);
+    struct arc_buf *buf, dmu_tx_t *tx);
+#define	dmu_assign_arcbuf	dmu_assign_arcbuf_by_dbuf
 void dmu_convert_to_raw(dmu_buf_t *handle, boolean_t byteorder,
     const uint8_t *salt, const uint8_t *iv, const uint8_t *mac, dmu_tx_t *tx);
 void dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset,

include/sys/hkdf.h

@@ -0,0 +1,29 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Datto, Inc. All rights reserved.
+ */
+
+#ifndef	_SYS_HKDF_H_
+#define	_SYS_HKDF_H_
+
+#include <sys/types.h>
+
+int hkdf_sha512(uint8_t *key_material, uint_t km_len, uint8_t *salt,
+    uint_t salt_len, uint8_t *info, uint_t info_len, uint8_t *output_key,
+    uint_t out_len);
+
+#endif	/* _SYS_HKDF_H_ */

include/sys/zio_crypt.h

@@ -32,18 +32,9 @@ struct zbookmark_phys;
 
 #define	WRAPPING_KEY_LEN	32
 #define	WRAPPING_IV_LEN		ZIO_DATA_IV_LEN
-#define	WRAPPING_MAC_LEN	16
+#define	WRAPPING_MAC_LEN	ZIO_DATA_MAC_LEN
-
-#define	SHA1_DIGEST_LEN		20
-#define	SHA512_DIGEST_LEN	64
-#define	SHA512_HMAC_KEYLEN	64
-
 #define	MASTER_KEY_MAX_LEN	32
-#define	L2ARC_DEFAULT_CRYPT ZIO_CRYPT_AES_256_CCM
+#define	SHA512_HMAC_KEYLEN	64
-
-/* utility macros */
-#define	BITS_TO_BYTES(x) ((x + NBBY - 1) / NBBY)
-#define	BYTES_TO_BITS(x) (x * NBBY)
 
 typedef enum zio_crypt_type {
 	ZC_TYPE_NONE = 0,
@@ -133,7 +124,7 @@ int zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf,
 int zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd,
     uint_t datalen, boolean_t byteswap, uint8_t *cksum);
 int zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen,
-    uint8_t *digestbuf);
+    uint8_t *digestbuf, uint_t digestlen);
 int zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen,
     boolean_t byteswap, uint8_t *portable_mac, uint8_t *local_mac);
 int zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, uint8_t *salt,

lib/libzfs/Makefile.am

@@ -30,6 +30,7 @@ USER_C = \
 	libzfs_util.c
 
 KERNEL_C = \
+	algs/sha2/sha2.c \
 	zfeature_common.c \
 	zfs_comutil.c \
 	zfs_deleg.c \
@@ -52,15 +53,13 @@ nodist_libzfs_la_SOURCES = \
 
 libzfs_la_LIBADD = \
         $(top_builddir)/lib/libefi/libefi.la \
-	$(top_builddir)/lib/libicp/libicp.la \
 	$(top_builddir)/lib/libnvpair/libnvpair.la \
 	$(top_builddir)/lib/libshare/libshare.la \
 	$(top_builddir)/lib/libtpool/libtpool.la \
 	$(top_builddir)/lib/libuutil/libuutil.la \
-	$(top_builddir)/lib/libzpool/libzpool.la \
 	$(top_builddir)/lib/libzfs_core/libzfs_core.la
 
-libzfs_la_LIBADD += -lm $(LIBBLKID) $(LIBUDEV)
+libzfs_la_LIBADD += -lm $(LIBBLKID) $(LIBUDEV) $(LIBSSL)
 libzfs_la_LDFLAGS = -version-info 2:0:0
 
 EXTRA_DIST = $(libzfs_pc_DATA) $(USER_C)

lib/libzfs/THIRDPARTYLICENSE.openssl

@@ -0,0 +1,127 @@
+
+  LICENSE ISSUES
+  ==============
+
+  The OpenSSL toolkit stays under a dual license, i.e. both the conditions of
+  the OpenSSL License and the original SSLeay license apply to the toolkit.
+  See below for the actual license texts. Actually both licenses are BSD-style
+  Open Source licenses. In case of any license issues related to OpenSSL
+  please contact openssl-core@openssl.org.
+
+  OpenSSL License
+  ---------------
+
+/* ====================================================================
+ * Copyright (c) 1998-2008 The OpenSSL Project.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ *    software must display the following acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For written permission, please contact
+ *    openssl-core@openssl.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ *    nor may "OpenSSL" appear in their names without prior written
+ *    permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This product includes cryptographic software written by Eric Young
+ * (eay@cryptsoft.com).  This product includes software written by Tim
+ * Hudson (tjh@cryptsoft.com).
+ *
+ */
+
+ Original SSLeay License
+ -----------------------
+
+/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
+ * All rights reserved.
+ *
+ * This package is an SSL implementation written
+ * by Eric Young (eay@cryptsoft.com).
+ * The implementation was written so as to conform with Netscapes SSL.
+ * 
+ * This library is free for commercial and non-commercial use as long as
+ * the following conditions are aheared to.  The following conditions
+ * apply to all code found in this distribution, be it the RC4, RSA,
+ * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
+ * included with this distribution is covered by the same copyright terms
+ * except that the holder is Tim Hudson (tjh@cryptsoft.com).
+ * 
+ * Copyright remains Eric Young's, and as such any Copyright notices in
+ * the code are not to be removed.
+ * If this package is used in a product, Eric Young should be given attribution
+ * as the author of the parts of the library used.
+ * This can be in the form of a textual message at program startup or
+ * in documentation (online or textual) provided with the package.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    "This product includes cryptographic software written by
+ *     Eric Young (eay@cryptsoft.com)"
+ *    The word 'cryptographic' can be left out if the rouines from the library
+ *    being used are not cryptographic related :-).
+ * 4. If you include any Windows specific code (or a derivative thereof) from 
+ *    the apps directory (application code) you must include an acknowledgement:
+ *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
+ * 
+ * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * 
+ * The licence and distribution terms for any publically available version or
+ * derivative of this code cannot be changed.  i.e. this code cannot simply be
+ * copied and put under another distribution licence
+ * [including the GNU Public Licence.]
+ */
+

lib/libzfs/THIRDPARTYLICENSE.openssl.descrip

@@ -0,0 +1 @@
+PBKDF2 IMPLEMENTATION

lib/libzfs/libzfs_crypto.c

@@ -20,11 +20,11 @@
 #include <sys/zfs_context.h>
 #include <sys/fs/zfs.h>
 #include <sys/dsl_crypt.h>
-#include <sys/crypto/icp.h>
 #include <libintl.h>
 #include <termios.h>
 #include <signal.h>
 #include <errno.h>
+#include <openssl/evp.h>
 #include <libzfs.h>
 #include "libzfs_impl.h"
 #include "zfeature_common.h"
@@ -437,139 +437,6 @@ error:
 	return (ret);
 }
 
-static int
-pbkdf2(uint8_t *passphrase, size_t passphraselen, uint8_t *salt,
-    size_t saltlen, uint64_t iterations, uint8_t *output,
-    size_t outputlen)
-{
-	int ret;
-	uint64_t iter;
-	uint32_t blockptr, i;
-	uint16_t hmac_key_len;
-	uint8_t *hmac_key;
-	uint8_t block[SHA1_DIGEST_LEN * 2];
-	uint8_t *hmacresult = block + SHA1_DIGEST_LEN;
-	crypto_mechanism_t mech;
-	crypto_key_t key;
-	crypto_data_t in_data, out_data;
-	crypto_ctx_template_t tmpl = NULL;
-
-	/* initialize output */
-	memset(output, 0, outputlen);
-
-	/* initialize icp for use */
-	icp_init();
-
-	/* HMAC key size is max(sizeof(uint32_t) + salt len, sha 256 len) */
-	if (saltlen > SHA1_DIGEST_LEN) {
-		hmac_key_len = saltlen + sizeof (uint32_t);
-	} else {
-		hmac_key_len = SHA1_DIGEST_LEN;
-	}
-
-	hmac_key = calloc(hmac_key_len, 1);
-	if (!hmac_key) {
-		ret = ENOMEM;
-		goto error;
-	}
-
-	/* initialize sha 256 hmac mechanism */
-	mech.cm_type = crypto_mech2id(SUN_CKM_SHA1_HMAC);
-	mech.cm_param = NULL;
-	mech.cm_param_len = 0;
-
-	/* initialize passphrase as a crypto key */
-	key.ck_format = CRYPTO_KEY_RAW;
-	key.ck_length = BYTES_TO_BITS(passphraselen);
-	key.ck_data = passphrase;
-
-	/*
-	 * initialize crypto data for the input data. length will change
-	 * after the first iteration, so we will initialize it in the loop.
-	 */
-	in_data.cd_format = CRYPTO_DATA_RAW;
-	in_data.cd_offset = 0;
-	in_data.cd_raw.iov_base = (char *)hmac_key;
-
-	/* initialize crypto data for the output data */
-	out_data.cd_format = CRYPTO_DATA_RAW;
-	out_data.cd_offset = 0;
-	out_data.cd_length = SHA1_DIGEST_LEN;
-	out_data.cd_raw.iov_base = (char *)hmacresult;
-	out_data.cd_raw.iov_len = out_data.cd_length;
-
-	/* initialize the context template */
-	ret = crypto_create_ctx_template(&mech, &key, &tmpl, KM_SLEEP);
-	if (ret != CRYPTO_SUCCESS) {
-		ret = EIO;
-		goto error;
-	}
-
-	/* main loop */
-	for (blockptr = 0; blockptr < outputlen; blockptr += SHA1_DIGEST_LEN) {
-
-		/*
-		 * for the first iteration, the HMAC key is the user-provided
-		 * salt concatenated with the block index (1-indexed)
-		 */
-		i = htobe32(1 + (blockptr / SHA1_DIGEST_LEN));
-		memmove(hmac_key, salt, saltlen);
-		memmove(hmac_key + saltlen, (uint8_t *)(&i), sizeof (uint32_t));
-
-		/* block initializes to zeroes (no XOR) */
-		memset(block, 0, SHA1_DIGEST_LEN);
-
-		for (iter = 0; iter < iterations; iter++) {
-			if (iter > 0) {
-				in_data.cd_length = SHA1_DIGEST_LEN;
-				in_data.cd_raw.iov_len = in_data.cd_length;
-			} else {
-				in_data.cd_length = saltlen + sizeof (uint32_t);
-				in_data.cd_raw.iov_len = in_data.cd_length;
-			}
-
-			ret = crypto_mac(&mech, &in_data, &key, tmpl,
-			    &out_data, NULL);
-			if (ret != CRYPTO_SUCCESS) {
-				ret = EIO;
-				goto error;
-			}
-
-			/* HMAC key now becomes the output of this iteration */
-			memmove(hmac_key, hmacresult, SHA1_DIGEST_LEN);
-
-			/* XOR this iteration's result with the current block */
-			for (i = 0; i < SHA1_DIGEST_LEN; i++) {
-				block[i] ^= hmacresult[i];
-			}
-		}
-
-		/*
-		 * compute length of this block, make sure we don't write
-		 * beyond the end of the output, truncating if necessary
-		 */
-		if (blockptr + SHA1_DIGEST_LEN > outputlen) {
-			memmove(output + blockptr, block, outputlen - blockptr);
-		} else {
-			memmove(output + blockptr, block, SHA1_DIGEST_LEN);
-		}
-	}
-
-	crypto_destroy_ctx_template(tmpl);
-	free(hmac_key);
-	icp_fini();
-
-	return (0);
-
-error:
-	crypto_destroy_ctx_template(tmpl);
-	if (hmac_key != NULL)
-		free(hmac_key);
-	icp_fini();
-
-	return (ret);
-}
-
 static int
 derive_key(libzfs_handle_t *hdl, zfs_keyformat_t format, uint64_t iters,
     uint8_t *key_material, size_t key_material_len, uint64_t salt,
@@ -599,10 +466,12 @@ derive_key(libzfs_handle_t *hdl, zfs_keyformat_t format, uint64_t iters,
 		break;
 	case ZFS_KEYFORMAT_PASSPHRASE:
 		salt = LE_64(salt);
-		ret = pbkdf2(key_material, strlen((char *)key_material),
+
-		    ((uint8_t *)&salt), sizeof (uint64_t), iters,
+		ret = PKCS5_PBKDF2_HMAC_SHA1((char *)key_material,
-		    key, WRAPPING_KEY_LEN);
+		    strlen((char *)key_material), ((uint8_t *)&salt),
-		if (ret != 0) {
+		    sizeof (uint64_t), iters, WRAPPING_KEY_LEN, key);
+		if (ret != 1) {
+			ret = EIO;
 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 			    "Failed to generate key from passphrase."));
 			goto error;
@@ -1207,9 +1076,13 @@ try_again:
 	ret = lzc_load_key(zhp->zfs_name, noop, key_data, WRAPPING_KEY_LEN);
 	if (ret != 0) {
 		switch (ret) {
+		case EPERM:
+			zfs_error_aux(zhp->zfs_hdl, dgettext(TEXT_DOMAIN,
+			    "Permission denied."));
+			break;
 		case EINVAL:
 			zfs_error_aux(zhp->zfs_hdl, dgettext(TEXT_DOMAIN,
-			    "Invalid parameters provided for %s."),
+			    "Invalid parameters provided for dataset %s."),
 			    zfs_get_name(zhp));
 			break;
 		case EEXIST:
@@ -1318,6 +1191,10 @@ zfs_crypto_unload_key(zfs_handle_t *zhp)
 
 	if (ret != 0) {
 		switch (ret) {
+		case EPERM:
+			zfs_error_aux(zhp->zfs_hdl, dgettext(TEXT_DOMAIN,
+			    "Permission denied."));
+			break;
 		case ENOENT:
 			zfs_error_aux(zhp->zfs_hdl, dgettext(TEXT_DOMAIN,
 			    "Key already unloaded for '%s'."),
@@ -1375,8 +1252,10 @@ zfs_crypto_verify_rewrap_nvlist(zfs_handle_t *zhp, nvlist_t *props,
 	new_props = zfs_valid_proplist(zhp->zfs_hdl, zhp->zfs_type, props,
 	    zfs_prop_get_int(zhp, ZFS_PROP_ZONED), NULL, zhp->zpool_hdl,
 	    B_TRUE, errbuf);
-	if (new_props == NULL)
+	if (new_props == NULL) {
+		ret = EINVAL;
 		goto error;
+	}
 
 	*props_out = new_props;
 	return (0);
@@ -1475,6 +1354,13 @@ zfs_crypto_rewrap(zfs_handle_t *zhp, nvlist_t *raw_props, boolean_t inheritkey)
 				ret = nvlist_add_uint64(props,
 				    zfs_prop_to_name(ZFS_PROP_KEYFORMAT),
 				    keyformat);
+				if (ret != 0) {
+					zfs_error_aux(zhp->zfs_hdl,
+					    dgettext(TEXT_DOMAIN, "Failed to "
+					    "get existing keyformat "
+					    "property."));
+					goto error;
+				}
 			}
 
 			if (keylocation == NULL) {
@@ -1578,6 +1464,10 @@ zfs_crypto_rewrap(zfs_handle_t *zhp, nvlist_t *raw_props, boolean_t inheritkey)
 	ret = lzc_change_key(zhp->zfs_name, cmd, props, wkeydata, wkeylen);
 	if (ret != 0) {
 		switch (ret) {
+		case EPERM:
+			zfs_error_aux(zhp->zfs_hdl, dgettext(TEXT_DOMAIN,
+			    "Permission denied."));
+			break;
 		case EINVAL:
 			zfs_error_aux(zhp->zfs_hdl, dgettext(TEXT_DOMAIN,
 			    "Invalid properties for key change."));

lib/libzfs/libzfs_sendrecv.c

@@ -2657,6 +2657,13 @@ recv_fix_encryption_heirarchy(libzfs_handle_t *hdl, const char *destname,
 	int err;
 	nvpair_t *fselem = NULL;
 	nvlist_t *stream_fss;
+	char *cp;
+	char top_zfs[ZFS_MAX_DATASET_NAME_LEN];
+
+	(void) strcpy(top_zfs, destname);
+	cp = strrchr(top_zfs, '@');
+	if (cp != NULL)
+		*cp = '\0';
 
 	VERIFY(0 == nvlist_lookup_nvlist(stream_nv, "fss", &stream_fss));
 
@@ -2758,9 +2765,11 @@ recv_fix_encryption_heirarchy(libzfs_handle_t *hdl, const char *destname,
 		/*
 		 * If the dataset is not flagged as an encryption root and is
 		 * currently an encryption root, force it to inherit from its
-		 * parent.
+		 * parent. The root of a raw send should never be
+		 * force-inherited.
 		 */
-		if (!stream_encroot && is_encroot) {
+		if (!stream_encroot && is_encroot &&
+		    strcmp(top_zfs, fsname) != 0) {
 			err = lzc_change_key(fsname, DCP_CMD_FORCE_INHERIT,
 			    NULL, NULL, 0);
 			if (err != 0) {

lib/libzpool/Makefile.am

@@ -68,6 +68,7 @@ KERNEL_C = \
 	dsl_destroy.c \
 	dsl_userhold.c \
 	edonr_zfs.c \
+	hkdf.c \
 	fm.c \
 	gzip.c \
 	lzjb.c \

man/man8/zfs.8

@@ -2179,18 +2179,31 @@ and
 .Sy pbkdf2iters .
 After entering an encryption key, the
 created dataset will become an encryption root. Any descendant datasets will
-inherit their encryption key from the encryption root, meaning that loading,
+inherit their encryption key from the encryption root by default, meaning that
-unloading, or changing the key for the encryption root will implicitly do the
+loading, unloading, or changing the key for the encryption root will implicitly
-same for all inheriting datasets. If this inheritence is not desired, simply
+do the same for all inheriting datasets. If this inheritance is not desired,
-supply a new
+simply supply a
-.Sy encryption
-and
 .Sy keyformat
 when creating the child dataset or use
 .Nm zfs Cm change-key
-to break the relationship. The one exception is that clones will always use
+to break an existing relationship, creating a new encryption root on the child.
-their origin's encryption key. Encryption root inheritence can be tracked via
+Note that the child's
-the read-only
+.Sy keyformat
+may match that of the parent while still creating a new encryption root, and
+that changing the
+.Sy encryption
+property alone does not create a new encryption root; this would simply use a
+different cipher suite with the same key as its encryption root. The one
+exception is that clones will always use their origin's encryption key.
+As a result of this exception, some encryption-related properties (namely
+.Sy keystatus ,
+.Sy keyformat ,
+.Sy keylocation ,
+and
+.Sy pbkdf2iters )
+do not inherit like other ZFS properties and instead use the value determined
+by their encryption root. Encryption root inheritance can be tracked via the
+read-only
 .Sy encryptionroot
 property.
 .Pp
@@ -3165,7 +3178,10 @@ feature enabled.
 If the
 .Sy lz4_compress
 feature is active on the sending system, then the receiving system must have
-that feature enabled as well.
+that feature enabled as well. Datasets that are sent with this flag may not be
+received as an encrypted dataset, since encrypted datasets cannot use the
+.Sy embedded_data
+feature.
 See
 .Xr zpool-features 5
 for details on ZFS feature flags and the
@@ -3248,7 +3264,10 @@ Include the dataset's properties in the stream.
 This flag is implicit when
 .Fl R
 is specified.
-The receiving system must also support this feature.
+The receiving system must also support this feature. Sends of encrypted datasets
+must use
+.Fl w
+when using this flag.
 .It Fl v, -verbose
 Print verbose information about the stream package generated.
 This information includes a per-second report of how much data has been sent.
@@ -3339,8 +3358,10 @@ feature enabled.
 If the
 .Sy lz4_compress
 feature is active on the sending system, then the receiving system must have
-that feature enabled as well. Note that streams generated using this flag are
+that feature enabled as well. Datasets that are sent with this flag may not be
-unable to be received into an encrypted dataset.
+received as an encrypted dataset, since encrypted datasets cannot use the
+.Sy embedded_data
+feature.
 See
 .Xr zpool-features 5
 for details on ZFS feature flags and the
@@ -3463,6 +3484,15 @@ is a special case because, even if
 is a read-only property and cannot be set, it's allowed to receive the send
 stream as a clone of the given snapshot.
 .Pp
+Raw encrypted send streams (created with
+.Nm zfs Cm send Fl w
+) may only be received as is, and cannot be re-encrypted, decrypted, or
+recompressed by the receive process. Unencrypted streams can be received as
+encrypted datasets, either through inheritance or by specifying encryption
+parameters with the
+.Fl o
+options.
+.Pp
 The name of the snapshot
 .Pq and file system, if a full stream is received
 that this subcommand creates depends on the argument type and the use of the

module/icp/algs/sha2/sha2.c

@@ -52,7 +52,8 @@
 static void Encode(uint8_t *, uint32_t *, size_t);
 static void Encode64(uint8_t *, uint64_t *, size_t);
 
-#if	defined(__amd64)
+/* userspace only supports the generic version */
+#if	defined(__amd64) && defined(_KERNEL)
 #define	SHA512Transform(ctx, in) SHA512TransformBlocks((ctx), (in), 1)
 #define	SHA256Transform(ctx, in) SHA256TransformBlocks((ctx), (in), 1)
 
@@ -62,7 +63,7 @@ void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num);
 #else
 static void SHA256Transform(SHA2_CTX *, const uint8_t *);
 static void SHA512Transform(SHA2_CTX *, const uint8_t *);
-#endif	/* __amd64 */
+#endif	/* __amd64 && _KERNEL */
 
 static uint8_t PADDING[128] = { 0x80, /* all zeros */ };
 
@@ -142,7 +143,7 @@ static uint8_t PADDING[128] = { 0x80, /* all zeros */ };
 #endif	/* _BIG_ENDIAN */
 
 
-#if	!defined(__amd64)
+#if	!defined(__amd64) || !defined(_KERNEL)
 /* SHA256 Transform */
 
 static void
@@ -600,7 +601,7 @@ SHA512Transform(SHA2_CTX *ctx, const uint8_t *blk)
 	ctx->state.s64[7] += h;
 
 }
-#endif	/* !__amd64 */
+#endif	/* !__amd64 || !_KERNEL */
 
 
 /*
@@ -838,7 +839,7 @@ SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len)
 			i = buf_len;
 		}
 
-#if !defined(__amd64)
+#if !defined(__amd64) || !defined(_KERNEL)
 		if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
 			for (; i + buf_limit - 1 < input_len; i += buf_limit) {
 				SHA256Transform(ctx, &input[i]);
@@ -866,7 +867,7 @@ SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len)
 				i += block_count << 7;
 			}
 		}
-#endif	/* !__amd64 */
+#endif	/* !__amd64 || !_KERNEL */
 
 		/*
 		 * general optimization:

module/zfs/Makefile.in

@@ -41,6 +41,7 @@ $(MODULE)-objs += dsl_synctask.o
 $(MODULE)-objs += edonr_zfs.o
 $(MODULE)-objs += fm.o
 $(MODULE)-objs += gzip.o
+$(MODULE)-objs += hkdf.o
 $(MODULE)-objs += lzjb.o
 $(MODULE)-objs += lz4.o
 $(MODULE)-objs += metaslab.o

module/zfs/arc.c

@@ -3155,13 +3155,14 @@ arc_buf_destroy_impl(arc_buf_t *buf)
 			hdr->b_crypt_hdr.b_ebufcnt -= 1;
 
 		/*
-		 * if we have no more encrypted buffers and we've already
+		 * If we have no more encrypted buffers and we've already
 		 * gotten a copy of the decrypted data we can free b_rabd to
 		 * save some space.
 		 */
 		if (hdr->b_crypt_hdr.b_ebufcnt == 0 && HDR_HAS_RABD(hdr) &&
-		    hdr->b_l1hdr.b_pabd != NULL)
+		    hdr->b_l1hdr.b_pabd != NULL && !HDR_IO_IN_PROGRESS(hdr)) {
 			arc_hdr_free_abd(hdr, B_TRUE);
+		}
 	}
 
 	arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
@@ -3716,9 +3717,8 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
 			arc_hdr_free_abd(hdr, B_FALSE);
 		}
 
-		if (HDR_HAS_RABD(hdr)) {
+		if (HDR_HAS_RABD(hdr))
 			arc_hdr_free_abd(hdr, B_TRUE);
-		}
 	}
 
 	ASSERT3P(hdr->b_hash_next, ==, NULL);
@@ -5746,16 +5746,15 @@ arc_read_done(zio_t *zio)
 		callback_cnt++;
 
 		int error = arc_buf_alloc_impl(hdr, zio->io_spa,
-		    zio->io_bookmark.zb_objset, acb->acb_private,
+		    acb->acb_dsobj, acb->acb_private, acb->acb_encrypted,
-		    acb->acb_encrypted, acb->acb_compressed, acb->acb_noauth,
+		    acb->acb_compressed, acb->acb_noauth, no_zio_error,
-		    no_zio_error, &acb->acb_buf);
+		    &acb->acb_buf);
 
 		/*
-		 * assert non-speculative zios didn't fail because an
+		 * Assert non-speculative zios didn't fail because an
 		 * encryption key wasn't loaded
 		 */
-		ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) ||
+		ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) || error == 0);
-		    error == 0 || error != ENOENT);
 
 		/*
 		 * If we failed to decrypt, report an error now (as the zio
@@ -5778,10 +5777,8 @@ arc_read_done(zio_t *zio)
 	}
 	hdr->b_l1hdr.b_acb = NULL;
 	arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
-	if (callback_cnt == 0) {
+	if (callback_cnt == 0)
-		ASSERT(HDR_PREFETCH(hdr) || HDR_HAS_RABD(hdr));
 		ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
-	}
 
 	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
 	    callback_list != NULL);
@@ -5943,6 +5940,9 @@ top:
 				acb->acb_done = done;
 				acb->acb_private = private;
 				acb->acb_compressed = compressed_read;
+				acb->acb_encrypted = encrypted_read;
+				acb->acb_noauth = noauth_read;
+				acb->acb_dsobj = zb->zb_objset;
 				if (pio != NULL)
 					acb->acb_zio_dummy = zio_null(pio,
 					    spa, NULL, NULL, NULL, zio_flags);
@@ -5981,9 +5981,7 @@ top:
 			rc = arc_buf_alloc_impl(hdr, spa, zb->zb_objset,
 			    private, encrypted_read, compressed_read,
 			    noauth_read, B_TRUE, &buf);
-
+			ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) || rc == 0);
-			ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) ||
-			    rc == 0 || rc != ENOENT);
 		} else if (*arc_flags & ARC_FLAG_PREFETCH &&
 		    refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
 			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
@@ -6008,7 +6006,7 @@ top:
 		uint64_t addr = 0;
 		boolean_t devw = B_FALSE;
 		uint64_t size;
-		void *hdr_abd;
+		abd_t *hdr_abd;
 
 		/*
 		 * Gracefully handle a damaged logical block size as a
@@ -6131,6 +6129,7 @@ top:
 		acb->acb_compressed = compressed_read;
 		acb->acb_encrypted = encrypted_read;
 		acb->acb_noauth = noauth_read;
+		acb->acb_dsobj = zb->zb_objset;
 
 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
 		hdr->b_l1hdr.b_acb = acb;
@@ -6698,6 +6697,9 @@ arc_write_ready(zio_t *zio)
 	HDR_SET_PSIZE(hdr, psize);
 	arc_hdr_set_compress(hdr, compress);
 
+	if (zio->io_error != 0 || psize == 0)
+		goto out;
+
 	/*
 	 * Fill the hdr with data. If the buffer is encrypted we have no choice
 	 * but to copy the data into b_radb. If the hdr is compressed, the data
@@ -6713,6 +6715,7 @@ arc_write_ready(zio_t *zio)
 	 * the data into it; otherwise, we share the data directly if we can.
 	 */
 	if (ARC_BUF_ENCRYPTED(buf)) {
+		ASSERT3U(psize, >, 0);
 		ASSERT(ARC_BUF_COMPRESSED(buf));
 		arc_hdr_alloc_abd(hdr, B_TRUE);
 		abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
@@ -6745,6 +6748,7 @@ arc_write_ready(zio_t *zio)
 		arc_share_buf(hdr, buf);
 	}
 
+out:
 	arc_hdr_verify(hdr, bp);
 	spl_fstrans_unmark(cookie);
 }
@@ -7956,9 +7960,15 @@ l2arc_untransform(zio_t *zio, l2arc_read_callback_t *cb)
 	 */
 	ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_INTENT_LOG);
 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
+	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
 
-	/* If the data was encrypted, decrypt it now */
+	/*
-	if (HDR_ENCRYPTED(hdr)) {
+	 * If the data was encrypted, decrypt it now. Note that
+	 * we must check the bp here and not the hdr, since the
+	 * hdr does not have its encryption parameters updated
+	 * until arc_read_done().
+	 */
+	if (BP_IS_ENCRYPTED(bp)) {
 		abd_t *eabd = arc_get_data_abd(hdr,
 		    arc_hdr_size(hdr), hdr);
 
@@ -8084,7 +8094,16 @@ l2arc_read_done(zio_t *zio)
 		 */
 		abd_free(cb->l2rcb_abd);
 		zio->io_size = zio->io_orig_size = arc_hdr_size(hdr);
-		zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;
+
+		if (BP_IS_ENCRYPTED(&cb->l2rcb_bp) &&
+		    (cb->l2rcb_flags & ZIO_FLAG_RAW_ENCRYPT)) {
+			ASSERT(HDR_HAS_RABD(hdr));
+			zio->io_abd = zio->io_orig_abd =
+			    hdr->b_crypt_hdr.b_rabd;
+		} else {
+			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
+			zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;
+		}
 	}
 
 	ASSERT3P(zio->io_abd, !=, NULL);
@@ -8321,7 +8340,7 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
 	boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
 	dsl_crypto_key_t *dck = NULL;
 	uint8_t mac[ZIO_DATA_MAC_LEN] = { 0 };
-	boolean_t no_crypt;
+	boolean_t no_crypt = B_FALSE;
 
 	ASSERT((HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
 	    !HDR_COMPRESSION_ENABLED(hdr)) ||
@@ -8333,6 +8352,15 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
 	 * and copy the data. This may be done to elimiate a depedency on a
 	 * shared buffer or to reallocate the buffer to match asize.
 	 */
+	if (HDR_HAS_RABD(hdr) && asize != psize) {
+		ASSERT3U(size, ==, psize);
+		to_write = abd_alloc_for_io(asize, ismd);
+		abd_copy(to_write, hdr->b_crypt_hdr.b_rabd, size);
+		if (size != asize)
+			abd_zero_off(to_write, size, asize - size);
+		goto out;
+	}
+
 	if ((compress == ZIO_COMPRESS_OFF || HDR_COMPRESSION_ENABLED(hdr)) &&
 	    !HDR_ENCRYPTED(hdr)) {
 		ASSERT3U(size, ==, psize);
@@ -8377,11 +8405,8 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
 		if (ret != 0)
 			goto error;
 
-		if (no_crypt) {
+		if (no_crypt)
-			spa_keystore_dsl_key_rele(spa, dck, FTAG);
+			abd_copy(eabd, to_write, psize);
-			abd_free(eabd);
-			goto out;
-		}
 
 		if (psize != asize)
 			abd_zero_off(eabd, psize, asize - psize);

module/zfs/dbuf.c

@@ -1175,7 +1175,7 @@ dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
 	 * or (if there a no active holders)
 	 *	just null out the current db_data pointer.
 	 */
-	ASSERT(dr->dr_txg >= txg - 2);
+	ASSERT3U(dr->dr_txg, >=, txg - 2);
 	if (db->db_blkid == DMU_BONUS_BLKID) {
 		dnode_t *dn = DB_DNODE(db);
 		int bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots);
@@ -2153,6 +2153,13 @@ dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx)
 
 	if (db->db_state == DB_CACHED &&
 	    refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) {
+		/*
+		 * In practice, we will never have a case where we have an
+		 * encrypted arc buffer while additional holds exist on the
+		 * dbuf. We don't handle this here so we simply assert that
+		 * fact instead.
+		 */
+		ASSERT(!arc_is_encrypted(buf));
 		mutex_exit(&db->db_mtx);
 		(void) dbuf_dirty(db, tx);
 		bcopy(buf->b_data, db->db.db_data, db->db.db_size);
@@ -2168,6 +2175,8 @@ dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx)
 		ASSERT(db->db_buf != NULL);
 		if (dr != NULL && dr->dr_txg == tx->tx_txg) {
 			ASSERT(dr->dt.dl.dr_data == db->db_buf);
+			IMPLY(arc_is_encrypted(buf), dr->dt.dl.dr_raw);
+
 			if (!arc_released(db->db_buf)) {
 				ASSERT(dr->dt.dl.dr_override_state ==
 				    DR_OVERRIDDEN);
@@ -3458,7 +3467,6 @@ dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
 			    dn->dn_type, psize, lsize, compress_type);
 		} else if (compress_type != ZIO_COMPRESS_OFF) {
 			ASSERT3U(type, ==, ARC_BUFC_DATA);
-			int lsize = arc_buf_lsize(*datap);
 			*datap = arc_alloc_compressed_buf(os->os_spa, db,
 			    psize, lsize, compress_type);
 		} else {

module/zfs/dmu.c

@@ -761,7 +761,7 @@ dmu_objset_zfs_unmounting(objset_t *os)
 
 static int
 dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset,
-    uint64_t length)
+    uint64_t length, boolean_t raw)
 {
 	uint64_t object_size;
 	int err;
@@ -844,6 +844,17 @@ dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset,
 		    uint64_t, long_free_dirty_all_txgs, uint64_t, chunk_len,
 		    uint64_t, dmu_tx_get_txg(tx));
 		dnode_free_range(dn, chunk_begin, chunk_len, tx);
+
+		/* if this is a raw free, mark the dirty record as such */
+		if (raw) {
+			dbuf_dirty_record_t *dr = dn->dn_dbuf->db_last_dirty;
+
+			while (dr != NULL && dr->dr_txg > tx->tx_txg)
+				dr = dr->dr_next;
+			if (dr != NULL && dr->dr_txg == tx->tx_txg)
+				dr->dt.dl.dr_raw = B_TRUE;
+		}
+
 		dmu_tx_commit(tx);
 
 		length -= chunk_len;
@@ -861,7 +872,7 @@ dmu_free_long_range(objset_t *os, uint64_t object,
 	err = dnode_hold(os, object, FTAG, &dn);
 	if (err != 0)
 		return (err);
-	err = dmu_free_long_range_impl(os, dn, offset, length);
+	err = dmu_free_long_range_impl(os, dn, offset, length, B_FALSE);
 
 	/*
 	 * It is important to zero out the maxblkid when freeing the entire
@@ -876,8 +887,37 @@ dmu_free_long_range(objset_t *os, uint64_t object,
 	return (err);
 }
 
+/*
+ * This function is equivalent to dmu_free_long_range(), but also
+ * marks the new dirty record as a raw write.
+ */
 int
-dmu_free_long_object(objset_t *os, uint64_t object)
+dmu_free_long_range_raw(objset_t *os, uint64_t object,
+    uint64_t offset, uint64_t length)
+{
+	dnode_t *dn;
+	int err;
+
+	err = dnode_hold(os, object, FTAG, &dn);
+	if (err != 0)
+		return (err);
+	err = dmu_free_long_range_impl(os, dn, offset, length, B_TRUE);
+
+	/*
+	 * It is important to zero out the maxblkid when freeing the entire
+	 * file, so that (a) subsequent calls to dmu_free_long_range_impl()
+	 * will take the fast path, and (b) dnode_reallocate() can verify
+	 * that the entire file has been freed.
+	 */
+	if (err == 0 && offset == 0 && length == DMU_OBJECT_END)
+		dn->dn_maxblkid = 0;
+
+	dnode_rele(dn, FTAG);
+	return (err);
+}
+
+static int
+dmu_free_long_object_impl(objset_t *os, uint64_t object, boolean_t raw)
 {
 	dmu_tx_t *tx;
 	int err;
@@ -893,6 +933,9 @@ dmu_free_long_object(objset_t *os, uint64_t object)
 	err = dmu_tx_assign(tx, TXG_WAIT);
 	if (err == 0) {
 		err = dmu_object_free(os, object, tx);
+		if (err == 0 && raw)
+			VERIFY0(dmu_object_dirty_raw(os, object, tx));
+
 		dmu_tx_commit(tx);
 	} else {
 		dmu_tx_abort(tx);
@@ -901,6 +944,19 @@ dmu_free_long_object(objset_t *os, uint64_t object)
 	return (err);
 }
 
+int
+dmu_free_long_object(objset_t *os, uint64_t object)
+{
+	return (dmu_free_long_object_impl(os, object, B_FALSE));
+}
+
+int
+dmu_free_long_object_raw(objset_t *os, uint64_t object)
+{
+	return (dmu_free_long_object_impl(os, object, B_TRUE));
+}
+
+
 int
 dmu_free_range(objset_t *os, uint64_t object, uint64_t offset,
     uint64_t size, dmu_tx_t *tx)
@@ -1486,13 +1542,6 @@ dmu_return_arcbuf(arc_buf_t *buf)
 	arc_buf_destroy(buf, FTAG);
 }
 
-void
-dmu_assign_arcbuf_impl(dmu_buf_t *handle, arc_buf_t *buf, dmu_tx_t *tx)
-{
-	dmu_buf_impl_t *db = (dmu_buf_impl_t *)handle;
-	dbuf_assign_arcbuf(db, buf, tx);
-}
-
 void
 dmu_convert_to_raw(dmu_buf_t *handle, boolean_t byteorder, const uint8_t *salt,
     const uint8_t *iv, const uint8_t *mac, dmu_tx_t *tx)
@@ -1569,22 +1618,19 @@ dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset,
  * dmu_write().
  */
 void
-dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
+dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset, arc_buf_t *buf,
     dmu_tx_t *tx)
 {
-	dmu_buf_impl_t *dbuf = (dmu_buf_impl_t *)handle;
-	dnode_t *dn;
 	dmu_buf_impl_t *db;
+	objset_t *os = dn->dn_objset;
+	uint64_t object = dn->dn_object;
 	uint32_t blksz = (uint32_t)arc_buf_lsize(buf);
 	uint64_t blkid;
 
-	DB_DNODE_ENTER(dbuf);
-	dn = DB_DNODE(dbuf);
 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
 	blkid = dbuf_whichblock(dn, 0, offset);
 	VERIFY((db = dbuf_hold(dn, blkid, FTAG)) != NULL);
 	rw_exit(&dn->dn_struct_rwlock);
-	DB_DNODE_EXIT(dbuf);
 
 	/*
 	 * We can only assign if the offset is aligned, the arc buf is the
@@ -1594,19 +1640,10 @@ dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
 		dbuf_assign_arcbuf(db, buf, tx);
 		dbuf_rele(db, FTAG);
 	} else {
-		objset_t *os;
-		uint64_t object;
-
 		/* compressed bufs must always be assignable to their dbuf */
 		ASSERT3U(arc_get_compression(buf), ==, ZIO_COMPRESS_OFF);
 		ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED));
 
-		DB_DNODE_ENTER(dbuf);
-		dn = DB_DNODE(dbuf);
-		os = dn->dn_objset;
-		object = dn->dn_object;
-		DB_DNODE_EXIT(dbuf);
-
 		dbuf_rele(db, FTAG);
 		dmu_write(os, object, offset, blksz, buf->b_data, tx);
 		dmu_return_arcbuf(buf);
@@ -1614,6 +1651,17 @@ dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
 	}
 }
 
+void
+dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
+    dmu_tx_t *tx)
+{
+	dmu_buf_impl_t *dbuf = (dmu_buf_impl_t *)handle;
+
+	DB_DNODE_ENTER(dbuf);
+	dmu_assign_arcbuf_by_dnode(DB_DNODE(dbuf), offset, buf, tx);
+	DB_DNODE_EXIT(dbuf);
+}
+
 typedef struct {
 	dbuf_dirty_record_t	*dsa_dr;
 	dmu_sync_cb_t		*dsa_done;
@@ -2424,7 +2472,9 @@ EXPORT_SYMBOL(dmu_buf_rele_array);
 EXPORT_SYMBOL(dmu_prefetch);
 EXPORT_SYMBOL(dmu_free_range);
 EXPORT_SYMBOL(dmu_free_long_range);
+EXPORT_SYMBOL(dmu_free_long_range_raw);
 EXPORT_SYMBOL(dmu_free_long_object);
+EXPORT_SYMBOL(dmu_free_long_object_raw);
 EXPORT_SYMBOL(dmu_read);
 EXPORT_SYMBOL(dmu_read_by_dnode);
 EXPORT_SYMBOL(dmu_write);
@@ -2443,7 +2493,8 @@ EXPORT_SYMBOL(dmu_write_policy);
 EXPORT_SYMBOL(dmu_sync);
 EXPORT_SYMBOL(dmu_request_arcbuf);
 EXPORT_SYMBOL(dmu_return_arcbuf);
-EXPORT_SYMBOL(dmu_assign_arcbuf);
+EXPORT_SYMBOL(dmu_assign_arcbuf_by_dnode);
+EXPORT_SYMBOL(dmu_assign_arcbuf_by_dbuf);
 EXPORT_SYMBOL(dmu_buf_hold);
 EXPORT_SYMBOL(dmu_ot);
 

module/zfs/dmu_objset.c

@@ -706,7 +706,9 @@ dmu_objset_own(const char *name, dmu_objset_type_t type,
 
 	dsl_pool_rele(dp, FTAG);
 
-	if (dmu_objset_userobjspace_upgradable(*osp))
+	/* user accounting requires the dataset to be decrypted */
+	if (dmu_objset_userobjspace_upgradable(*osp) &&
+	    (ds->ds_dir->dd_crypto_obj == 0 || decrypt))
 		dmu_objset_userobjspace_upgrade(*osp);
 
 	return (0);
@@ -932,7 +934,7 @@ dmu_objset_create_impl_dnstats(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 
 	if (blksz == 0)
 		blksz = DNODE_BLOCK_SIZE;
-	if (blksz == 0)
+	if (ibs == 0)
 		ibs = DN_MAX_INDBLKSHIFT;
 
 	if (ds != NULL)
@@ -1096,7 +1098,7 @@ dmu_objset_create_sync(void *arg, dmu_tx_t *tx)
 	}
 
 	/*
-	 * The doca_userfunc() will write out some data that needs to be
+	 * The doca_userfunc() may write out some data that needs to be
 	 * encrypted if the dataset is encrypted (specifically the root
 	 * directory).  This data must be written out before the encryption
 	 * key mapping is removed by dsl_dataset_rele_flags().  Force the
@@ -1107,10 +1109,14 @@ dmu_objset_create_sync(void *arg, dmu_tx_t *tx)
 		dsl_dataset_t *tmpds = NULL;
 		boolean_t need_sync_done = B_FALSE;
 
+		mutex_enter(&ds->ds_lock);
+		ds->ds_owner = FTAG;
+		mutex_exit(&ds->ds_lock);
+
 		rzio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
-		tmpds = txg_list_remove(&dp->dp_dirty_datasets, tx->tx_txg);
+		tmpds = txg_list_remove_this(&dp->dp_dirty_datasets, ds,
+		    tx->tx_txg);
 		if (tmpds != NULL) {
-			ASSERT3P(ds, ==, tmpds);
 			dsl_dataset_sync(ds, rzio, tx);
 			need_sync_done = B_TRUE;
 		}
@@ -1120,9 +1126,9 @@ dmu_objset_create_sync(void *arg, dmu_tx_t *tx)
 		taskq_wait(dp->dp_sync_taskq);
 
 		rzio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
-		tmpds = txg_list_remove(&dp->dp_dirty_datasets, tx->tx_txg);
+		tmpds = txg_list_remove_this(&dp->dp_dirty_datasets, ds,
+		    tx->tx_txg);
 		if (tmpds != NULL) {
-			ASSERT3P(ds, ==, tmpds);
 			dmu_buf_rele(ds->ds_dbuf, ds);
 			dsl_dataset_sync(ds, rzio, tx);
 		}
@@ -1130,6 +1136,10 @@ dmu_objset_create_sync(void *arg, dmu_tx_t *tx)
 
 		if (need_sync_done)
 			dsl_dataset_sync_done(ds, tx);
+
+		mutex_enter(&ds->ds_lock);
+		ds->ds_owner = NULL;
+		mutex_exit(&ds->ds_lock);
 	}
 
 	spa_history_log_internal_ds(ds, "create", tx, "");
@@ -1336,6 +1346,7 @@ dmu_objset_upgrade_stop(objset_t *os)
 		mutex_exit(&os->os_upgrade_lock);
 
 		taskq_cancel_id(os->os_spa->spa_upgrade_taskq, id);
+		txg_wait_synced(os->os_spa->spa_dsl_pool, 0);
 	} else {
 		mutex_exit(&os->os_upgrade_lock);
 	}

module/zfs/dmu_send.c

@@ -517,7 +517,7 @@ dump_dnode(dmu_sendarg_t *dsp, const blkptr_t *bp, uint64_t object,
     dnode_phys_t *dnp)
 {
 	struct drr_object *drro = &(dsp->dsa_drr->drr_u.drr_object);
-	int bonuslen = P2ROUNDUP(dnp->dn_bonuslen, 8);
+	int bonuslen;
 
 	if (object < dsp->dsa_resume_object) {
 		/*
@@ -558,6 +558,8 @@ dump_dnode(dmu_sendarg_t *dsp, const blkptr_t *bp, uint64_t object,
 	    drro->drr_blksz > SPA_OLD_MAXBLOCKSIZE)
 		drro->drr_blksz = SPA_OLD_MAXBLOCKSIZE;
 
+	bonuslen = P2ROUNDUP(dnp->dn_bonuslen, 8);
+
 	if ((dsp->dsa_featureflags & DMU_BACKUP_FEATURE_RAW)) {
 		ASSERT(BP_IS_ENCRYPTED(bp));
 
@@ -571,7 +573,7 @@ dump_dnode(dmu_sendarg_t *dsp, const blkptr_t *bp, uint64_t object,
 
 		/*
 		 * Since we encrypt the entire bonus area, the (raw) part
-		 * beyond the the bonuslen is actually nonzero, so we need
+		 * beyond the bonuslen is actually nonzero, so we need
 		 * to send it.
 		 */
 		if (bonuslen != 0) {
@@ -2062,7 +2064,8 @@ dmu_recv_resume_begin_sync(void *arg, dmu_tx_t *tx)
 	dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_INCONSISTENT;
 
 	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
-	ASSERT(!BP_IS_HOLE(dsl_dataset_get_blkptr(ds)));
+	ASSERT(!BP_IS_HOLE(dsl_dataset_get_blkptr(ds)) ||
+	    drba->drba_cookie->drc_raw);
 	rrw_exit(&ds->ds_bp_rwlock, FTAG);
 
 	drba->drba_cookie->drc_ds = ds;
@@ -2590,7 +2593,11 @@ receive_freeobjects(struct receive_writer_arg *rwa,
 		else if (err != 0)
 			return (err);
 
-		err = dmu_free_long_object(rwa->os, obj);
+		if (rwa->raw)
+			err = dmu_free_long_object_raw(rwa->os, obj);
+		else
+			err = dmu_free_long_object(rwa->os, obj);
+
 		if (err != 0)
 			return (err);
 
@@ -2606,9 +2613,9 @@ noinline static int
 receive_write(struct receive_writer_arg *rwa, struct drr_write *drrw,
     arc_buf_t *abuf)
 {
-	dmu_tx_t *tx;
-	dmu_buf_t *bonus;
 	int err;
+	dmu_tx_t *tx;
+	dnode_t *dn;
 
 	if (drrw->drr_offset + drrw->drr_logical_size < drrw->drr_offset ||
 	    !DMU_OT_IS_VALID(drrw->drr_type))
@@ -2633,7 +2640,6 @@ receive_write(struct receive_writer_arg *rwa, struct drr_write *drrw,
 		return (SET_ERROR(EINVAL));
 
 	tx = dmu_tx_create(rwa->os);
-
 	dmu_tx_hold_write(tx, drrw->drr_object,
 	    drrw->drr_offset, drrw->drr_logical_size);
 	err = dmu_tx_assign(tx, TXG_WAIT);
@@ -2653,10 +2659,9 @@ receive_write(struct receive_writer_arg *rwa, struct drr_write *drrw,
 		    DRR_WRITE_PAYLOAD_SIZE(drrw));
 	}
 
-	/* use the bonus buf to look up the dnode in dmu_assign_arcbuf */
+	VERIFY0(dnode_hold(rwa->os, drrw->drr_object, FTAG, &dn));
-	if (dmu_bonus_hold(rwa->os, drrw->drr_object, FTAG, &bonus) != 0)
+	dmu_assign_arcbuf_by_dnode(dn, drrw->drr_offset, abuf, tx);
-		return (SET_ERROR(EINVAL));
+	dnode_rele(dn, FTAG);
-	dmu_assign_arcbuf(bonus, drrw->drr_offset, abuf, tx);
 
 	/*
 	 * Note: If the receive fails, we want the resume stream to start
@@ -2666,7 +2671,6 @@ receive_write(struct receive_writer_arg *rwa, struct drr_write *drrw,
 	 */
 	save_resume_state(rwa, drrw->drr_object, drrw->drr_offset, tx);
 	dmu_tx_commit(tx);
-	dmu_buf_rele(bonus, FTAG);
 
 	return (0);
 }
@@ -2765,6 +2769,8 @@ receive_write_embedded(struct receive_writer_arg *rwa,
 		return (SET_ERROR(EINVAL));
 	if (drrwe->drr_compression >= ZIO_COMPRESS_FUNCTIONS)
 		return (SET_ERROR(EINVAL));
+	if (rwa->raw)
+		return (SET_ERROR(EINVAL));
 
 	if (drrwe->drr_object > rwa->max_object)
 		rwa->max_object = drrwe->drr_object;
@@ -2839,7 +2845,7 @@ receive_spill(struct receive_writer_arg *rwa, struct drr_spill *drrs,
 	if (db_spill->db_size < drrs->drr_length)
 		VERIFY(0 == dbuf_spill_set_blksz(db_spill,
 		    drrs->drr_length, tx));
-	dmu_assign_arcbuf_impl(db_spill, abuf, tx);
+	dbuf_assign_arcbuf((dmu_buf_impl_t *)db_spill, abuf, tx);
 
 	dmu_buf_rele(db, FTAG);
 	dmu_buf_rele(db_spill, FTAG);
@@ -2864,8 +2870,13 @@ receive_free(struct receive_writer_arg *rwa, struct drr_free *drrf)
 	if (drrf->drr_object > rwa->max_object)
 		rwa->max_object = drrf->drr_object;
 
-	err = dmu_free_long_range(rwa->os, drrf->drr_object,
+	if (rwa->raw) {
-	    drrf->drr_offset, drrf->drr_length);
+		err = dmu_free_long_range_raw(rwa->os, drrf->drr_object,
+		    drrf->drr_offset, drrf->drr_length);
+	} else {
+		err = dmu_free_long_range(rwa->os, drrf->drr_object,
+		    drrf->drr_offset, drrf->drr_length);
+	}
 
 	return (err);
 }
@@ -2948,6 +2959,7 @@ receive_object_range(struct receive_writer_arg *rwa,
 static void
 dmu_recv_cleanup_ds(dmu_recv_cookie_t *drc)
 {
+	dsl_dataset_t *ds = drc->drc_ds;
 	ds_hold_flags_t dsflags = (drc->drc_raw) ? 0 : DS_HOLD_FLAG_DECRYPT;
 
 	/*
@@ -2957,14 +2969,17 @@ dmu_recv_cleanup_ds(dmu_recv_cookie_t *drc)
 	 * that the user accounting code will not attempt to do anything
 	 * after we stopped receiving the dataset.
 	 */
-	txg_wait_synced(drc->drc_ds->ds_dir->dd_pool, 0);
+	txg_wait_synced(ds->ds_dir->dd_pool, 0);
 
-	if (drc->drc_resumable) {
+	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
-		dsl_dataset_disown(drc->drc_ds, dsflags, dmu_recv_tag);
+	if (drc->drc_resumable && !BP_IS_HOLE(dsl_dataset_get_blkptr(ds))) {
+		rrw_exit(&ds->ds_bp_rwlock, FTAG);
+		dsl_dataset_disown(ds, dsflags, dmu_recv_tag);
 	} else {
 		char name[ZFS_MAX_DATASET_NAME_LEN];
-		dsl_dataset_name(drc->drc_ds, name);
+		rrw_exit(&ds->ds_bp_rwlock, FTAG);
-		dsl_dataset_disown(drc->drc_ds, dsflags, dmu_recv_tag);
+		dsl_dataset_name(ds, name);
+		dsl_dataset_disown(ds, dsflags, dmu_recv_tag);
 		(void) dsl_destroy_head(name);
 	}
 }

module/zfs/dmu_traverse.c

@@ -181,7 +181,7 @@ traverse_prefetch_metadata(traverse_data_t *td,
     const blkptr_t *bp, const zbookmark_phys_t *zb)
 {
 	arc_flags_t flags = ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
-	int zio_flags = ZIO_FLAG_CANFAIL;
+	int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE;
 
 	if (!(td->td_flags & TRAVERSE_PREFETCH_METADATA))
 		return;

module/zfs/dsl_crypt.c

@@ -90,9 +90,9 @@ dsl_wrapping_key_free(dsl_wrapping_key_t *wkey)
 
 	if (wkey->wk_key.ck_data) {
 		bzero(wkey->wk_key.ck_data,
-		    BITS_TO_BYTES(wkey->wk_key.ck_length));
+		    CRYPTO_BITS2BYTES(wkey->wk_key.ck_length));
 		kmem_free(wkey->wk_key.ck_data,
-		    BITS_TO_BYTES(wkey->wk_key.ck_length));
+		    CRYPTO_BITS2BYTES(wkey->wk_key.ck_length));
 	}
 
 	refcount_destroy(&wkey->wk_refcnt);
@@ -119,7 +119,7 @@ dsl_wrapping_key_create(uint8_t *wkeydata, zfs_keyformat_t keyformat,
 	}
 
 	wkey->wk_key.ck_format = CRYPTO_KEY_RAW;
-	wkey->wk_key.ck_length = BYTES_TO_BITS(WRAPPING_KEY_LEN);
+	wkey->wk_key.ck_length = CRYPTO_BYTES2BITS(WRAPPING_KEY_LEN);
 	bcopy(wkeydata, wkey->wk_key.ck_data, WRAPPING_KEY_LEN);
 
 	/* initialize the rest of the struct */
@@ -433,7 +433,6 @@ dsl_crypto_can_set_keylocation(const char *dsname, const char *keylocation)
 	int ret = 0;
 	dsl_dir_t *dd = NULL;
 	dsl_pool_t *dp = NULL;
-	dsl_wrapping_key_t *wkey = NULL;
 	uint64_t rddobj;
 
 	/* hold the dsl dir */
@@ -472,16 +471,12 @@ dsl_crypto_can_set_keylocation(const char *dsname, const char *keylocation)
 		goto out;
 	}
 
-	if (wkey != NULL)
-		dsl_wrapping_key_rele(wkey, FTAG);
 	dsl_dir_rele(dd, FTAG);
 	dsl_pool_rele(dp, FTAG);
 
 	return (0);
 
 out:
-	if (wkey != NULL)
-		dsl_wrapping_key_rele(wkey, FTAG);
 	if (dd != NULL)
 		dsl_dir_rele(dd, FTAG);
 	if (dp != NULL)
@@ -728,6 +723,7 @@ spa_keystore_load_wkey(const char *dsname, dsl_crypto_params_t *dcp,
 	dsl_crypto_key_t *dck = NULL;
 	dsl_wrapping_key_t *wkey = dcp->cp_wkey;
 	dsl_pool_t *dp = NULL;
+	uint64_t keyformat, salt, iters;
 
 	/*
 	 * We don't validate the wrapping key's keyformat, salt, or iters
@@ -762,8 +758,36 @@ spa_keystore_load_wkey(const char *dsname, dsl_crypto_params_t *dcp,
 	if (ret != 0)
 		goto error;
 
+	/* initialize the wkey encryption parameters from the DSL Crypto Key */
+	ret = zap_lookup(dp->dp_meta_objset, dd->dd_crypto_obj,
+	    zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 8, 1, &keyformat);
+	if (ret != 0)
+		goto error;
+
+	ret = zap_lookup(dp->dp_meta_objset, dd->dd_crypto_obj,
+	    zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 8, 1, &salt);
+	if (ret != 0)
+		goto error;
+
+	ret = zap_lookup(dp->dp_meta_objset, dd->dd_crypto_obj,
+	    zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 8, 1, &iters);
+	if (ret != 0)
+		goto error;
+
+	ASSERT3U(keyformat, <, ZFS_KEYFORMAT_FORMATS);
+	ASSERT3U(keyformat, !=, ZFS_KEYFORMAT_NONE);
+	IMPLY(keyformat == ZFS_KEYFORMAT_PASSPHRASE, iters != 0);
+	IMPLY(keyformat == ZFS_KEYFORMAT_PASSPHRASE, salt != 0);
+	IMPLY(keyformat != ZFS_KEYFORMAT_PASSPHRASE, iters == 0);
+	IMPLY(keyformat != ZFS_KEYFORMAT_PASSPHRASE, salt == 0);
+
+	wkey->wk_keyformat = keyformat;
+	wkey->wk_salt = salt;
+	wkey->wk_iters = iters;
+
 	/*
-	 * At this point we have verified the key. We can simply cleanup and
+	 * At this point we have verified the wkey and confirmed that it can
+	 * be used to decrypt a DSL Crypto Key. We can simply cleanup and
 	 * return if this is all the user wanted to do.
 	 */
 	if (noop)
@@ -1326,10 +1350,12 @@ spa_keystore_change_key_sync_impl(uint64_t rddobj, uint64_t ddobj,
 		return;
 	}
 
-	/* stop recursing if this dsl dir didn't inherit from the root */
+	/*
+	 * Stop recursing if this dsl dir didn't inherit from the root
+	 * or if this dd is a clone.
+	 */
 	VERIFY0(dsl_dir_get_encryption_root_ddobj(dd, &curr_rddobj));
-
+	if (curr_rddobj != rddobj || dsl_dir_is_clone(dd)) {
-	if (curr_rddobj != rddobj) {
 		dsl_dir_rele(dd, FTAG);
 		return;
 	}
@@ -1355,7 +1381,7 @@ spa_keystore_change_key_sync_impl(uint64_t rddobj, uint64_t ddobj,
 	zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP);
 	za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
 
-	/* Recurse into all child and clone dsl dirs. */
+	/* Recurse into all child dsl dirs. */
 	for (zap_cursor_init(zc, dp->dp_meta_objset,
 	    dsl_dir_phys(dd)->dd_child_dir_zapobj);
 	    zap_cursor_retrieve(zc, za) == 0;
@@ -1365,20 +1391,6 @@ spa_keystore_change_key_sync_impl(uint64_t rddobj, uint64_t ddobj,
 	}
 	zap_cursor_fini(zc);
 
-	for (zap_cursor_init(zc, dp->dp_meta_objset,
-	    dsl_dir_phys(dd)->dd_clones);
-	    zap_cursor_retrieve(zc, za) == 0;
-	    zap_cursor_advance(zc)) {
-		dsl_dataset_t *clone;
-
-		VERIFY0(dsl_dataset_hold_obj(dp,
-		    za->za_first_integer, FTAG, &clone));
-		spa_keystore_change_key_sync_impl(rddobj,
-		    clone->ds_dir->dd_object, new_rddobj, wkey, tx);
-		dsl_dataset_rele(clone, FTAG);
-	}
-	zap_cursor_fini(zc);
-
 	kmem_free(za, sizeof (zap_attribute_t));
 	kmem_free(zc, sizeof (zap_cursor_t));
 
@@ -1831,6 +1843,8 @@ dsl_dataset_create_crypt_sync(uint64_t dsobj, dsl_dir_t *dd,
 		wkey->wk_ddobj = dd->dd_object;
 	}
 
+	ASSERT3P(wkey, !=, NULL);
+
 	/* Create or clone the DSL crypto key and activate the feature */
 	dd->dd_crypto_obj = dsl_crypto_key_create_sync(crypt, wkey, tx);
 	VERIFY0(zap_add(dp->dp_meta_objset, dd->dd_object,
@@ -2191,6 +2205,7 @@ dsl_crypto_populate_key_nvlist(dsl_dataset_t *ds, nvlist_t **nvl_out)
 	uint64_t rddobj;
 	nvlist_t *nvl = NULL;
 	uint64_t dckobj = ds->ds_dir->dd_crypto_obj;
+	dsl_dir_t *rdd = NULL;
 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
 	objset_t *mos = dp->dp_meta_objset;
 	uint64_t crypt = 0, guid = 0, format = 0, iters = 0, salt = 0;
@@ -2209,10 +2224,6 @@ dsl_crypto_populate_key_nvlist(dsl_dataset_t *ds, nvlist_t **nvl_out)
 		goto error;
 
 	/* lookup values from the DSL Crypto Key */
-	ret = dsl_dir_get_encryption_root_ddobj(ds->ds_dir, &rddobj);
-	if (ret != 0)
-		goto error;
-
 	ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_CRYPTO_SUITE, 8, 1,
 	    &crypt);
 	if (ret != 0)
@@ -2242,24 +2253,43 @@ dsl_crypto_populate_key_nvlist(dsl_dataset_t *ds, nvlist_t **nvl_out)
 	if (ret != 0)
 		goto error;
 
-	/* lookup wrapping key properties */
+	/*
-	ret = zap_lookup(dp->dp_meta_objset, dckobj,
+	 * Lookup wrapping key properties. An early version of the code did
-	    zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 8, 1, &format);
+	 * not correctly add these values to the wrapping key or the DSL
+	 * Crypto Key on disk for non encryption roots, so to be safe we
+	 * always take the slightly circuitous route of looking it up from
+	 * the encryption root's key.
+	 */
+	ret = dsl_dir_get_encryption_root_ddobj(ds->ds_dir, &rddobj);
 	if (ret != 0)
 		goto error;
 
+	dsl_pool_config_enter(dp, FTAG);
+
+	ret = dsl_dir_hold_obj(dp, rddobj, NULL, FTAG, &rdd);
+	if (ret != 0)
+		goto error_unlock;
+
+	ret = zap_lookup(dp->dp_meta_objset, rdd->dd_crypto_obj,
+	    zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 8, 1, &format);
+	if (ret != 0)
+		goto error_unlock;
+
 	if (format == ZFS_KEYFORMAT_PASSPHRASE) {
-		ret = zap_lookup(dp->dp_meta_objset, dckobj,
+		ret = zap_lookup(dp->dp_meta_objset, rdd->dd_crypto_obj,
 		    zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 8, 1, &iters);
 		if (ret != 0)
-			goto error;
+			goto error_unlock;
 
-		ret = zap_lookup(dp->dp_meta_objset, dckobj,
+		ret = zap_lookup(dp->dp_meta_objset, rdd->dd_crypto_obj,
 		    zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 8, 1, &salt);
 		if (ret != 0)
-			goto error;
+			goto error_unlock;
 	}
 
+	dsl_dir_rele(rdd, FTAG);
+	dsl_pool_config_exit(dp, FTAG);
+
 	fnvlist_add_uint64(nvl, DSL_CRYPTO_KEY_CRYPTO_SUITE, crypt);
 	fnvlist_add_uint64(nvl, DSL_CRYPTO_KEY_GUID, guid);
 	VERIFY0(nvlist_add_uint8_array(nvl, DSL_CRYPTO_KEY_MASTER_KEY,
@@ -2285,7 +2315,11 @@ dsl_crypto_populate_key_nvlist(dsl_dataset_t *ds, nvlist_t **nvl_out)
 	*nvl_out = nvl;
 	return (0);
 
+error_unlock:
+	dsl_pool_config_exit(dp, FTAG);
 error:
+	if (rdd != NULL)
+		dsl_dir_rele(rdd, FTAG);
 	nvlist_free(nvl);
 
 	*nvl_out = NULL;
@@ -2488,7 +2522,8 @@ spa_do_crypt_mac_abd(boolean_t generate, spa_t *spa, uint64_t dsobj, abd_t *abd,
 		goto error;
 
 	/* perform the hmac */
-	ret = zio_crypt_do_hmac(&dck->dck_key, buf, datalen, digestbuf);
+	ret = zio_crypt_do_hmac(&dck->dck_key, buf, datalen,
+	    digestbuf, ZIO_DATA_MAC_LEN);
 	if (ret != 0)
 		goto error;
 
@@ -2604,8 +2639,7 @@ error:
 		abd_return_buf(cabd, cipherbuf, datalen);
 	}
 
-	if (dck != NULL)
+	spa_keystore_dsl_key_rele(spa, dck, FTAG);
-		spa_keystore_dsl_key_rele(spa, dck, FTAG);
 
 	return (ret);
 }

module/zfs/hkdf.c

@@ -0,0 +1,171 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Datto, Inc. All rights reserved.
+ */
+
+#include <sys/crypto/api.h>
+#include <sys/sha2.h>
+#include <sys/hkdf.h>
+
+static int
+hkdf_sha512_extract(uint8_t *salt, uint_t salt_len, uint8_t *key_material,
+    uint_t km_len, uint8_t *out_buf)
+{
+	int ret;
+	crypto_mechanism_t mech;
+	crypto_key_t key;
+	crypto_data_t input_cd, output_cd;
+
+	/* initialize HMAC mechanism */
+	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
+	mech.cm_param = NULL;
+	mech.cm_param_len = 0;
+
+	/* initialize the salt as a crypto key */
+	key.ck_format = CRYPTO_KEY_RAW;
+	key.ck_length = CRYPTO_BYTES2BITS(salt_len);
+	key.ck_data = salt;
+
+	/* initialize crypto data for the input and output data */
+	input_cd.cd_format = CRYPTO_DATA_RAW;
+	input_cd.cd_offset = 0;
+	input_cd.cd_length = km_len;
+	input_cd.cd_raw.iov_base = (char *)key_material;
+	input_cd.cd_raw.iov_len = input_cd.cd_length;
+
+	output_cd.cd_format = CRYPTO_DATA_RAW;
+	output_cd.cd_offset = 0;
+	output_cd.cd_length = SHA512_DIGEST_LENGTH;
+	output_cd.cd_raw.iov_base = (char *)out_buf;
+	output_cd.cd_raw.iov_len = output_cd.cd_length;
+
+	ret = crypto_mac(&mech, &input_cd, &key, NULL, &output_cd, NULL);
+	if (ret != CRYPTO_SUCCESS)
+		return (SET_ERROR(EIO));
+
+	return (0);
+}
+
+static int
+hkdf_sha512_expand(uint8_t *extract_key, uint8_t *info, uint_t info_len,
+    uint8_t *out_buf, uint_t out_len)
+{
+	int ret;
+	crypto_mechanism_t mech;
+	crypto_context_t ctx;
+	crypto_key_t key;
+	crypto_data_t T_cd, info_cd, c_cd;
+	uint_t i, T_len = 0, pos = 0;
+	uint8_t c;
+	uint_t N = (out_len + SHA512_DIGEST_LENGTH) / SHA512_DIGEST_LENGTH;
+	uint8_t T[SHA512_DIGEST_LENGTH];
+
+	if (N > 255)
+		return (SET_ERROR(EINVAL));
+
+	/* initialize HMAC mechanism */
+	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
+	mech.cm_param = NULL;
+	mech.cm_param_len = 0;
+
+	/* initialize the salt as a crypto key */
+	key.ck_format = CRYPTO_KEY_RAW;
+	key.ck_length = CRYPTO_BYTES2BITS(SHA512_DIGEST_LENGTH);
+	key.ck_data = extract_key;
+
+	/* initialize crypto data for the input and output data */
+	T_cd.cd_format = CRYPTO_DATA_RAW;
+	T_cd.cd_offset = 0;
+	T_cd.cd_raw.iov_base = (char *)T;
+
+	c_cd.cd_format = CRYPTO_DATA_RAW;
+	c_cd.cd_offset = 0;
+	c_cd.cd_length = 1;
+	c_cd.cd_raw.iov_base = (char *)&c;
+	c_cd.cd_raw.iov_len = c_cd.cd_length;
+
+	info_cd.cd_format = CRYPTO_DATA_RAW;
+	info_cd.cd_offset = 0;
+	info_cd.cd_length = info_len;
+	info_cd.cd_raw.iov_base = (char *)info;
+	info_cd.cd_raw.iov_len = info_cd.cd_length;
+
+	for (i = 1; i <= N; i++) {
+		c = i;
+
+		T_cd.cd_length = T_len;
+		T_cd.cd_raw.iov_len = T_cd.cd_length;
+
+		ret = crypto_mac_init(&mech, &key, NULL, &ctx, NULL);
+		if (ret != CRYPTO_SUCCESS)
+			return (SET_ERROR(EIO));
+
+		ret = crypto_mac_update(ctx, &T_cd, NULL);
+		if (ret != CRYPTO_SUCCESS)
+			return (SET_ERROR(EIO));
+
+		ret = crypto_mac_update(ctx, &info_cd, NULL);
+		if (ret != CRYPTO_SUCCESS)
+			return (SET_ERROR(EIO));
+
+		ret = crypto_mac_update(ctx, &c_cd, NULL);
+		if (ret != CRYPTO_SUCCESS)
+			return (SET_ERROR(EIO));
+
+		T_len = SHA512_DIGEST_LENGTH;
+		T_cd.cd_length = T_len;
+		T_cd.cd_raw.iov_len = T_cd.cd_length;
+
+		ret = crypto_mac_final(ctx, &T_cd, NULL);
+		if (ret != CRYPTO_SUCCESS)
+			return (SET_ERROR(EIO));
+
+		bcopy(T, out_buf + pos,
+		    (i != N) ? SHA512_DIGEST_LENGTH : (out_len - pos));
+		pos += SHA512_DIGEST_LENGTH;
+	}
+
+	return (0);
+}
+
+/*
+ * HKDF is designed to be a relatively fast function for deriving keys from a
+ * master key + a salt. We use this function to generate new encryption keys
+ * so as to avoid hitting the cryptographic limits of the underlying
+ * encryption modes. Note that, for the sake of deriving encryption keys, the
+ * info parameter is called the "salt" everywhere else in the code.
+ */
+int
+hkdf_sha512(uint8_t *key_material, uint_t km_len, uint8_t *salt,
+    uint_t salt_len, uint8_t *info, uint_t info_len, uint8_t *output_key,
+    uint_t out_len)
+{
+	int ret;
+	uint8_t extract_key[SHA512_DIGEST_LENGTH];
+
+	ret = hkdf_sha512_extract(salt, salt_len, key_material, km_len,
+	    extract_key);
+	if (ret != 0)
+		return (ret);
+
+	ret = hkdf_sha512_expand(extract_key, info, info_len, output_key,
+	    out_len);
+	if (ret != 0)
+		return (ret);
+
+	return (0);
+}

module/zfs/metaslab.c

@@ -1937,7 +1937,8 @@ metaslab_passivate(metaslab_t *msp, uint64_t weight)
 	 * this metaslab again.  In that case, it had better be empty,
 	 * or we would be leaving space on the table.
 	 */
-	ASSERT(size >= SPA_MINBLOCKSIZE ||
+	ASSERT(!WEIGHT_IS_SPACEBASED(msp->ms_weight) ||
+	    size >= SPA_MINBLOCKSIZE ||
 	    range_tree_space(msp->ms_tree) == 0);
 	ASSERT0(weight & METASLAB_ACTIVE_MASK);
 

module/zfs/zfs_vnops.c

@@ -841,8 +841,8 @@ zfs_write(struct inode *ip, uio_t *uio, int ioflag, cred_t *cr)
 				xuio_stat_wbuf_copied();
 			} else {
 				ASSERT(xuio || tx_bytes == max_blksz);
-				dmu_assign_arcbuf(sa_get_db(zp->z_sa_hdl),
+				dmu_assign_arcbuf_by_dbuf(
-				    woff, abuf, tx);
+				    sa_get_db(zp->z_sa_hdl), woff, abuf, tx);
 			}
 			ASSERT(tx_bytes <= uio->uio_resid);
 			uioskip(uio, tx_bytes);

module/zfs/zil.c

@@ -2115,6 +2115,21 @@ zil_suspend(const char *osname, void **cookiep)
 		return (0);
 	}
 
+	/*
+	 * The ZIL has work to do. Ensure that the associated encryption
+	 * key will remain mapped while we are committing the log by
+	 * grabbing a reference to it. If the key isn't loaded we have no
+	 * choice but to return an error until the wrapping key is loaded.
+	 */
+	if (os->os_encrypted && spa_keystore_create_mapping(os->os_spa,
+	    dmu_objset_ds(os), FTAG) != 0) {
+		zilog->zl_suspend--;
+		mutex_exit(&zilog->zl_lock);
+		dsl_dataset_long_rele(dmu_objset_ds(os), suspend_tag);
+		dsl_dataset_rele(dmu_objset_ds(os), suspend_tag);
+		return (SET_ERROR(EBUSY));
+	}
+
 	zilog->zl_suspending = B_TRUE;
 	mutex_exit(&zilog->zl_lock);
 
@@ -2127,6 +2142,20 @@ zil_suspend(const char *osname, void **cookiep)
 	cv_broadcast(&zilog->zl_cv_suspend);
 	mutex_exit(&zilog->zl_lock);
 
+	if (os->os_encrypted) {
+		/*
+		 * Encrypted datasets need to wait for all data to be
+		 * synced out before removing the mapping.
+		 *
+		 * XXX: Depending on the number of datasets with
+		 * outstanding ZIL data on a given log device, this
+		 * might cause spa_offline_log() to take a long time.
+		 */
+		txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg);
+		VERIFY0(spa_keystore_remove_mapping(os->os_spa,
+		    dmu_objset_id(os), FTAG));
+	}
+
 	if (cookiep == NULL)
 		zil_resume(os);
 	else

module/zfs/zio.c

@@ -2518,13 +2518,14 @@ zio_write_gang_block(zio_t *pio)
 
 		zp.zp_checksum = gio->io_prop.zp_checksum;
 		zp.zp_compress = ZIO_COMPRESS_OFF;
-		zp.zp_encrypt = gio->io_prop.zp_encrypt;
 		zp.zp_type = DMU_OT_NONE;
 		zp.zp_level = 0;
 		zp.zp_copies = gio->io_prop.zp_copies;
 		zp.zp_dedup = B_FALSE;
 		zp.zp_dedup_verify = B_FALSE;
 		zp.zp_nopwrite = B_FALSE;
+		zp.zp_encrypt = gio->io_prop.zp_encrypt;
+		zp.zp_byteorder = gio->io_prop.zp_byteorder;
 		bzero(zp.zp_salt, ZIO_DATA_SALT_LEN);
 		bzero(zp.zp_iv, ZIO_DATA_IV_LEN);
 		bzero(zp.zp_mac, ZIO_DATA_MAC_LEN);

module/zfs/zio_crypt.c

@@ -25,6 +25,7 @@
 #include <sys/zio.h>
 #include <sys/zil.h>
 #include <sys/sha2.h>
+#include <sys/hkdf.h>
 
 /*
  * This file is responsible for handling all of the details of generating
@@ -198,176 +199,6 @@ zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = {
 	{SUN_CKM_AES_GCM,	ZC_TYPE_GCM,	32,	"aes-256-gcm"}
 };
 
-static int
-hkdf_sha512_extract(uint8_t *salt, uint_t salt_len, uint8_t *key_material,
-    uint_t km_len, uint8_t *out_buf)
-{
-	int ret;
-	crypto_mechanism_t mech;
-	crypto_key_t key;
-	crypto_data_t input_cd, output_cd;
-
-	/* initialize HMAC mechanism */
-	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
-	mech.cm_param = NULL;
-	mech.cm_param_len = 0;
-
-	/* initialize the salt as a crypto key */
-	key.ck_format = CRYPTO_KEY_RAW;
-	key.ck_length = BYTES_TO_BITS(salt_len);
-	key.ck_data = salt;
-
-	/* initialize crypto data for the input and output data */
-	input_cd.cd_format = CRYPTO_DATA_RAW;
-	input_cd.cd_offset = 0;
-	input_cd.cd_length = km_len;
-	input_cd.cd_raw.iov_base = (char *)key_material;
-	input_cd.cd_raw.iov_len = input_cd.cd_length;
-
-	output_cd.cd_format = CRYPTO_DATA_RAW;
-	output_cd.cd_offset = 0;
-	output_cd.cd_length = SHA512_DIGEST_LEN;
-	output_cd.cd_raw.iov_base = (char *)out_buf;
-	output_cd.cd_raw.iov_len = output_cd.cd_length;
-
-	ret = crypto_mac(&mech, &input_cd, &key, NULL, &output_cd, NULL);
-	if (ret != CRYPTO_SUCCESS) {
-		ret = SET_ERROR(EIO);
-		goto error;
-	}
-
-	return (0);
-
-error:
-	return (ret);
-}
-
-static int
-hkdf_sha512_expand(uint8_t *extract_key, uint8_t *info, uint_t info_len,
-    uint8_t *out_buf, uint_t out_len)
-{
-	int ret;
-	crypto_mechanism_t mech;
-	crypto_context_t ctx;
-	crypto_key_t key;
-	crypto_data_t T_cd, info_cd, c_cd;
-	uint_t i, T_len = 0, pos = 0;
-	uint8_t c;
-	uint_t N = (out_len + SHA512_DIGEST_LEN) / SHA512_DIGEST_LEN;
-	uint8_t T[SHA512_DIGEST_LEN];
-
-	if (N > 255)
-		return (SET_ERROR(EINVAL));
-
-	/* initialize HMAC mechanism */
-	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
-	mech.cm_param = NULL;
-	mech.cm_param_len = 0;
-
-	/* initialize the salt as a crypto key */
-	key.ck_format = CRYPTO_KEY_RAW;
-	key.ck_length = BYTES_TO_BITS(SHA512_DIGEST_LEN);
-	key.ck_data = extract_key;
-
-	/* initialize crypto data for the input and output data */
-	T_cd.cd_format = CRYPTO_DATA_RAW;
-	T_cd.cd_offset = 0;
-	T_cd.cd_raw.iov_base = (char *)T;
-
-	c_cd.cd_format = CRYPTO_DATA_RAW;
-	c_cd.cd_offset = 0;
-	c_cd.cd_length = 1;
-	c_cd.cd_raw.iov_base = (char *)&c;
-	c_cd.cd_raw.iov_len = c_cd.cd_length;
-
-	info_cd.cd_format = CRYPTO_DATA_RAW;
-	info_cd.cd_offset = 0;
-	info_cd.cd_length = info_len;
-	info_cd.cd_raw.iov_base = (char *)info;
-	info_cd.cd_raw.iov_len = info_cd.cd_length;
-
-	for (i = 1; i <= N; i++) {
-		c = i;
-
-		T_cd.cd_length = T_len;
-		T_cd.cd_raw.iov_len = T_cd.cd_length;
-
-		ret = crypto_mac_init(&mech, &key, NULL, &ctx, NULL);
-		if (ret != CRYPTO_SUCCESS) {
-			ret = SET_ERROR(EIO);
-			goto error;
-		}
-
-		ret = crypto_mac_update(ctx, &T_cd, NULL);
-		if (ret != CRYPTO_SUCCESS) {
-			ret = SET_ERROR(EIO);
-			goto error;
-		}
-
-		ret = crypto_mac_update(ctx, &info_cd, NULL);
-		if (ret != CRYPTO_SUCCESS) {
-			ret = SET_ERROR(EIO);
-			goto error;
-		}
-
-		ret = crypto_mac_update(ctx, &c_cd, NULL);
-		if (ret != CRYPTO_SUCCESS) {
-			ret = SET_ERROR(EIO);
-			goto error;
-		}
-
-		T_len = SHA512_DIGEST_LEN;
-		T_cd.cd_length = T_len;
-		T_cd.cd_raw.iov_len = T_cd.cd_length;
-
-		ret = crypto_mac_final(ctx, &T_cd, NULL);
-		if (ret != CRYPTO_SUCCESS) {
-			ret = SET_ERROR(EIO);
-			goto error;
-		}
-
-		bcopy(T, out_buf + pos,
-		    (i != N) ? SHA512_DIGEST_LEN : (out_len - pos));
-		pos += SHA512_DIGEST_LEN;
-	}
-
-	return (0);
-
-error:
-	return (ret);
-}
-
-/*
- * HKDF is designed to be a relatively fast function for deriving keys from a
- * master key + a salt. We use this function to generate new encryption keys
- * so as to avoid hitting the cryptographic limits of the underlying
- * encryption modes. Note that, for the sake of deriving encryption keys, the
- * info parameter is called the "salt" everywhere else in the code.
- */
-static int
-hkdf_sha512(uint8_t *key_material, uint_t km_len, uint8_t *salt,
-    uint_t salt_len, uint8_t *info, uint_t info_len, uint8_t *output_key,
-    uint_t out_len)
-{
-	int ret;
-	uint8_t extract_key[SHA512_DIGEST_LEN];
-
-	ret = hkdf_sha512_extract(salt, salt_len, key_material, km_len,
-	    extract_key);
-	if (ret != 0)
-		goto error;
-
-	ret = hkdf_sha512_expand(extract_key, info, info_len, output_key,
-	    out_len);
-	if (ret != 0)
-		goto error;
-
-	return (0);
-
-error:
-	return (ret);
-}
-
 void
 zio_crypt_key_destroy(zio_crypt_key_t *key)
 {
@@ -421,11 +252,11 @@ zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key)
 	/* initialize keys for the ICP */
 	key->zk_current_key.ck_format = CRYPTO_KEY_RAW;
 	key->zk_current_key.ck_data = key->zk_current_keydata;
-	key->zk_current_key.ck_length = BYTES_TO_BITS(keydata_len);
+	key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len);
 
 	key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW;
 	key->zk_hmac_key.ck_data = &key->zk_hmac_key;
-	key->zk_hmac_key.ck_length = BYTES_TO_BITS(SHA512_HMAC_KEYLEN);
+	key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
 
 	/*
 	 * Initialize the crypto templates. It's ok if this fails because
@@ -588,10 +419,10 @@ zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key,
 		mech.cm_param_len = sizeof (CK_AES_CCM_PARAMS);
 	} else {
 		gcmp.ulIvLen = ZIO_DATA_IV_LEN;
-		gcmp.ulIvBits = BYTES_TO_BITS(ZIO_DATA_IV_LEN);
+		gcmp.ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN);
 		gcmp.ulAADLen = auth_len;
 		gcmp.pAAD = authbuf;
-		gcmp.ulTagBits = BYTES_TO_BITS(maclen);
+		gcmp.ulTagBits = CRYPTO_BYTES2BITS(maclen);
 		gcmp.pIv = ivbuf;
 
 		mech.cm_param = (char *)(&gcmp);
@@ -748,11 +579,11 @@ zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t guid,
 	/* initialize keys for ICP */
 	key->zk_current_key.ck_format = CRYPTO_KEY_RAW;
 	key->zk_current_key.ck_data = key->zk_current_keydata;
-	key->zk_current_key.ck_length = BYTES_TO_BITS(keydata_len);
+	key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len);
 
 	key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW;
 	key->zk_hmac_key.ck_data = key->zk_hmac_keydata;
-	key->zk_hmac_key.ck_length = BYTES_TO_BITS(SHA512_HMAC_KEYLEN);
+	key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
 
 	/*
 	 * Initialize the crypto templates. It's ok if this fails because
@@ -801,12 +632,14 @@ error:
 
 int
 zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen,
-    uint8_t *digestbuf)
+    uint8_t *digestbuf, uint_t digestlen)
 {
 	int ret;
 	crypto_mechanism_t mech;
 	crypto_data_t in_data, digest_data;
-	uint8_t raw_digestbuf[SHA512_DIGEST_LEN];
+	uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH];
+
+	ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH);
 
 	/* initialize sha512-hmac mechanism and crypto data */
 	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
@@ -822,7 +655,7 @@ zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen,
 
 	digest_data.cd_format = CRYPTO_DATA_RAW;
 	digest_data.cd_offset = 0;
-	digest_data.cd_length = SHA512_DIGEST_LEN;
+	digest_data.cd_length = SHA512_DIGEST_LENGTH;
 	digest_data.cd_raw.iov_base = (char *)raw_digestbuf;
 	digest_data.cd_raw.iov_len = digest_data.cd_length;
 
@@ -834,12 +667,12 @@ zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen,
 		goto error;
 	}
 
-	bcopy(raw_digestbuf, digestbuf, ZIO_DATA_MAC_LEN);
+	bcopy(raw_digestbuf, digestbuf, digestlen);
 
 	return (0);
 
 error:
-	bzero(digestbuf, ZIO_DATA_MAC_LEN);
+	bzero(digestbuf, digestlen);
 	return (ret);
 }
 
@@ -848,9 +681,10 @@ zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data,
     uint_t datalen, uint8_t *ivbuf, uint8_t *salt)
 {
 	int ret;
-	uint8_t digestbuf[SHA512_DIGEST_LEN];
+	uint8_t digestbuf[SHA512_DIGEST_LENGTH];
 
-	ret = zio_crypt_do_hmac(key, data, datalen, digestbuf);
+	ret = zio_crypt_do_hmac(key, data, datalen,
+	    digestbuf, SHA512_DIGEST_LENGTH);
 	if (ret != 0)
 		return (ret);
 
@@ -1212,8 +1046,8 @@ zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen,
 	objset_phys_t *osp = data;
 	uint64_t intval;
 	boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER);
-	uint8_t raw_portable_mac[SHA512_DIGEST_LEN];
+	uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH];
-	uint8_t raw_local_mac[SHA512_DIGEST_LEN];
+	uint8_t raw_local_mac[SHA512_DIGEST_LENGTH];
 
 	/* initialize HMAC mechanism */
 	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
@@ -1267,7 +1101,7 @@ zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen,
 		goto error;
 
 	/* store the final digest in a temporary buffer and copy what we need */
-	cd.cd_length = SHA512_DIGEST_LEN;
+	cd.cd_length = SHA512_DIGEST_LENGTH;
 	cd.cd_raw.iov_base = (char *)raw_portable_mac;
 	cd.cd_raw.iov_len = cd.cd_length;
 
@@ -1284,7 +1118,7 @@ zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen,
 	 * objects are not present, the local MAC is zeroed out.
 	 */
 	if (osp->os_userused_dnode.dn_type == DMU_OT_NONE &&
-	    osp->os_userused_dnode.dn_type == DMU_OT_NONE) {
+	    osp->os_groupused_dnode.dn_type == DMU_OT_NONE) {
 		bzero(local_mac, ZIO_OBJSET_MAC_LEN);
 		return (0);
 	}
@@ -1326,7 +1160,7 @@ zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen,
 		goto error;
 
 	/* store the final digest in a temporary buffer and copy what we need */
-	cd.cd_length = SHA512_DIGEST_LEN;
+	cd.cd_length = SHA512_DIGEST_LENGTH;
 	cd.cd_raw.iov_base = (char *)raw_local_mac;
 	cd.cd_raw.iov_len = cd.cd_length;
 
@@ -1367,7 +1201,7 @@ zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf,
 	blkptr_t *bp;
 	int i, epb = datalen >> SPA_BLKPTRSHIFT;
 	SHA2_CTX ctx;
-	uint8_t digestbuf[SHA512_DIGEST_LEN];
+	uint8_t digestbuf[SHA512_DIGEST_LENGTH];
 
 	/* checksum all of the MACs from the layer below */
 	SHA2Init(SHA512, &ctx);
@@ -1417,8 +1251,8 @@ zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf,
     boolean_t *no_crypt)
 {
 	int ret;
-	uint64_t txtype;
+	uint64_t txtype, lr_len;
-	uint_t nr_src, nr_dst, lr_len, crypt_len;
+	uint_t nr_src, nr_dst, crypt_len;
 	uint_t aad_len = 0, nr_iovecs = 0, total_len = 0;
 	iovec_t *src_iovecs = NULL, *dst_iovecs = NULL;
 	uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp;
@@ -1468,7 +1302,7 @@ zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf,
 	/* allocate the iovec arrays */
 	if (nr_src != 0) {
 		src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP);
-		if (!src_iovecs) {
+		if (src_iovecs == NULL) {
 			ret = SET_ERROR(ENOMEM);
 			goto error;
 		}
@@ -1476,7 +1310,7 @@ zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf,
 
 	if (nr_dst != 0) {
 		dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP);
-		if (!dst_iovecs) {
+		if (dst_iovecs == NULL) {
 			ret = SET_ERROR(ENOMEM);
 			goto error;
 		}
@@ -1515,6 +1349,9 @@ zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf,
 		aadp += sizeof (lr_t);
 		aad_len += sizeof (lr_t);
 
+		ASSERT3P(src_iovecs, !=, NULL);
+		ASSERT3P(dst_iovecs, !=, NULL);
+
 		/*
 		 * If this is a TX_WRITE record we want to encrypt everything
 		 * except the bp if exists. If the bp does exist we want to
@@ -1655,7 +1492,7 @@ zio_crypt_init_uios_dnode(boolean_t encrypt, uint8_t *plainbuf,
 
 	if (nr_src != 0) {
 		src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP);
-		if (!src_iovecs) {
+		if (src_iovecs == NULL) {
 			ret = SET_ERROR(ENOMEM);
 			goto error;
 		}
@@ -1663,7 +1500,7 @@ zio_crypt_init_uios_dnode(boolean_t encrypt, uint8_t *plainbuf,
 
 	if (nr_dst != 0) {
 		dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP);
-		if (!dst_iovecs) {
+		if (dst_iovecs == NULL) {
 			ret = SET_ERROR(ENOMEM);
 			goto error;
 		}
@@ -1729,6 +1566,10 @@ zio_crypt_init_uios_dnode(boolean_t encrypt, uint8_t *plainbuf,
 		if (dnp->dn_type != DMU_OT_NONE &&
 		    DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) &&
 		    dnp->dn_bonuslen != 0) {
+			ASSERT3U(nr_iovecs, <, nr_src);
+			ASSERT3U(nr_iovecs, <, nr_dst);
+			ASSERT3P(src_iovecs, !=, NULL);
+			ASSERT3P(dst_iovecs, !=, NULL);
 			src_iovecs[nr_iovecs].iov_base = DN_BONUS(dnp);
 			src_iovecs[nr_iovecs].iov_len = crypt_len;
 			dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]);
@@ -1942,7 +1783,7 @@ zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, uint8_t *salt,
 
 		tmp_ckey.ck_format = CRYPTO_KEY_RAW;
 		tmp_ckey.ck_data = enc_keydata;
-		tmp_ckey.ck_length = BYTES_TO_BITS(keydata_len);
+		tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len);
 
 		ckey = &tmp_ckey;
 		tmpl = NULL;

rpm/generic/zfs.spec.in

@@ -87,6 +87,8 @@ BuildRequires:  libuuid-devel
 BuildRequires:  libblkid-devel
 BuildRequires:  libudev-devel
 BuildRequires:  libattr-devel
+BuildRequires:  openssl-devel
+Requires:       openssl
 %endif
 %if 0%{?_systemd}
 Requires(post): systemd
@@ -183,7 +185,7 @@ Requires:	fio
 Requires:	acl
 Requires:	sudo
 Requires:	sysstat
-Requires:       rng-tools
+Requires:	rng-tools
 
 %description test
 This package contains test infrastructure and support scripts for

scripts/zloop.sh

@@ -146,6 +146,15 @@ function store_core
 	fi
 }
 
+rngdpid=""
+function on_exit
+{
+	if [ -n "$rngdpid" ]; then
+		kill -9 "$rngdpid"
+	fi
+}
+trap on_exit EXIT
+
 # parse arguments
 # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
 coredir=$DEFAULTCOREDIR
@@ -191,6 +200,9 @@ or_die rm -f ztest.history
 or_die rm -f ztest.ddt
 or_die rm -f ztest.cores
 
+# start rngd in the background so we don't run out of entropy
+or_die read -r rngdpid < <(rngd -f -r /dev/urandom & echo $!)
+
 ztrc=0		# ztest return value
 foundcrashes=0	# number of crashes found so far
 starttime=$(date +%s)

tests/runfiles/linux.run

@@ -405,6 +405,9 @@ tests = ['history_001_pos', 'history_002_pos', 'history_003_pos',
     'history_007_pos', 'history_008_pos', 'history_009_pos',
     'history_010_pos']
 
+[tests/functional/hkdf]
+tests = ['run_hkdf_test']
+
 [tests/functional/inheritance]
 tests = ['inherit_001_pos']
 pre =

tests/zfs-tests/include/commands.cfg

@@ -12,6 +12,7 @@ export SYSTEM_FILES='arp
     attr
     basename
     bc
+    blkid
     blockdev
     bunzip2
     bzcat

tests/zfs-tests/tests/functional/Makefile.am

@@ -21,6 +21,7 @@ SUBDIRS = \
 	grow_pool \
 	grow_replicas \
 	history \
+	hkdf \
 	inheritance \
 	inuse \
 	large_files \

tests/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_008_pos.ksh

@@ -44,6 +44,18 @@
 
 verify_runnable "global"
 
+if is_linux; then
+	# Versions of libblkid older than 2.27.0 will not always detect member
+	# devices of a pool, therefore skip this test case for old versions.
+	currentver="$(blkid -v | tr ',' ' ' | awk '/libblkid/ { print $6 }')"
+	requiredver="2.27.0"
+
+	if [ "$(printf "$requiredver\n$currentver" | sort -V | head -n1)" ==  \
+	    "$currentver" ] && [ "$currentver" != "$requiredver" ]; then
+		log_unsupported "libblkid ($currentver) may not detect pools"
+	fi
+fi
+
 function cleanup
 {
 	if [[ $exported_pool == true ]]; then

tests/zfs-tests/tests/functional/hkdf/.gitignore

@@ -0,0 +1 @@
+hkdf_test

tests/zfs-tests/tests/functional/hkdf/Makefile.am

@@ -0,0 +1,21 @@
+include $(top_srcdir)/config/Rules.am
+AM_CPPFLAGS += -I$(top_srcdir)/include
+AM_CPPFLAGS += -I$(top_srcdir)/lib/libspl/include
+LDADD = $(top_srcdir)/lib/libicp/libicp.la
+LDADD += $(top_srcdir)/lib/libzpool/libzpool.la
+
+AUTOMAKE_OPTIONS = subdir-objects
+
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/hkdf
+
+dist_pkgdata_SCRIPTS = \
+	setup.ksh \
+	cleanup.ksh \
+	run_hkdf_test.ksh
+
+pkgexecdir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/hkdf
+
+pkgexec_PROGRAMS = \
+	hkdf_test
+
+hkdf_test_SOURCES = hkdf_test.c

tests/zfs-tests/tests/functional/hkdf/cleanup.ksh

@@ -0,0 +1,22 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2017 by Datto Inc. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+log_pass

tests/zfs-tests/tests/functional/hkdf/hkdf_test.c

@@ -0,0 +1,235 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Datto, Inc. All rights reserved.
+ */
+
+#include <stdio.h>
+#include <strings.h>
+#include <sys/crypto/icp.h>
+#include <sys/sha2.h>
+#include <sys/hkdf.h>
+
+#define	NELEMS(x)  (sizeof (x) / sizeof ((x)[0]))
+
+/*
+ * Byte arrays are given as char pointers so that they
+ * can be specified as strings.
+ */
+typedef struct hkdf_tv {
+	/* test vector input values */
+	char		*ikm;
+	uint_t		ikm_len;
+	char		*salt;
+	uint_t		salt_len;
+	char		*info;
+	uint_t		info_len;
+	uint_t		okm_len;
+
+	/* expected output */
+	char		*okm;
+} hkdf_tv_t;
+
+/*
+ * XXX Neither NIST nor IETF has published official test
+ * vectors for testing HKDF with SHA512. The following
+ * vectors should be updated if these are ever published.
+ * The current vectors were taken from:
+ * https://www.kullo.net/blog/hkdf-sha-512-test-vectors/
+ */
+static hkdf_tv_t test_vectors[] = {
+	{
+		.ikm =	"\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
+			"\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
+			"\x0b\x0b\x0b\x0b\x0b\x0b",
+		.ikm_len = 22,
+		.salt =	"\x00\x01\x02\x03\x04\x05\x06\x07"
+			"\x08\x09\x0a\x0b\x0c",
+		.salt_len = 13,
+		.info =	"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			"\xf8\xf9",
+		.info_len = 10,
+		.okm_len = 42,
+		.okm =	"\x83\x23\x90\x08\x6c\xda\x71\xfb"
+			"\x47\x62\x5b\xb5\xce\xb1\x68\xe4"
+			"\xc8\xe2\x6a\x1a\x16\xed\x34\xd9"
+			"\xfc\x7f\xe9\x2c\x14\x81\x57\x93"
+			"\x38\xda\x36\x2c\xb8\xd9\xf9\x25"
+			"\xd7\xcb",
+	},
+	{
+		.ikm = 	"\x00\x01\x02\x03\x04\x05\x06\x07"
+			"\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			"\x10\x11\x12\x13\x14\x15\x16\x17"
+			"\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			"\x20\x21\x22\x23\x24\x25\x26\x27"
+			"\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			"\x30\x31\x32\x33\x34\x35\x36\x37"
+			"\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			"\x40\x41\x42\x43\x44\x45\x46\x47"
+			"\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f",
+		.ikm_len = 80,
+		.salt =	"\x60\x61\x62\x63\x64\x65\x66\x67"
+			"\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			"\x70\x71\x72\x73\x74\x75\x76\x77"
+			"\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			"\x80\x81\x82\x83\x84\x85\x86\x87"
+			"\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			"\x90\x91\x92\x93\x94\x95\x96\x97"
+			"\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			"\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			"\xa8\xa9\xaa\xab\xac\xad\xae\xaf",
+		.salt_len = 80,
+		.info =	"\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			"\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			"\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			"\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			"\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			"\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			"\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			"\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.info_len = 80,
+		.okm_len = 42,
+		.okm =	"\xce\x6c\x97\x19\x28\x05\xb3\x46"
+			"\xe6\x16\x1e\x82\x1e\xd1\x65\x67"
+			"\x3b\x84\xf4\x00\xa2\xb5\x14\xb2"
+			"\xfe\x23\xd8\x4c\xd1\x89\xdd\xf1"
+			"\xb6\x95\xb4\x8c\xbd\x1c\x83\x88"
+			"\x44\x11\x37\xb3\xce\x28\xf1\x6a"
+			"\xa6\x4b\xa3\x3b\xa4\x66\xb2\x4d"
+			"\xf6\xcf\xcb\x02\x1e\xcf\xf2\x35"
+			"\xf6\xa2\x05\x6c\xe3\xaf\x1d\xe4"
+			"\x4d\x57\x20\x97\xa8\x50\x5d\x9e"
+			"\x7a\x93",
+	},
+	{
+		.ikm =	"\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
+			"\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
+			"\x0b\x0b\x0b\x0b\x0b\x0b",
+		.ikm_len = 22,
+		.salt =	NULL,
+		.salt_len = 0,
+		.info =	NULL,
+		.info_len = 0,
+		.okm_len = 42,
+		.okm =	"\xf5\xfa\x02\xb1\x82\x98\xa7\x2a"
+			"\x8c\x23\x89\x8a\x87\x03\x47\x2c"
+			"\x6e\xb1\x79\xdc\x20\x4c\x03\x42"
+			"\x5c\x97\x0e\x3b\x16\x4b\xf9\x0f"
+			"\xff\x22\xd0\x48\x36\xd0\xe2\x34"
+			"\x3b\xac",
+	},
+	{
+		.ikm =	"\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
+			"\x0b\x0b\x0b",
+		.ikm_len = 11,
+		.salt =	"\x00\x01\x02\x03\x04\x05\x06\x07"
+			"\x08\x09\x0a\x0b\x0c",
+		.salt_len = 13,
+		.info =	"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			"\xf8\xf9",
+		.info_len = 10,
+		.okm_len = 42,
+		.okm =	"\x74\x13\xe8\x99\x7e\x02\x06\x10"
+			"\xfb\xf6\x82\x3f\x2c\xe1\x4b\xff"
+			"\x01\x87\x5d\xb1\xca\x55\xf6\x8c"
+			"\xfc\xf3\x95\x4d\xc8\xaf\xf5\x35"
+			"\x59\xbd\x5e\x30\x28\xb0\x80\xf7"
+			"\xc0\x68",
+	},
+	{
+		.ikm =	"\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c"
+			"\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c"
+			"\x0c\x0c\x0c\x0c\x0c\x0c",
+		.ikm_len = 22,
+		.salt =	NULL,
+		.salt_len = 0,
+		.info = NULL,
+		.info_len = 0,
+		.okm_len = 42,
+		.okm =	"\x14\x07\xd4\x60\x13\xd9\x8b\xc6"
+			"\xde\xce\xfc\xfe\xe5\x5f\x0f\x90"
+			"\xb0\xc7\xf6\x3d\x68\xeb\x1a\x80"
+			"\xea\xf0\x7e\x95\x3c\xfc\x0a\x3a"
+			"\x52\x40\xa1\x55\xd6\xe4\xda\xa9"
+			"\x65\xbb",
+	},
+};
+
+static void
+hexdump(char *str, uint8_t *src, uint_t len)
+{
+	int i;
+
+	printf("\t%s\t", str);
+	for (i = 0; i < len; i++) {
+		printf("%02x", src[i] & 0xff);
+	}
+	printf("\n");
+}
+
+static int
+run_test(int i, hkdf_tv_t *tv)
+{
+	int ret;
+	uint8_t okey[SHA512_DIGEST_LENGTH];
+
+	printf("TEST %d:\t", i);
+
+	ret = hkdf_sha512((uint8_t *)tv->ikm, tv->ikm_len, (uint8_t *)tv->salt,
+	    tv->salt_len, (uint8_t *)tv->info, tv->info_len, okey, tv->okm_len);
+	if (ret != 0) {
+		printf("HKDF failed with error code %d\n", ret);
+		return (ret);
+	}
+
+	if (bcmp(okey, tv->okm, tv->okm_len) != 0) {
+		printf("Output Mismatch\n");
+		hexdump("Expected:", (uint8_t *)tv->okm, tv->okm_len);
+		hexdump("Actual:  ", okey, tv->okm_len);
+		return (1);
+	}
+
+	printf("Passed\n");
+
+	return (0);
+}
+
+int
+main(int argc, char **argv)
+{
+	int ret, i;
+
+	icp_init();
+
+	for (i = 0; i < NELEMS(test_vectors); i++) {
+		ret = run_test(i, &test_vectors[i]);
+		if (ret != 0)
+			break;
+	}
+
+	icp_fini();
+
+	if (ret == 0) {
+		printf("All tests passed successfully.\n");
+		return (0);
+	} else {
+		printf("Test failed.\n");
+		return (1);
+	}
+}

tests/zfs-tests/tests/functional/hkdf/run_hkdf_test.ksh

@@ -0,0 +1,30 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2017 by Datto Inc. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+#	Call the hkdf_test tool to test ZFS's HKDF implementation against
+#	a few test vectors.
+#
+
+log_assert "Run the tests for the HKDF algorithm."
+
+log_must $STF_SUITE/tests/functional/hkdf/hkdf_test
+
+log_pass "HKDF tests pass."

tests/zfs-tests/tests/functional/hkdf/setup.ksh

@@ -0,0 +1,22 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2017 by Datto Inc. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+log_pass