Remove deduplicated send/receive code

Deduplicated send streams (i.e. `zfs send -D` and `zfs receive` of such
streams) are deprecated.  Deduplicated send streams can be received by
first converting them to non-deduplicated with the `zstream redup`
command.

This commit removes the code for sending and receiving deduplicated send
streams.  `zfs send -D` will now print a warning, ignore the `-D` flag,
and generate a regular (non-deduplicated) send stream.  `zfs receive` of
a deduplicated send stream will print an error message and fail.

The resulting code simplification (especially in the kernel's support
for receiving dedup streams) should help enable future performance
enhancements.

Several new tests are added which leverage `zstream redup`.

Reviewed-by: Paul Dagnelie <pcd@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Matthew Ahrens <mahrens@delphix.com>
Issue #7887
Issue #10117
Issue #10156
Closes #10212

cmd/zfs/zfs_main.c

@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2019 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  * Copyright 2012 Milan Jurik. All rights reserved.
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright (c) 2013 Steven Hartland.  All rights reserved.
@@ -4266,7 +4266,10 @@ zfs_do_send(int argc, char **argv)
 			flags.progress = B_TRUE;
 			break;
 		case 'D':
-			flags.dedup = B_TRUE;
+			(void) fprintf(stderr,
+			    gettext("WARNING: deduplicated send is no "
+			    "longer supported.  A regular,\n"
+			    "non-deduplicated stream will be generated.\n\n"));
 			break;
 		case 'n':
 			flags.dryrun = B_TRUE;
@@ -4333,16 +4336,6 @@ zfs_do_send(int argc, char **argv)
 		}
 	}
 
-	if (flags.dedup) {
-		(void) fprintf(stderr,
-		    gettext("WARNING: deduplicated send is "
-		    "deprecated, and will be removed in a\n"
-		    "future release. (In the future, the flag will be "
-		    "accepted, but a\n"
-		    "regular, non-deduplicated stream will be "
-		    "generated.)\n\n"));
-	}
-
 	if (flags.parsable && flags.verbosity == 0)
 		flags.verbosity = 1;
 
@@ -4351,7 +4344,7 @@ zfs_do_send(int argc, char **argv)
 
 	if (resume_token != NULL) {
 		if (fromname != NULL || flags.replicate || flags.props ||
-		    flags.backup || flags.dedup || flags.holds ||
+		    flags.backup || flags.holds ||
 		    flags.saved || redactbook != NULL) {
 			(void) fprintf(stderr,
 			    gettext("invalid flags combined with -t\n"));
@@ -4375,7 +4368,7 @@ zfs_do_send(int argc, char **argv)
 
 	if (flags.saved) {
 		if (fromname != NULL || flags.replicate || flags.props ||
-		    flags.doall || flags.backup || flags.dedup ||
+		    flags.doall || flags.backup ||
 		    flags.holds || flags.largeblock || flags.embed_data ||
 		    flags.compress || flags.raw || redactbook != NULL) {
 			(void) fprintf(stderr, gettext("incompatible flags "

include/libzfs.h

@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  * Copyright Joyent, Inc.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  * Copyright (c) 2016, Intel Corporation.
@@ -651,8 +651,8 @@ typedef struct sendflags {
 	/* if dataset is a clone, do incremental from its origin */
 	boolean_t fromorigin;
 
-	/* do deduplication */
+	/* field no longer used, maintained for backwards compatibility */
-	boolean_t dedup;
+	boolean_t pad;
 
 	/* send properties (ie, -p) */
 	boolean_t props;

include/libzfs_impl.h

@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2018 Datto Inc.
  */
 
@@ -71,7 +71,6 @@ struct libzfs_handle {
 	int libzfs_pool_iter;
 	char libzfs_chassis_id[256];
 	boolean_t libzfs_prop_debug;
-	boolean_t libzfs_dedup_warning_printed;
 };
 
 #define	ZFSSHARE_MISS	0x01	/* Didn't find entry in cache */

include/sys/dmu.h

@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright 2014 HybridCluster. All rights reserved.
@@ -864,8 +864,6 @@ int dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset,
 int dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset,
     struct arc_buf *buf, dmu_tx_t *tx);
 #define	dmu_assign_arcbuf	dmu_assign_arcbuf_by_dbuf
-void dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset,
-    dmu_buf_t *handle, dmu_tx_t *tx);
 #ifdef HAVE_UIO_ZEROCOPY
 int dmu_xuio_init(struct xuio *uio, int niov);
 void dmu_xuio_fini(struct xuio *uio);

include/sys/dmu_recv.h

@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  */
@@ -53,10 +53,8 @@ typedef struct dmu_recv_cookie {
 	boolean_t drc_raw;
 	boolean_t drc_clone;
 	boolean_t drc_spill;
-	struct avl_tree *drc_guid_to_ds_map;
 	nvlist_t *drc_keynvl;
 	uint64_t drc_fromsnapobj;
-	uint64_t drc_newsnapobj;
 	uint64_t drc_ivset_guid;
 	void *drc_owner;
 	cred_t *drc_cred;
@@ -80,13 +78,11 @@ typedef struct dmu_recv_cookie {
 	objlist_t *drc_ignore_objlist;
 } dmu_recv_cookie_t;
 
-int dmu_recv_begin(char *tofs, char *tosnap, dmu_replay_record_t *drr_begin,
+int dmu_recv_begin(char *, char *, dmu_replay_record_t *,
-    boolean_t force, boolean_t resumable, nvlist_t *localprops,
+    boolean_t, boolean_t, nvlist_t *, nvlist_t *, char *,
-    nvlist_t *hidden_args, char *origin, dmu_recv_cookie_t *drc,
+    dmu_recv_cookie_t *, zfs_file_t *, offset_t *);
-    zfs_file_t *fp, offset_t *voffp);
+int dmu_recv_stream(dmu_recv_cookie_t *, offset_t *);
-int dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd,
+int dmu_recv_end(dmu_recv_cookie_t *, void *);
-    uint64_t *action_handlep, offset_t *voffp);
+boolean_t dmu_objset_is_receiving(objset_t *);
-int dmu_recv_end(dmu_recv_cookie_t *drc, void *owner);
-boolean_t dmu_objset_is_receiving(objset_t *os);
 
 #endif /* _DMU_RECV_H */

include/sys/zfs_ioctl.h

@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
  * Copyright 2016 RackTop Systems.
  * Copyright (c) 2017, Intel Corporation.
  */
@@ -111,8 +111,7 @@ typedef enum drr_headertype {
 /*
  * Mask of all supported backup features
  */
-#define	DMU_BACKUP_FEATURE_MASK	(DMU_BACKUP_FEATURE_DEDUP | \
+#define	DMU_BACKUP_FEATURE_MASK	(DMU_BACKUP_FEATURE_SA_SPILL | \
-    DMU_BACKUP_FEATURE_DEDUPPROPS | DMU_BACKUP_FEATURE_SA_SPILL | \
     DMU_BACKUP_FEATURE_EMBED_DATA | DMU_BACKUP_FEATURE_LZ4 | \
     DMU_BACKUP_FEATURE_RESUMING | DMU_BACKUP_FEATURE_LARGE_BLOCKS | \
     DMU_BACKUP_FEATURE_COMPRESSED | DMU_BACKUP_FEATURE_LARGE_DNODE | \

include/sys/zfs_onexit.h

@@ -21,6 +21,7 @@
 
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020 by Delphix. All rights reserved.
  */
 
 #ifndef	_SYS_ZFS_ONEXIT_H
@@ -54,10 +55,6 @@ extern int zfs_onexit_fd_hold(int fd, minor_t *minorp);
 extern void zfs_onexit_fd_rele(int fd);
 extern int zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
     uint64_t *action_handle);
-extern int zfs_onexit_del_cb(minor_t minor, uint64_t action_handle,
-    boolean_t fire);
-extern int zfs_onexit_cb_data(minor_t minor, uint64_t action_handle,
-    void **data);
 
 #ifdef	__cplusplus
 }

lib/libzfs/libzfs_sendrecv.c

@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright (c) 2012 Pawel Jakub Dawidek <pawel@dawidek.net>.
  * All rights reserved
@@ -73,22 +73,14 @@
 extern void zfs_setprop_error(libzfs_handle_t *, zfs_prop_t, int, char *);
 
 static int zfs_receive_impl(libzfs_handle_t *, const char *, const char *,
-    recvflags_t *, int, const char *, nvlist_t *, avl_tree_t *, char **, int,
+    recvflags_t *, int, const char *, nvlist_t *, avl_tree_t *, char **,
-    uint64_t *, const char *, nvlist_t *);
+    const char *, nvlist_t *);
 static int guid_to_name_redact_snaps(libzfs_handle_t *hdl, const char *parent,
     uint64_t guid, boolean_t bookmark_ok, uint64_t *redact_snap_guids,
     uint64_t num_redact_snaps, char *name);
 static int guid_to_name(libzfs_handle_t *, const char *,
     uint64_t, boolean_t, char *);
 
-static const zio_cksum_t zero_cksum = { { 0 } };
-
-typedef struct dedup_arg {
-	int	inputfd;
-	int	outputfd;
-	libzfs_handle_t  *dedup_hdl;
-} dedup_arg_t;
-
 typedef struct progress_arg {
 	zfs_handle_t *pa_zhp;
 	int pa_fd;
@@ -97,112 +89,6 @@ typedef struct progress_arg {
 	int pa_verbosity;
 } progress_arg_t;
 
-typedef struct dataref {
-	uint64_t ref_guid;
-	uint64_t ref_object;
-	uint64_t ref_offset;
-} dataref_t;
-
-typedef struct dedup_entry {
-	struct dedup_entry	*dde_next;
-	zio_cksum_t dde_chksum;
-	uint64_t dde_prop;
-	dataref_t dde_ref;
-} dedup_entry_t;
-
-#define	MAX_DDT_PHYSMEM_PERCENT		20
-#define	SMALLEST_POSSIBLE_MAX_DDT_MB		128
-
-typedef struct dedup_table {
-	dedup_entry_t	**dedup_hash_array;
-	umem_cache_t	*ddecache;
-	uint64_t	max_ddt_size;  /* max dedup table size in bytes */
-	uint64_t	cur_ddt_size;  /* current dedup table size in bytes */
-	uint64_t	ddt_count;
-	int		numhashbits;
-	boolean_t	ddt_full;
-} dedup_table_t;
-
-static int
-high_order_bit(uint64_t n)
-{
-	int count;
-
-	for (count = 0; n != 0; count++)
-		n >>= 1;
-	return (count);
-}
-
-static size_t
-ssread(void *buf, size_t len, FILE *stream)
-{
-	size_t outlen;
-
-	if ((outlen = fread(buf, len, 1, stream)) == 0)
-		return (0);
-
-	return (outlen);
-}
-
-static void
-ddt_hash_append(libzfs_handle_t *hdl, dedup_table_t *ddt, dedup_entry_t **ddepp,
-    zio_cksum_t *cs, uint64_t prop, dataref_t *dr)
-{
-	dedup_entry_t	*dde;
-
-	if (ddt->cur_ddt_size >= ddt->max_ddt_size) {
-		if (ddt->ddt_full == B_FALSE) {
-			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-			    "Dedup table full.  Deduplication will continue "
-			    "with existing table entries"));
-			ddt->ddt_full = B_TRUE;
-		}
-		return;
-	}
-
-	if ((dde = umem_cache_alloc(ddt->ddecache, UMEM_DEFAULT))
-	    != NULL) {
-		assert(*ddepp == NULL);
-		dde->dde_next = NULL;
-		dde->dde_chksum = *cs;
-		dde->dde_prop = prop;
-		dde->dde_ref = *dr;
-		*ddepp = dde;
-		ddt->cur_ddt_size += sizeof (dedup_entry_t);
-		ddt->ddt_count++;
-	}
-}
-
-/*
- * Using the specified dedup table, do a lookup for an entry with
- * the checksum cs.  If found, return the block's reference info
- * in *dr. Otherwise, insert a new entry in the dedup table, using
- * the reference information specified by *dr.
- *
- * return value:  true - entry was found
- *		  false - entry was not found
- */
-static boolean_t
-ddt_update(libzfs_handle_t *hdl, dedup_table_t *ddt, zio_cksum_t *cs,
-    uint64_t prop, dataref_t *dr)
-{
-	uint32_t hashcode;
-	dedup_entry_t **ddepp;
-
-	hashcode = BF64_GET(cs->zc_word[0], 0, ddt->numhashbits);
-
-	for (ddepp = &(ddt->dedup_hash_array[hashcode]); *ddepp != NULL;
-	    ddepp = &((*ddepp)->dde_next)) {
-		if (ZIO_CHECKSUM_EQUAL(((*ddepp)->dde_chksum), *cs) &&
-		    (*ddepp)->dde_prop == prop) {
-			*dr = (*ddepp)->dde_ref;
-			return (B_TRUE);
-		}
-	}
-	ddt_hash_append(hdl, ddt, ddepp, cs, prop, dr);
-	return (B_FALSE);
-}
-
 static int
 dump_record(dmu_replay_record_t *drr, void *payload, int payload_len,
     zio_cksum_t *zc, int outfd)
@@ -228,274 +114,6 @@ dump_record(dmu_replay_record_t *drr, void *payload, int payload_len,
 	return (0);
 }
 
-/*
- * This function is started in a separate thread when the dedup option
- * has been requested.  The main send thread determines the list of
- * snapshots to be included in the send stream and makes the ioctl calls
- * for each one.  But instead of having the ioctl send the output to the
- * the output fd specified by the caller of zfs_send()), the
- * ioctl is told to direct the output to a pipe, which is read by the
- * alternate thread running THIS function.  This function does the
- * dedup'ing by:
- *  1. building a dedup table (the DDT)
- *  2. doing checksums on each data block and inserting a record in the DDT
- *  3. looking for matching checksums, and
- *  4.  sending a DRR_WRITE_BYREF record instead of a write record whenever
- *      a duplicate block is found.
- * The output of this function then goes to the output fd requested
- * by the caller of zfs_send().
- */
-static void *
-cksummer(void *arg)
-{
-	dedup_arg_t *dda = arg;
-	char *buf = zfs_alloc(dda->dedup_hdl, SPA_MAXBLOCKSIZE);
-	dmu_replay_record_t thedrr = { 0 };
-	dmu_replay_record_t *drr = &thedrr;
-	FILE *ofp;
-	int outfd;
-	dedup_table_t ddt;
-	zio_cksum_t stream_cksum;
-	uint64_t numbuckets;
-
-#ifdef _ILP32
-	ddt.max_ddt_size = SMALLEST_POSSIBLE_MAX_DDT_MB << 20;
-#else
-	uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
-	ddt.max_ddt_size =
-	    MAX((physmem * MAX_DDT_PHYSMEM_PERCENT) / 100,
-	    SMALLEST_POSSIBLE_MAX_DDT_MB << 20);
-#endif
-
-	numbuckets = ddt.max_ddt_size / (sizeof (dedup_entry_t));
-
-	/*
-	 * numbuckets must be a power of 2.  Increase number to
-	 * a power of 2 if necessary.
-	 */
-	if (!ISP2(numbuckets))
-		numbuckets = 1ULL << high_order_bit(numbuckets);
-
-	ddt.dedup_hash_array = calloc(numbuckets, sizeof (dedup_entry_t *));
-	ddt.ddecache = umem_cache_create("dde", sizeof (dedup_entry_t), 0,
-	    NULL, NULL, NULL, NULL, NULL, 0);
-	ddt.cur_ddt_size = numbuckets * sizeof (dedup_entry_t *);
-	ddt.numhashbits = high_order_bit(numbuckets) - 1;
-	ddt.ddt_full = B_FALSE;
-
-	outfd = dda->outputfd;
-	ofp = fdopen(dda->inputfd, "r");
-	while (ssread(drr, sizeof (*drr), ofp) != 0) {
-
-		/*
-		 * kernel filled in checksum, we are going to write same
-		 * record, but need to regenerate checksum.
-		 */
-		if (drr->drr_type != DRR_BEGIN) {
-			bzero(&drr->drr_u.drr_checksum.drr_checksum,
-			    sizeof (drr->drr_u.drr_checksum.drr_checksum));
-		}
-
-		switch (drr->drr_type) {
-		case DRR_BEGIN:
-		{
-			struct drr_begin *drrb = &drr->drr_u.drr_begin;
-			int fflags;
-			int sz = 0;
-			ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0);
-
-			ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC);
-
-			/* set the DEDUP feature flag for this stream */
-			fflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
-			fflags |= (DMU_BACKUP_FEATURE_DEDUP |
-			    DMU_BACKUP_FEATURE_DEDUPPROPS);
-			DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, fflags);
-
-			if (drr->drr_payloadlen != 0) {
-				sz = drr->drr_payloadlen;
-
-				if (sz > SPA_MAXBLOCKSIZE) {
-					buf = zfs_realloc(dda->dedup_hdl, buf,
-					    SPA_MAXBLOCKSIZE, sz);
-				}
-				(void) ssread(buf, sz, ofp);
-				if (ferror(stdin))
-					perror("fread");
-			}
-			if (dump_record(drr, buf, sz, &stream_cksum,
-			    outfd) != 0)
-				goto out;
-			break;
-		}
-
-		case DRR_END:
-		{
-			struct drr_end *drre = &drr->drr_u.drr_end;
-			/* use the recalculated checksum */
-			drre->drr_checksum = stream_cksum;
-			if (dump_record(drr, NULL, 0, &stream_cksum,
-			    outfd) != 0)
-				goto out;
-			break;
-		}
-
-		case DRR_OBJECT:
-		{
-			struct drr_object *drro = &drr->drr_u.drr_object;
-			if (drro->drr_bonuslen > 0) {
-				(void) ssread(buf,
-				    DRR_OBJECT_PAYLOAD_SIZE(drro), ofp);
-			}
-			if (dump_record(drr, buf, DRR_OBJECT_PAYLOAD_SIZE(drro),
-			    &stream_cksum, outfd) != 0)
-				goto out;
-			break;
-		}
-
-		case DRR_SPILL:
-		{
-			struct drr_spill *drrs = &drr->drr_u.drr_spill;
-			(void) ssread(buf, DRR_SPILL_PAYLOAD_SIZE(drrs), ofp);
-			if (dump_record(drr, buf, DRR_SPILL_PAYLOAD_SIZE(drrs),
-			    &stream_cksum, outfd) != 0)
-				goto out;
-			break;
-		}
-
-		case DRR_FREEOBJECTS:
-		{
-			if (dump_record(drr, NULL, 0, &stream_cksum,
-			    outfd) != 0)
-				goto out;
-			break;
-		}
-
-		case DRR_WRITE:
-		{
-			struct drr_write *drrw = &drr->drr_u.drr_write;
-			dataref_t	dataref;
-			uint64_t	payload_size;
-
-			payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw);
-			(void) ssread(buf, payload_size, ofp);
-
-			/*
-			 * Use the existing checksum if it's dedup-capable,
-			 * else calculate a SHA256 checksum for it.
-			 */
-
-			if (ZIO_CHECKSUM_EQUAL(drrw->drr_key.ddk_cksum,
-			    zero_cksum) ||
-			    !DRR_IS_DEDUP_CAPABLE(drrw->drr_flags)) {
-				SHA2_CTX ctx;
-				zio_cksum_t tmpsha256;
-
-				SHA2Init(SHA256, &ctx);
-				SHA2Update(&ctx, buf, payload_size);
-				SHA2Final(&tmpsha256, &ctx);
-
-				drrw->drr_key.ddk_cksum.zc_word[0] =
-				    BE_64(tmpsha256.zc_word[0]);
-				drrw->drr_key.ddk_cksum.zc_word[1] =
-				    BE_64(tmpsha256.zc_word[1]);
-				drrw->drr_key.ddk_cksum.zc_word[2] =
-				    BE_64(tmpsha256.zc_word[2]);
-				drrw->drr_key.ddk_cksum.zc_word[3] =
-				    BE_64(tmpsha256.zc_word[3]);
-				drrw->drr_checksumtype = ZIO_CHECKSUM_SHA256;
-				drrw->drr_flags |= DRR_CHECKSUM_DEDUP;
-			}
-
-			dataref.ref_guid = drrw->drr_toguid;
-			dataref.ref_object = drrw->drr_object;
-			dataref.ref_offset = drrw->drr_offset;
-
-			if (ddt_update(dda->dedup_hdl, &ddt,
-			    &drrw->drr_key.ddk_cksum, drrw->drr_key.ddk_prop,
-			    &dataref)) {
-				dmu_replay_record_t wbr_drr = {0};
-				struct drr_write_byref *wbr_drrr =
-				    &wbr_drr.drr_u.drr_write_byref;
-
-				/* block already present in stream */
-				wbr_drr.drr_type = DRR_WRITE_BYREF;
-
-				wbr_drrr->drr_object = drrw->drr_object;
-				wbr_drrr->drr_offset = drrw->drr_offset;
-				wbr_drrr->drr_length = drrw->drr_logical_size;
-				wbr_drrr->drr_toguid = drrw->drr_toguid;
-				wbr_drrr->drr_refguid = dataref.ref_guid;
-				wbr_drrr->drr_refobject =
-				    dataref.ref_object;
-				wbr_drrr->drr_refoffset =
-				    dataref.ref_offset;
-
-				wbr_drrr->drr_checksumtype =
-				    drrw->drr_checksumtype;
-				wbr_drrr->drr_flags = drrw->drr_flags;
-				wbr_drrr->drr_key.ddk_cksum =
-				    drrw->drr_key.ddk_cksum;
-				wbr_drrr->drr_key.ddk_prop =
-				    drrw->drr_key.ddk_prop;
-
-				if (dump_record(&wbr_drr, NULL, 0,
-				    &stream_cksum, outfd) != 0)
-					goto out;
-			} else {
-				/* block not previously seen */
-				if (dump_record(drr, buf, payload_size,
-				    &stream_cksum, outfd) != 0)
-					goto out;
-			}
-			break;
-		}
-
-		case DRR_WRITE_EMBEDDED:
-		{
-			struct drr_write_embedded *drrwe =
-			    &drr->drr_u.drr_write_embedded;
-			(void) ssread(buf,
-			    P2ROUNDUP((uint64_t)drrwe->drr_psize, 8), ofp);
-			if (dump_record(drr, buf,
-			    P2ROUNDUP((uint64_t)drrwe->drr_psize, 8),
-			    &stream_cksum, outfd) != 0)
-				goto out;
-			break;
-		}
-
-		case DRR_FREE:
-		{
-			if (dump_record(drr, NULL, 0, &stream_cksum,
-			    outfd) != 0)
-				goto out;
-			break;
-		}
-
-		case DRR_OBJECT_RANGE:
-		{
-			if (dump_record(drr, NULL, 0, &stream_cksum,
-			    outfd) != 0)
-				goto out;
-			break;
-		}
-
-		default:
-			(void) fprintf(stderr, "INVALID record type 0x%x\n",
-			    drr->drr_type);
-			/* should never happen, so assert */
-			assert(B_FALSE);
-		}
-	}
-out:
-	umem_cache_destroy(ddt.ddecache);
-	free(ddt.dedup_hash_array);
-	free(buf);
-	(void) fclose(ofp);
-
-	return (NULL);
-}
-
 /*
  * Routines for dealing with the AVL tree of fs-nvlists
  */
@@ -2478,7 +2096,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 	int spa_version;
 	pthread_t tid = 0;
 	int pipefd[2];
-	dedup_arg_t dda = { 0 };
 	int featureflags = 0;
 	FILE *fout;
 
@@ -2502,33 +2119,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 	if (flags->holds)
 		featureflags |= DMU_BACKUP_FEATURE_HOLDS;
 
-	/*
-	 * Start the dedup thread if this is a dedup stream. We do not bother
-	 * doing this if this a raw send of an encrypted dataset with dedup off
-	 * because normal encrypted blocks won't dedup.
-	 */
-	if (flags->dedup && !flags->dryrun && !(flags->raw &&
-	    zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION) != ZIO_CRYPT_OFF &&
-	    zfs_prop_get_int(zhp, ZFS_PROP_DEDUP) == ZIO_CHECKSUM_OFF)) {
-		featureflags |= (DMU_BACKUP_FEATURE_DEDUP |
-		    DMU_BACKUP_FEATURE_DEDUPPROPS);
-		if ((err = socketpair(AF_UNIX, SOCK_STREAM, 0, pipefd)) != 0) {
-			zfs_error_aux(zhp->zfs_hdl, strerror(errno));
-			return (zfs_error(zhp->zfs_hdl, EZFS_PIPEFAILED,
-			    errbuf));
-		}
-		dda.outputfd = outfd;
-		dda.inputfd = pipefd[1];
-		dda.dedup_hdl = zhp->zfs_hdl;
-		if ((err = pthread_create(&tid, NULL, cksummer, &dda)) != 0) {
-			(void) close(pipefd[0]);
-			(void) close(pipefd[1]);
-			zfs_error_aux(zhp->zfs_hdl, strerror(errno));
-			return (zfs_error(zhp->zfs_hdl,
-			    EZFS_THREADCREATEFAILED, errbuf));
-		}
-	}
-
 	if (flags->replicate || flags->doall || flags->props ||
 	    flags->holds || flags->backup) {
 		char full_tosnap_name[ZFS_MAX_DATASET_NAME_LEN];
@@ -2706,34 +2296,6 @@ err_out:
 	return (err);
 }
 
-static int
-get_dedup_fd(zfs_handle_t *zhp, dedup_arg_t *dda, int fd, pthread_t *tid,
-    int *outfd)
-{
-	int pipefd[2];
-	char errbuf[1024];
-	int err;
-	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
-	    "warning: cannot send '%s'"), zhp->zfs_name);
-	if ((err = socketpair(AF_UNIX, SOCK_STREAM, 0, pipefd)) != 0) {
-		zfs_error_aux(zhp->zfs_hdl, strerror(errno));
-		return (zfs_error(zhp->zfs_hdl, EZFS_PIPEFAILED,
-		    errbuf));
-	}
-	dda->outputfd = fd;
-	dda->inputfd = pipefd[1];
-	dda->dedup_hdl = zhp->zfs_hdl;
-	if ((err = pthread_create(tid, NULL, cksummer, dda)) != 0) {
-		(void) close(pipefd[0]);
-		(void) close(pipefd[1]);
-		zfs_error_aux(zhp->zfs_hdl, strerror(err));
-		return (zfs_error(zhp->zfs_hdl, EZFS_THREADCREATEFAILED,
-		    errbuf));
-	}
-	*outfd = pipefd[0];
-	return (0);
-}
-
 zfs_handle_t *
 name_to_dir_handle(libzfs_handle_t *hdl, const char *snapname)
 {
@@ -2819,9 +2381,8 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
 	libzfs_handle_t *hdl = zhp->zfs_hdl;
 	char *name = zhp->zfs_name;
 	int orig_fd = fd;
-	pthread_t ddtid, ptid;
+	pthread_t ptid;
 	progress_arg_t pa = { 0 };
-	dedup_arg_t dda = { 0 };
 
 	char errbuf[1024];
 	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
@@ -2914,16 +2475,6 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
 	if (flags->dryrun)
 		return (0);
 
-	/*
-	 * If deduplication is requested, spawn a thread that will deduplicate
-	 * the data coming out of the kernel.
-	 */
-	if (flags->dedup) {
-		err = get_dedup_fd(zhp, &dda, fd, &ddtid, &fd);
-		if (err != 0)
-			return (err);
-	}
-
 	/*
 	 * If progress reporting is requested, spawn a new thread to poll
 	 * ZFS_IOC_SEND_PROGRESS at a regular interval.
@@ -2939,11 +2490,6 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
 		    send_progress_thread, &pa);
 		if (err != 0) {
 			zfs_error_aux(zhp->zfs_hdl, strerror(errno));
-			if (flags->dedup) {
-				(void) pthread_cancel(ddtid);
-				(void) close(fd);
-				(void) pthread_join(ddtid, NULL);
-			}
 			return (zfs_error(zhp->zfs_hdl,
 			    EZFS_THREADCREATEFAILED, errbuf));
 		}
@@ -2966,12 +2512,6 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
 			return (zfs_standard_error(hdl, error, errbuf));
 		}
 	}
-	if (flags->dedup) {
-		if (err != 0)
-			(void) pthread_cancel(ddtid);
-		(void) close(fd);
-		(void) pthread_join(ddtid, NULL);
-	}
 
 	if (flags->props || flags->holds || flags->backup) {
 		/* Write the final end record. */
@@ -3965,8 +3505,7 @@ doagain:
 static int
 zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname,
     recvflags_t *flags, dmu_replay_record_t *drr, zio_cksum_t *zc,
-    char **top_zfs, int cleanup_fd, uint64_t *action_handlep,
+    char **top_zfs, nvlist_t *cmdprops)
-    nvlist_t *cmdprops)
 {
 	nvlist_t *stream_nv = NULL;
 	avl_tree_t *stream_avl = NULL;
@@ -4143,8 +3682,7 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname,
 		 * recv_skip() and return 0).
 		 */
 		error = zfs_receive_impl(hdl, destname, NULL, flags, fd,
-		    sendfs, stream_nv, stream_avl, top_zfs, cleanup_fd,
+		    sendfs, stream_nv, stream_avl, top_zfs, sendsnap, cmdprops);
-		    action_handlep, sendsnap, cmdprops);
 		if (error == ENODATA) {
 			error = 0;
 			break;
@@ -4530,8 +4068,8 @@ static int
 zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
     const char *originsnap, recvflags_t *flags, dmu_replay_record_t *drr,
     dmu_replay_record_t *drr_noswap, const char *sendfs, nvlist_t *stream_nv,
-    avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd,
+    avl_tree_t *stream_avl, char **top_zfs,
-    uint64_t *action_handlep, const char *finalsnap, nvlist_t *cmdprops)
+    const char *finalsnap, nvlist_t *cmdprops)
 {
 	time_t begin_time;
 	int ioctl_err, ioctl_errno, err;
@@ -4741,24 +4279,16 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 			(void) printf("found clone origin %s\n", origin);
 	}
 
-	if (!hdl->libzfs_dedup_warning_printed &&
+	if ((DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
-	    (DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
 	    DMU_BACKUP_FEATURE_DEDUP)) {
 		(void) fprintf(stderr,
-		    gettext("WARNING: This is a deduplicated send stream.  "
+		    gettext("ERROR: \"zfs receive\" no longer supports "
-		    "The ability to send and\n"
+		    "deduplicated send streams.  Use\n"
-		    "receive deduplicated send streams is deprecated.  "
+		    "the \"zstream redup\" command to convert this stream "
-		    "In the future, the\n"
+		    "to a regular,\n"
-		    "ability to receive a deduplicated send stream with "
+		    "non-deduplicated stream.\n"));
-		    "\"zfs receive\" will be\n"
+		err = zfs_error(hdl, EZFS_NOTSUP, errbuf);
-		    "removed. However, in the future, a utility will be "
+		goto out;
-		    "provided to convert a\n"
-		    "deduplicated send stream to a regular "
-		    "(non-deduplicated) stream. This\n"
-		    "future utility will require that the send stream be "
-		    "located in a\n"
-		    "seek-able file, rather than provided by a pipe.\n\n"));
-		hdl->libzfs_dedup_warning_printed = B_TRUE;
 	}
 
 	boolean_t resuming = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
@@ -5103,8 +4633,8 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 
 	err = ioctl_err = lzc_receive_with_cmdprops(destsnap, rcvprops,
 	    oxprops, wkeydata, wkeylen, origin, flags->force, flags->resumable,
-	    raw, infd, drr_noswap, cleanup_fd, &read_bytes, &errflags,
+	    raw, infd, drr_noswap, -1, &read_bytes, &errflags,
-	    action_handlep, &prop_errors);
+	    NULL, &prop_errors);
 	ioctl_errno = ioctl_err;
 	prop_errflags = errflags;
 
@@ -5435,8 +4965,8 @@ zfs_receive_checkprops(libzfs_handle_t *hdl, nvlist_t *props,
 static int
 zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap,
     const char *originsnap, recvflags_t *flags, int infd, const char *sendfs,
-    nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd,
+    nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs,
-    uint64_t *action_handlep, const char *finalsnap, nvlist_t *cmdprops)
+    const char *finalsnap, nvlist_t *cmdprops)
 {
 	int err;
 	dmu_replay_record_t drr, drr_noswap;
@@ -5546,12 +5076,12 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap,
 		}
 		return (zfs_receive_one(hdl, infd, tosnap, originsnap, flags,
 		    &drr, &drr_noswap, sendfs, stream_nv, stream_avl, top_zfs,
-		    cleanup_fd, action_handlep, finalsnap, cmdprops));
+		    finalsnap, cmdprops));
 	} else {
 		assert(DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
 		    DMU_COMPOUNDSTREAM);
 		return (zfs_receive_package(hdl, infd, tosnap, flags, &drr,
-		    &zcksum, top_zfs, cleanup_fd, action_handlep, cmdprops));
+		    &zcksum, top_zfs, cmdprops));
 	}
 }
 
@@ -5568,8 +5098,6 @@ zfs_receive(libzfs_handle_t *hdl, const char *tosnap, nvlist_t *props,
 {
 	char *top_zfs = NULL;
 	int err;
-	int cleanup_fd;
-	uint64_t action_handle = 0;
 	struct stat sb;
 	char *originsnap = NULL;
 
@@ -5595,13 +5123,8 @@ zfs_receive(libzfs_handle_t *hdl, const char *tosnap, nvlist_t *props,
 			return (err);
 	}
 
-	cleanup_fd = open(ZFS_DEV, O_RDWR);
-	VERIFY(cleanup_fd >= 0);
-
 	err = zfs_receive_impl(hdl, tosnap, originsnap, flags, infd, NULL, NULL,
-	    stream_avl, &top_zfs, cleanup_fd, &action_handle, NULL, props);
+	    stream_avl, &top_zfs, NULL, props);
-
-	VERIFY(0 == close(cleanup_fd));
 
 	if (err == 0 && !flags->nomount && flags->domount && top_zfs) {
 		zfs_handle_t *zhp = NULL;

lib/libzfs_core/libzfs_core.c

@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  * Copyright (c) 2017 Datto Inc.
  * Copyright 2017 RackTop Systems.
@@ -783,9 +783,8 @@ static int
 recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops,
     uint8_t *wkeydata, uint_t wkeylen, const char *origin, boolean_t force,
     boolean_t resumable, boolean_t raw, int input_fd,
-    const dmu_replay_record_t *begin_record, int cleanup_fd,
+    const dmu_replay_record_t *begin_record, uint64_t *read_bytes,
-    uint64_t *read_bytes, uint64_t *errflags, uint64_t *action_handle,
+    uint64_t *errflags, nvlist_t **errors)
-    nvlist_t **errors)
 {
 	dmu_replay_record_t drr;
 	char fsname[MAXPATHLEN];
@@ -868,12 +867,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops,
 		if (resumable)
 			fnvlist_add_boolean(innvl, "resumable");
 
-		if (cleanup_fd >= 0)
-			fnvlist_add_int32(innvl, "cleanup_fd", cleanup_fd);
-
-		if (action_handle != NULL)
-			fnvlist_add_uint64(innvl, "action_handle",
-			    *action_handle);
 
 		error = lzc_ioctl(ZFS_IOC_RECV_NEW, fsname, innvl, &outnvl);
 
@@ -885,10 +878,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops,
 			error = nvlist_lookup_uint64(outnvl, "error_flags",
 			    errflags);
 
-		if (error == 0 && action_handle != NULL)
-			error = nvlist_lookup_uint64(outnvl, "action_handle",
-			    action_handle);
-
 		if (error == 0 && errors != NULL) {
 			nvlist_t *nvl;
 			error = nvlist_lookup_nvlist(outnvl, "errors", &nvl);
@@ -931,12 +920,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops,
 		zc.zc_cleanup_fd = -1;
 		zc.zc_action_handle = 0;
 
-		if (cleanup_fd >= 0)
-			zc.zc_cleanup_fd = cleanup_fd;
-
-		if (action_handle != NULL)
-			zc.zc_action_handle = *action_handle;
-
 		zc.zc_nvlist_dst_size = 128 * 1024;
 		zc.zc_nvlist_dst = (uint64_t)(uintptr_t)
 		    malloc(zc.zc_nvlist_dst_size);
@@ -951,9 +934,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops,
 			if (errflags != NULL)
 				*errflags = zc.zc_obj;
 
-			if (action_handle != NULL)
-				*action_handle = zc.zc_action_handle;
-
 			if (errors != NULL)
 				VERIFY0(nvlist_unpack(
 				    (void *)(uintptr_t)zc.zc_nvlist_dst,
@@ -986,7 +966,7 @@ lzc_receive(const char *snapname, nvlist_t *props, const char *origin,
     boolean_t force, boolean_t raw, int fd)
 {
 	return (recv_impl(snapname, props, NULL, NULL, 0, origin, force,
-	    B_FALSE, raw, fd, NULL, -1, NULL, NULL, NULL, NULL));
+	    B_FALSE, raw, fd, NULL, NULL, NULL, NULL));
 }
 
 /*
@@ -1000,7 +980,7 @@ lzc_receive_resumable(const char *snapname, nvlist_t *props, const char *origin,
     boolean_t force, boolean_t raw, int fd)
 {
 	return (recv_impl(snapname, props, NULL, NULL, 0, origin, force,
-	    B_TRUE, raw, fd, NULL, -1, NULL, NULL, NULL, NULL));
+	    B_TRUE, raw, fd, NULL, NULL, NULL, NULL));
 }
 
 /*
@@ -1023,7 +1003,7 @@ lzc_receive_with_header(const char *snapname, nvlist_t *props,
 		return (EINVAL);
 
 	return (recv_impl(snapname, props, NULL, NULL, 0, origin, force,
-	    resumable, raw, fd, begin_record, -1, NULL, NULL, NULL, NULL));
+	    resumable, raw, fd, begin_record, NULL, NULL, NULL));
 }
 
 /*
@@ -1039,9 +1019,7 @@ lzc_receive_with_header(const char *snapname, nvlist_t *props,
  * The 'errflags' value will contain zprop_errflags_t flags which are
  * used to describe any failures.
  *
- * The 'action_handle' is used to pass the handle for this guid/ds mapping.
+ * The 'action_handle' and 'cleanup_fd' are no longer used, and are ignored.
- * It should be set to zero on first call and will contain an updated handle
- * on success, it should be passed in subsequent calls.
  *
  * The 'errors' nvlist contains an entry for each unapplied received
  * property.  Callers are responsible for freeing this nvlist.
@@ -1053,8 +1031,8 @@ int lzc_receive_one(const char *snapname, nvlist_t *props,
     nvlist_t **errors)
 {
 	return (recv_impl(snapname, props, NULL, NULL, 0, origin, force,
-	    resumable, raw, input_fd, begin_record, cleanup_fd, read_bytes,
+	    resumable, raw, input_fd, begin_record,
-	    errflags, action_handle, errors));
+	    read_bytes, errflags, errors));
 }
 
 /*
@@ -1073,8 +1051,8 @@ int lzc_receive_with_cmdprops(const char *snapname, nvlist_t *props,
     nvlist_t **errors)
 {
 	return (recv_impl(snapname, props, cmdprops, wkeydata, wkeylen, origin,
-	    force, resumable, raw, input_fd, begin_record, cleanup_fd,
+	    force, resumable, raw, input_fd, begin_record,
-	    read_bytes, errflags, action_handle, errors));
+	    read_bytes, errflags, errors));
 }
 
 /*

man/man8/zfs-receive.8

@@ -105,17 +105,11 @@ destroyed by using the
 .Nm zfs Cm destroy Fl d
 command.
 .Pp
-Deduplicated send streams can be generated by using the
+The ability to send and receive deduplicated send streams has been removed.
-.Nm zfs Cm send Fl D
+However, a deduplicated send stream created with older software can be converted
+to a regular (non-deduplicated) stream by using the
+.Nm zstream Cm redup
 command.
-\fBThe ability to send and receive deduplicated send streams is deprecated.\fR
-In the future, the ability to receive a deduplicated send stream with
-.Nm zfs Cm receive
-will be removed.
-However, in the future, a utility will be provided to convert a
-deduplicated send stream to a regular (non-deduplicated) stream.
-This future utility will require that the send stream be located in a
-seek-able file, rather than provided by a pipe.
 .Pp
 If
 .Fl o Em property Ns = Ns Ar value
@@ -378,3 +372,4 @@ deleting its saved partially received state.
 .El
 .Sh SEE ALSO
 .Xr zfs-send 8
+.Xr zstream 8

man/man8/zfs-send.8

@@ -86,21 +86,9 @@ The output can be redirected to a file or to a different system
 By default, a full stream is generated.
 .Bl -tag -width "-D"
 .It Fl D, -dedup
-Generate a deduplicated stream.
+Deduplicated send is no longer supported.
-\fBDeduplicated send is deprecated and will be removed in a future release.\fR
+This flag is accepted for backwards compatibility, but a regular,
-(In the future, the flag will be accepted but a regular, non-deduplicated
+non-deduplicated stream will be generated.
-stream will be generated.)
-Blocks which would have been sent multiple times in the send stream will only be
-sent once.
-The receiving system must also support this feature to receive a deduplicated
-stream.
-This flag can be used regardless of the dataset's
-.Sy dedup
-property, but performance will be much better if the filesystem uses a
-dedup-capable checksum
-.Po for example,
-.Sy sha256
-.Pc .
 .It Fl I Ar snapshot
 Generate a stream package that sends all intermediary snapshots from the first
 snapshot to the second snapshot.

module/zfs/dmu.c

@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright (c) 2016, Nexenta Systems, Inc. All rights reserved.
@@ -1560,56 +1560,6 @@ dmu_return_arcbuf(arc_buf_t *buf)
 	arc_buf_destroy(buf, FTAG);
 }
 
-void
-dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset,
-    dmu_buf_t *handle, dmu_tx_t *tx)
-{
-	dmu_buf_t *dst_handle;
-	dmu_buf_impl_t *dstdb;
-	dmu_buf_impl_t *srcdb = (dmu_buf_impl_t *)handle;
-	dmu_object_type_t type;
-	arc_buf_t *abuf;
-	uint64_t datalen;
-	boolean_t byteorder;
-	uint8_t salt[ZIO_DATA_SALT_LEN];
-	uint8_t iv[ZIO_DATA_IV_LEN];
-	uint8_t mac[ZIO_DATA_MAC_LEN];
-
-	ASSERT3P(srcdb->db_buf, !=, NULL);
-
-	/* hold the db that we want to write to */
-	VERIFY0(dmu_buf_hold(os, object, offset, FTAG, &dst_handle,
-	    DMU_READ_NO_DECRYPT));
-	dstdb = (dmu_buf_impl_t *)dst_handle;
-	datalen = arc_buf_size(srcdb->db_buf);
-
-	DB_DNODE_ENTER(dstdb);
-	type = DB_DNODE(dstdb)->dn_type;
-	DB_DNODE_EXIT(dstdb);
-
-	/* allocated an arc buffer that matches the type of srcdb->db_buf */
-	if (arc_is_encrypted(srcdb->db_buf)) {
-		arc_get_raw_params(srcdb->db_buf, &byteorder, salt, iv, mac);
-		abuf = arc_loan_raw_buf(os->os_spa, dmu_objset_id(os),
-		    byteorder, salt, iv, mac, type,
-		    datalen, arc_buf_lsize(srcdb->db_buf),
-		    arc_get_compression(srcdb->db_buf));
-	} else {
-		/* we won't get a compressed db back from dmu_buf_hold() */
-		ASSERT3U(arc_get_compression(srcdb->db_buf),
-		    ==, ZIO_COMPRESS_OFF);
-		abuf = arc_loan_buf(os->os_spa,
-		    DMU_OT_IS_METADATA(type), datalen);
-	}
-
-	ASSERT3U(datalen, ==, arc_buf_size(abuf));
-
-	/* copy the data to the new buffer and assign it to the dstdb */
-	bcopy(srcdb->db_buf->b_data, abuf->b_data, datalen);
-	dbuf_assign_arcbuf(dstdb, abuf, tx);
-	dmu_buf_rele(dst_handle, FTAG);
-}
-
 /*
  * When possible directly assign passed loaned arc buffer to a dbuf.
  * If this is not possible copy the contents of passed arc buf via

module/zfs/dmu_recv.c

@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
- * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  * Copyright 2014 HybridCluster. All rights reserved.
  * Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
@@ -101,8 +101,6 @@ struct receive_writer_arg {
 	boolean_t done;
 
 	int err;
-	/* A map from guid to dataset to help handle dedup'd streams. */
-	avl_tree_t *guid_to_ds_map;
 	boolean_t resumable;
 	boolean_t raw;   /* DMU_BACKUP_FEATURE_RAW set */
 	boolean_t spill; /* DRR_FLAG_SPILL_BLOCK set */
@@ -123,13 +121,6 @@ struct receive_writer_arg {
 	boolean_t or_byteorder;
 };
 
-typedef struct guid_map_entry {
-	uint64_t	guid;
-	boolean_t	raw;
-	objset_t	*gme_os;
-	avl_node_t	avlnode;
-} guid_map_entry_t;
-
 typedef struct dmu_recv_begin_arg {
 	const char *drba_origin;
 	dmu_recv_cookie_t *drba_cookie;
@@ -1212,38 +1203,6 @@ dmu_recv_begin(char *tofs, char *tosnap, dmu_replay_record_t *drr_begin,
 	return (err);
 }
 
-static int
-guid_compare(const void *arg1, const void *arg2)
-{
-	const guid_map_entry_t *gmep1 = (const guid_map_entry_t *)arg1;
-	const guid_map_entry_t *gmep2 = (const guid_map_entry_t *)arg2;
-
-	return (TREE_CMP(gmep1->guid, gmep2->guid));
-}
-
-static void
-free_guid_map_onexit(void *arg)
-{
-	avl_tree_t *ca = arg;
-	void *cookie = NULL;
-	guid_map_entry_t *gmep;
-
-	while ((gmep = avl_destroy_nodes(ca, &cookie)) != NULL) {
-		ds_hold_flags_t dsflags = DS_HOLD_FLAG_DECRYPT;
-
-		if (gmep->raw) {
-			gmep->gme_os->os_raw_receive = B_FALSE;
-			dsflags &= ~DS_HOLD_FLAG_DECRYPT;
-		}
-
-		dsl_dataset_disown(gmep->gme_os->os_dsl_dataset,
-		    dsflags, gmep);
-		kmem_free(gmep, sizeof (guid_map_entry_t));
-	}
-	avl_destroy(ca);
-	kmem_free(ca, sizeof (avl_tree_t));
-}
-
 static int
 receive_read(dmu_recv_cookie_t *drc, int len, void *buf)
 {
@@ -1845,81 +1804,6 @@ receive_process_write_record(struct receive_writer_arg *rwa,
 	return (EAGAIN);
 }
 
-/*
- * Handle a DRR_WRITE_BYREF record.  This record is used in dedup'ed
- * streams to refer to a copy of the data that is already on the
- * system because it came in earlier in the stream.  This function
- * finds the earlier copy of the data, and uses that copy instead of
- * data from the stream to fulfill this write.
- */
-noinline static int
-receive_write_byref(struct receive_writer_arg *rwa,
-    struct drr_write_byref *drrwbr)
-{
-	dmu_tx_t *tx;
-	int err;
-	guid_map_entry_t gmesrch;
-	guid_map_entry_t *gmep;
-	avl_index_t where;
-	objset_t *ref_os = NULL;
-	int flags = DMU_READ_PREFETCH;
-	dmu_buf_t *dbp;
-
-	if (drrwbr->drr_offset + drrwbr->drr_length < drrwbr->drr_offset)
-		return (SET_ERROR(EINVAL));
-
-	/*
-	 * If the GUID of the referenced dataset is different from the
-	 * GUID of the target dataset, find the referenced dataset.
-	 */
-	if (drrwbr->drr_toguid != drrwbr->drr_refguid) {
-		gmesrch.guid = drrwbr->drr_refguid;
-		if ((gmep = avl_find(rwa->guid_to_ds_map, &gmesrch,
-		    &where)) == NULL) {
-			return (SET_ERROR(EINVAL));
-		}
-		ref_os = gmep->gme_os;
-	} else {
-		ref_os = rwa->os;
-	}
-
-	if (drrwbr->drr_object > rwa->max_object)
-		rwa->max_object = drrwbr->drr_object;
-
-	if (rwa->raw)
-		flags |= DMU_READ_NO_DECRYPT;
-
-	/* may return either a regular db or an encrypted one */
-	err = dmu_buf_hold(ref_os, drrwbr->drr_refobject,
-	    drrwbr->drr_refoffset, FTAG, &dbp, flags);
-	if (err != 0)
-		return (err);
-
-	tx = dmu_tx_create(rwa->os);
-
-	dmu_tx_hold_write(tx, drrwbr->drr_object,
-	    drrwbr->drr_offset, drrwbr->drr_length);
-	err = dmu_tx_assign(tx, TXG_WAIT);
-	if (err != 0) {
-		dmu_tx_abort(tx);
-		return (err);
-	}
-
-	if (rwa->raw) {
-		dmu_copy_from_buf(rwa->os, drrwbr->drr_object,
-		    drrwbr->drr_offset, dbp, tx);
-	} else {
-		dmu_write(rwa->os, drrwbr->drr_object,
-		    drrwbr->drr_offset, drrwbr->drr_length, dbp->db_data, tx);
-	}
-	dmu_buf_rele(dbp, FTAG);
-
-	/* See comment in restore_write. */
-	save_resume_state(rwa, drrwbr->drr_object, drrwbr->drr_offset, tx);
-	dmu_tx_commit(tx);
-	return (0);
-}
-
 static int
 receive_write_embedded(struct receive_writer_arg *rwa,
     struct drr_write_embedded *drrwe, void *data)
@@ -2607,13 +2491,6 @@ receive_process_record(struct receive_writer_arg *rwa,
 		}
 		break;
 	}
-	case DRR_WRITE_BYREF:
-	{
-		struct drr_write_byref *drrwbr =
-		    &rrd->header.drr_u.drr_write_byref;
-		err = receive_write_byref(rwa, drrwbr);
-		break;
-	}
 	case DRR_WRITE_EMBEDDED:
 	{
 		struct drr_write_embedded *drrwe =
@@ -2754,8 +2631,7 @@ resume_check(dmu_recv_cookie_t *drc, nvlist_t *begin_nvl)
  * NB: callers *must* call dmu_recv_end() if this succeeds.
  */
 int
-dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd,
+dmu_recv_stream(dmu_recv_cookie_t *drc, offset_t *voffp)
-    uint64_t *action_handlep, offset_t *voffp)
 {
 	int err = 0;
 	struct receive_writer_arg *rwa = kmem_zalloc(sizeof (*rwa), KM_SLEEP);
@@ -2779,41 +2655,6 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd,
 	ASSERT0(drc->drc_os->os_encrypted &&
 	    (drc->drc_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA));
 
-	/* if this stream is dedup'ed, set up the avl tree for guid mapping */
-	if (drc->drc_featureflags & DMU_BACKUP_FEATURE_DEDUP) {
-		minor_t minor;
-
-		if (cleanup_fd == -1) {
-			err = SET_ERROR(EBADF);
-			goto out;
-		}
-		err = zfs_onexit_fd_hold(cleanup_fd, &minor);
-		if (err != 0) {
-			cleanup_fd = -1;
-			goto out;
-		}
-
-		if (*action_handlep == 0) {
-			rwa->guid_to_ds_map =
-			    kmem_alloc(sizeof (avl_tree_t), KM_SLEEP);
-			avl_create(rwa->guid_to_ds_map, guid_compare,
-			    sizeof (guid_map_entry_t),
-			    offsetof(guid_map_entry_t, avlnode));
-			err = zfs_onexit_add_cb(minor,
-			    free_guid_map_onexit, rwa->guid_to_ds_map,
-			    action_handlep);
-			if (err != 0)
-				goto out;
-		} else {
-			err = zfs_onexit_cb_data(minor, *action_handlep,
-			    (void **)&rwa->guid_to_ds_map);
-			if (err != 0)
-				goto out;
-		}
-
-		drc->drc_guid_to_ds_map = rwa->guid_to_ds_map;
-	}
-
 	/* handle DSL encryption key payload */
 	if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) {
 		nvlist_t *keynvl = NULL;
@@ -2980,9 +2821,6 @@ out:
 
 	kmem_free(rwa, sizeof (*rwa));
 	nvlist_free(drc->drc_begin_nvl);
-	if ((drc->drc_featureflags & DMU_BACKUP_FEATURE_DEDUP) &&
-	    (cleanup_fd != -1))
-		zfs_onexit_fd_rele(cleanup_fd);
 
 	if (err != 0) {
 		/*
@@ -3083,6 +2921,7 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
 	dmu_recv_cookie_t *drc = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	boolean_t encrypted = drc->drc_ds->ds_dir->dd_crypto_obj != 0;
+	uint64_t newsnapobj;
 
 	spa_history_log_internal_ds(drc->drc_ds, "finish receiving",
 	    tx, "snap=%s", drc->drc_tosnap);
@@ -3148,7 +2987,7 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
 		dsl_dataset_phys(origin_head)->ds_flags &=
 		    ~DS_FLAG_INCONSISTENT;
 
-		drc->drc_newsnapobj =
+		newsnapobj =
 		    dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
 
 		dsl_dataset_rele(origin_head, FTAG);
@@ -3188,7 +3027,7 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
 			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
 			    DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, tx);
 		}
-		drc->drc_newsnapobj =
+		newsnapobj =
 		    dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
 	}
 
@@ -3203,9 +3042,9 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
 	 * value.
 	 */
 	if (drc->drc_raw && drc->drc_ivset_guid != 0) {
-		dmu_object_zapify(dp->dp_meta_objset, drc->drc_newsnapobj,
+		dmu_object_zapify(dp->dp_meta_objset, newsnapobj,
 		    DMU_OT_DSL_DATASET, tx);
-		VERIFY0(zap_update(dp->dp_meta_objset, drc->drc_newsnapobj,
+		VERIFY0(zap_update(dp->dp_meta_objset, newsnapobj,
 		    DS_FIELD_IVSET_GUID, sizeof (uint64_t), 1,
 		    &drc->drc_ivset_guid, tx));
 	}
@@ -3226,54 +3065,6 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
 	drc->drc_ds = NULL;
 }
 
-static int
-add_ds_to_guidmap(const char *name, avl_tree_t *guid_map, uint64_t snapobj,
-    boolean_t raw)
-{
-	dsl_pool_t *dp;
-	dsl_dataset_t *snapds;
-	guid_map_entry_t *gmep;
-	objset_t *os;
-	ds_hold_flags_t dsflags = (raw) ? 0 : DS_HOLD_FLAG_DECRYPT;
-	int err;
-
-	ASSERT(guid_map != NULL);
-
-	err = dsl_pool_hold(name, FTAG, &dp);
-	if (err != 0)
-		return (err);
-	gmep = kmem_alloc(sizeof (*gmep), KM_SLEEP);
-	err = dsl_dataset_own_obj(dp, snapobj, dsflags, gmep, &snapds);
-
-	if (err == 0) {
-		err = dmu_objset_from_ds(snapds, &os);
-		if (err != 0) {
-			dsl_dataset_disown(snapds, dsflags, FTAG);
-			dsl_pool_rele(dp, FTAG);
-			kmem_free(gmep, sizeof (*gmep));
-			return (err);
-		}
-		/*
-		 * If this is a deduplicated raw send stream, we need
-		 * to make sure that we can still read raw blocks from
-		 * earlier datasets in the stream, so we set the
-		 * os_raw_receive flag now.
-		 */
-		if (raw)
-			os->os_raw_receive = B_TRUE;
-
-		gmep->raw = raw;
-		gmep->guid = dsl_dataset_phys(snapds)->ds_guid;
-		gmep->gme_os = os;
-		avl_add(guid_map, gmep);
-	} else {
-		kmem_free(gmep, sizeof (*gmep));
-	}
-
-	dsl_pool_rele(dp, FTAG);
-	return (err);
-}
-
 static int dmu_recv_end_modified_blocks = 3;
 
 static int
@@ -3325,12 +3116,6 @@ dmu_recv_end(dmu_recv_cookie_t *drc, void *owner)
 		    drc->drc_tofs, drc->drc_tosnap);
 		zvol_create_minor(snapname);
 		kmem_strfree(snapname);
-
-		if (drc->drc_guid_to_ds_map != NULL) {
-			(void) add_ds_to_guidmap(drc->drc_tofs,
-			    drc->drc_guid_to_ds_map,
-			    drc->drc_newsnapobj, drc->drc_raw);
-		}
 	}
 	return (error);
 }

module/zfs/zfs_ioctl.c

@@ -27,7 +27,7 @@
  * Copyright (c) 2014, 2016 Joyent, Inc. All rights reserved.
  * Copyright 2016 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
- * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  * Copyright (c) 2014 Integros [integros.com]
@@ -4765,9 +4765,9 @@ static boolean_t zfs_ioc_recv_inject_err;
 static int
 zfs_ioc_recv_impl(char *tofs, char *tosnap, char *origin, nvlist_t *recvprops,
     nvlist_t *localprops, nvlist_t *hidden_args, boolean_t force,
-    boolean_t resumable, int input_fd, dmu_replay_record_t *begin_record,
+    boolean_t resumable, int input_fd,
-    int cleanup_fd, uint64_t *read_bytes, uint64_t *errflags,
+    dmu_replay_record_t *begin_record, uint64_t *read_bytes,
-    uint64_t *action_handle, nvlist_t **errors)
+    uint64_t *errflags, nvlist_t **errors)
 {
 	dmu_recv_cookie_t drc;
 	int error = 0;
@@ -4896,7 +4896,7 @@ zfs_ioc_recv_impl(char *tofs, char *tosnap, char *origin, nvlist_t *recvprops,
 		nvlist_free(xprops);
 	}
 
-	error = dmu_recv_stream(&drc, cleanup_fd, action_handle, &off);
+	error = dmu_recv_stream(&drc, &off);
 
 	if (error == 0) {
 		zfsvfs_t *zfsvfs = NULL;
@@ -5088,13 +5088,10 @@ out:
  * zc_cookie		file descriptor to recv from
  * zc_begin_record	the BEGIN record of the stream (not byteswapped)
  * zc_guid		force flag
- * zc_cleanup_fd	cleanup-on-exit file descriptor
- * zc_action_handle	handle for this guid/ds mapping (or zero on first call)
  *
  * outputs:
  * zc_cookie		number of bytes read
  * zc_obj		zprop_errflags_t
- * zc_action_handle	handle for this guid/ds mapping
  * zc_nvlist_dst{_size} error for each unapplied received property
  */
 static int
@@ -5137,8 +5134,7 @@ zfs_ioc_recv(zfs_cmd_t *zc)
 
 	error = zfs_ioc_recv_impl(tofs, tosnap, origin, recvdprops, localprops,
 	    NULL, zc->zc_guid, B_FALSE, zc->zc_cookie, &begin_record,
-	    zc->zc_cleanup_fd, &zc->zc_cookie, &zc->zc_obj,
+	    &zc->zc_cookie, &zc->zc_obj, &errors);
-	    &zc->zc_action_handle, &errors);
 	nvlist_free(recvdprops);
 	nvlist_free(localprops);
 
@@ -5171,15 +5167,14 @@ zfs_ioc_recv(zfs_cmd_t *zc)
  *     "input_fd" -> file descriptor to read stream from (int32)
  *     (optional) "force" -> force flag (value ignored)
  *     (optional) "resumable" -> resumable flag (value ignored)
- *     (optional) "cleanup_fd" -> cleanup-on-exit file descriptor
+ *     (optional) "cleanup_fd" -> unused
- *     (optional) "action_handle" -> handle for this guid/ds mapping
+ *     (optional) "action_handle" -> unused
  *     (optional) "hidden_args" -> { "wkeydata" -> value }
  * }
  *
  * outnvl: {
  *     "read_bytes" -> number of bytes read
  *     "error_flags" -> zprop_errflags_t
- *     "action_handle" -> handle for this guid/ds mapping
  *     "errors" -> error for each unapplied received property (nvlist)
  * }
  */
@@ -5212,11 +5207,9 @@ zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl)
 	char tofs[ZFS_MAX_DATASET_NAME_LEN];
 	boolean_t force;
 	boolean_t resumable;
-	uint64_t action_handle = 0;
 	uint64_t read_bytes = 0;
 	uint64_t errflags = 0;
 	int input_fd = -1;
-	int cleanup_fd = -1;
 	int error;
 
 	snapname = fnvlist_lookup_string(innvl, "snapname");
@@ -5244,14 +5237,6 @@ zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl)
 	force = nvlist_exists(innvl, "force");
 	resumable = nvlist_exists(innvl, "resumable");
 
-	error = nvlist_lookup_int32(innvl, "cleanup_fd", &cleanup_fd);
-	if (error && error != ENOENT)
-		return (error);
-
-	error = nvlist_lookup_uint64(innvl, "action_handle", &action_handle);
-	if (error && error != ENOENT)
-		return (error);
-
 	/* we still use "props" here for backwards compatibility */
 	error = nvlist_lookup_nvlist(innvl, "props", &recvprops);
 	if (error && error != ENOENT)
@@ -5266,12 +5251,11 @@ zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl)
 		return (error);
 
 	error = zfs_ioc_recv_impl(tofs, tosnap, origin, recvprops, localprops,
-	    hidden_args, force, resumable, input_fd, begin_record, cleanup_fd,
+	    hidden_args, force, resumable, input_fd, begin_record,
-	    &read_bytes, &errflags, &action_handle, &errors);
+	    &read_bytes, &errflags, &errors);
 
 	fnvlist_add_uint64(outnvl, "read_bytes", read_bytes);
 	fnvlist_add_uint64(outnvl, "error_flags", errflags);
-	fnvlist_add_uint64(outnvl, "action_handle", action_handle);
 	fnvlist_add_nvlist(outnvl, "errors", errors);
 
 	nvlist_free(errors);

module/zfs/zfs_onexit.c

@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2020 by Delphix. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -171,80 +171,3 @@ zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
 
 	return (0);
 }
-
-static zfs_onexit_action_node_t *
-zfs_onexit_find_cb(zfs_onexit_t *zo, uint64_t action_handle)
-{
-	zfs_onexit_action_node_t *match;
-	zfs_onexit_action_node_t *ap;
-	list_t *l;
-
-	ASSERT(MUTEX_HELD(&zo->zo_lock));
-
-	match = (zfs_onexit_action_node_t *)(uintptr_t)action_handle;
-	l = &zo->zo_actions;
-	for (ap = list_head(l); ap != NULL; ap = list_next(l, ap)) {
-		if (match == ap)
-			break;
-	}
-	return (ap);
-}
-
-/*
- * Delete the callback, triggering it first if 'fire' is set.
- */
-int
-zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, boolean_t fire)
-{
-	zfs_onexit_t *zo;
-	zfs_onexit_action_node_t *ap;
-	int error;
-
-	error = zfs_onexit_minor_to_state(minor, &zo);
-	if (error)
-		return (error);
-
-	mutex_enter(&zo->zo_lock);
-	ap = zfs_onexit_find_cb(zo, action_handle);
-	if (ap != NULL) {
-		list_remove(&zo->zo_actions, ap);
-		mutex_exit(&zo->zo_lock);
-		if (fire)
-			ap->za_func(ap->za_data);
-		kmem_free(ap, sizeof (zfs_onexit_action_node_t));
-	} else {
-		mutex_exit(&zo->zo_lock);
-		error = SET_ERROR(ENOENT);
-	}
-
-	return (error);
-}
-
-/*
- * Return the data associated with this callback.  This allows consumers
- * of the cleanup-on-exit interfaces to stash kernel data across system
- * calls, knowing that it will be cleaned up if the calling process exits.
- */
-int
-zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, void **data)
-{
-	zfs_onexit_t *zo;
-	zfs_onexit_action_node_t *ap;
-	int error;
-
-	*data = NULL;
-
-	error = zfs_onexit_minor_to_state(minor, &zo);
-	if (error)
-		return (error);
-
-	mutex_enter(&zo->zo_lock);
-	ap = zfs_onexit_find_cb(zo, action_handle);
-	if (ap != NULL)
-		*data = ap->za_data;
-	else
-		error = SET_ERROR(ENOENT);
-	mutex_exit(&zo->zo_lock);
-
-	return (error);
-}

tests/runfiles/common.run

@@ -766,22 +766,22 @@ tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos']
 tags = ['functional', 'rootpool']
 
 [tests/functional/rsend]
-tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
+tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos',
-    'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos',
+    'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'rsend_005_pos',
-    'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos',
+    'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos', 'rsend_009_pos',
-    'rsend_013_pos', 'rsend_014_pos', 'rsend_016_neg',
+    'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', 'rsend_013_pos',
-    'rsend_019_pos', 'rsend_020_pos',
+    'rsend_014_pos', 'rsend_016_neg', 'rsend_019_pos', 'rsend_020_pos',
     'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos',
     'send-c_verify_ratio', 'send-c_verify_contents', 'send-c_props',
     'send-c_incremental', 'send-c_volume', 'send-c_zstreamdump',
     'send-c_lz4_disabled', 'send-c_recv_lz4_disabled',
-    'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD',
+    'send-c_mixed_compression', 'send-c_stream_size_estimate',
     'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
     'send-c_recv_dedup', 'send_encrypted_hierarchy',
     'send_encrypted_props', 'send_encrypted_truncated_files',
     'send_freeobjects', 'send_realloc_files',
     'send_realloc_encrypted_files', 'send_spill_block', 'send_holds',
-    'send_hole_birth', 'send_mixed_raw', 'send-wDR_encrypted_zvol',
+    'send_hole_birth', 'send_mixed_raw', 'send-wR_encrypted_zvol',
     'send_partial_dataset']
 tags = ['functional', 'rsend']
 

tests/zfs-tests/tests/functional/rsend/Makefile.am

@@ -2,6 +2,8 @@ pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/rsend
 dist_pkgdata_SCRIPTS = \
 	setup.ksh \
 	cleanup.ksh \
+	recv_dedup.ksh \
+	recv_dedup_encrypted_zvol.ksh \
 	rsend_001_pos.ksh \
 	rsend_002_pos.ksh \
 	rsend_003_pos.ksh \
@@ -25,7 +27,6 @@ dist_pkgdata_SCRIPTS = \
 	send_encrypted_hierarchy.ksh \
 	send_encrypted_props.ksh \
 	send_encrypted_truncated_files.ksh \
-	send-cD.ksh \
 	send-c_embedded_blocks.ksh \
 	send-c_incremental.ksh \
 	send-c_lz4_disabled.ksh \
@@ -49,8 +50,12 @@ dist_pkgdata_SCRIPTS = \
 	send_holds.ksh \
 	send_hole_birth.ksh \
 	send_mixed_raw.ksh \
-	send-wDR_encrypted_zvol.ksh
+	send-wR_encrypted_zvol.ksh
 
 dist_pkgdata_DATA = \
+	dedup.zsend.bz2 \
+	dedup_encrypted_zvol.bz2 \
+	dedup_encrypted_zvol.zsend.bz2 \
+	fs.tar.gz \
 	rsend.cfg \
 	rsend.kshlib

tests/zfs-tests/tests/functional/rsend/recv_dedup.ksh

@@ -0,0 +1,53 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# DESCRIPTION:
+# Verifies that we can receive a dedup send stream by processing it with
+# "zstream redup".
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+	destroy_dataset $TESTPOOL/recv "-r"
+	rm -r /$TESTPOOL/tar
+	rm $sendfile
+}
+log_onexit cleanup
+
+log_assert "Verify zfs can receive dedup send streams with 'zstream redup'"
+
+typeset sendfile_compressed=$STF_SUITE/tests/functional/rsend/dedup.zsend.bz2
+typeset sendfile=/$TESTPOOL/dedup.zsend
+typeset tarfile=$STF_SUITE/tests/functional/rsend/fs.tar.gz
+
+log_must eval "bzcat <$sendfile_compressed >$sendfile"
+log_must zfs create $TESTPOOL/recv
+log_must eval "zstream redup $sendfile | zfs recv -d $TESTPOOL/recv"
+
+log_must mkdir /$TESTPOOL/tar
+log_must tar --directory /$TESTPOOL/tar -xzf $tarfile
+log_must diff -r /$TESTPOOL/tar /$TESTPOOL/recv
+
+log_pass "zfs can receive dedup send streams with 'zstream redup'"

tests/zfs-tests/tests/functional/rsend/recv_dedup_encrypted_zvol.ksh

@@ -0,0 +1,60 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# DESCRIPTION:
+# Verifies that we can receive a dedup send stream of a zvol by processing it
+# with "zstream redup".
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+	destroy_dataset $TESTPOOL/recv "-r"
+	rm $sendfile
+	rm $volfile
+	rm $keyfile
+}
+log_onexit cleanup
+
+log_assert "Verify zfs can receive raw, recursive, and deduplicated send streams"
+
+typeset keyfile=/$TESTPOOL/pkey
+typeset recvdev=$ZVOL_DEVDIR/$TESTPOOL/recv
+typeset sendfile_compressed=$STF_SUITE/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2
+typeset sendfile=/$TESTPOOL/dedup_encrypted_zvol.zsend
+typeset volfile_compressed=$STF_SUITE/tests/functional/rsend/dedup_encrypted_zvol.bz2
+typeset volfile=/$TESTPOOL/dedup_encrypted_zvol
+
+log_must eval "echo 'password' > $keyfile"
+
+log_must eval "bzcat <$sendfile_compressed >$sendfile"
+log_must eval "zstream redup $sendfile | zfs recv $TESTPOOL/recv"
+
+log_must zfs load-key $TESTPOOL/recv
+block_device_wait
+
+log_must eval "bzcat <$volfile_compressed >$volfile"
+log_must diff $volfile $recvdev
+
+log_pass "zfs can receive raw, recursive, and deduplicated send streams"

tests/zfs-tests/tests/functional/rsend/send-cD.ksh

@@ -1,89 +0,0 @@
-#!/bin/ksh -p
-
-#
-# This file and its contents are supplied under the terms of the
-# Common Development and Distribution License ("CDDL"), version 1.0.
-# You may only use this file in accordance with the terms of version
-# 1.0 of the CDDL.
-#
-# A full copy of the text of the CDDL should have accompanied this
-# source.  A copy of the CDDL is also available via the Internet at
-# http://www.illumos.org/license/CDDL.
-#
-
-#
-# Copyright (c) 2015, 2018 by Delphix. All rights reserved.
-#
-
-. $STF_SUITE/tests/functional/rsend/rsend.kshlib
-
-#
-# Description:
-# Verify that the -c and -D flags do not interfere with each other.
-#
-# Strategy:
-# 1. Write unique data to a filesystem and create a compressed, deduplicated
-#    full stream.
-# 2. Verify that the stream and send dataset show the same size
-# 3. Make several copies of the original data, and create both full and
-#    incremental compressed, deduplicated send streams
-# 4. Verify the full stream is no bigger than the stream from step 1
-# 5. Verify the streams can be received correctly.
-#
-
-verify_runnable "both"
-
-log_assert "Verify that the -c and -D flags do not interfere with each other"
-log_onexit cleanup_pool $POOL2
-
-typeset sendfs=$POOL2/sendfs
-typeset recvfs=$POOL2/recvfs
-typeset stream0=$BACKDIR/stream.0
-typeset stream1=$BACKDIR/stream.1
-typeset inc=$BACKDIR/stream.inc
-
-log_must zfs create -o compress=lz4 $sendfs
-log_must zfs create -o compress=lz4 $recvfs
-typeset dir=$(get_prop mountpoint $sendfs)
-# Don't use write_compressible: we want compressible but undeduplicable data.
-log_must eval "dd if=/dev/urandom bs=1024k count=4 | base64 >$dir/file"
-log_must zfs snapshot $sendfs@snap0
-log_must eval "zfs send -D -c $sendfs@snap0 >$stream0"
-
-# The stream size should match at this point because the data is all unique
-verify_stream_size $stream0 $sendfs
-
-for i in {0..3}; do
-	log_must cp $dir/file $dir/file.$i
-done
-log_must zfs snapshot $sendfs@snap1
-
-# The stream sizes should match, since the second stream contains no new blocks
-log_must eval "zfs send -D -c $sendfs@snap1 >$stream1"
-typeset size0=$(stat_size $stream0)
-typeset size1=$(stat_size $stream1)
-within_percent $size0 $size1 90 || log_fail "$size0 and $size1"
-
-# make sure the receive works correctly.
-log_must eval "zfs send -D -c -i snap0 $sendfs@snap1 >$inc"
-log_must eval "zfs recv -d $recvfs <$stream0"
-log_must eval "zfs recv -d $recvfs <$inc"
-cmp_ds_cont $sendfs $recvfs
-
-# check receive with redup.
-log_must zfs destroy -r $recvfs
-log_must zfs create -o compress=lz4 $recvfs
-log_must eval "zstream redup $stream0 | zfs recv -d $recvfs"
-log_must eval "zstream redup $inc | zfs recv -d $recvfs"
-cmp_ds_cont $sendfs $recvfs
-
-# The size of the incremental should be the same as the initial send.
-typeset size2=$(stat_size $inc)
-within_percent $size0 $size2 90 || log_fail "$size0 and $size1"
-
-# The redup'ed size should be 4x
-typeset size3=$(zstream redup $inc | wc -c)
-let size4=size0*4
-within_percent $size4 $size3 90 || log_fail "$size4 and $size3"
-
-log_pass "The -c and -D flags do not interfere with each other"

tests/zfs-tests/tests/functional/rsend/send-wR_encrypted_zvol.ksh

@@ -16,20 +16,21 @@
 
 #
 # Copyright (c) 2018 by Datto Inc. All rights reserved.
+# Copyright (c) 2020 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/tests/functional/rsend/rsend.kshlib
 
 #
 # DESCRIPTION:
-# Verify that zvols with dedup=on and encryption=on can be sent and received
+# Verify that zvols with encryption=on can be sent and received with a raw
-# with a deduplicated raw send stream.
+# send stream.
 #
 # STRATEGY:
-# 1. Create a zvol with dedup and encryption on and put a filesystem on it
+# 1. Create a zvol with encryption on and put a filesystem on it
 # 2. Copy a file into the zvol a few times and take a snapshot
 # 3. Repeat step 2 a few times to create more snapshots
-# 4. Send all snapshots in a recursive, raw, deduplicated send stream
+# 4. Send all snapshots in a recursive, raw send stream
 # 5. Mount the received zvol and verify that all of the data there is correct
 #
 
@@ -48,7 +49,7 @@ function cleanup
 }
 log_onexit cleanup
 
-log_assert "Verify zfs can receive raw, recursive, and deduplicated send streams"
+log_assert "Verify zfs can receive raw, recursive send streams"
 
 typeset keyfile=/$TESTPOOL/pkey
 typeset snap_count=5
@@ -93,7 +94,7 @@ for ((i = 1; i <= $snap_count; i++)); do
 	log_must mount $remount_rw $zdev $mntpnt
 done
 
-log_must eval "zfs send -wDR $TESTPOOL/$TESTVOL@snap$snap_count > $sendfile"
+log_must eval "zfs send -wR $TESTPOOL/$TESTVOL@snap$snap_count > $sendfile"
 log_must eval "zfs recv $TESTPOOL/recv < $sendfile"
 log_must zfs load-key $TESTPOOL/recv
 block_device_wait
@@ -104,4 +105,4 @@ md5_1=$(cat $mntpnt/* | md5digest)
 md5_2=$(cat $recvmnt/* | md5digest)
 [[ "$md5_1" == "$md5_2" ]] || log_fail "md5 mismatch: $md5_1 != $md5_2"
 
-log_pass "zfs can receive raw, recursive, and deduplicated send streams"
+log_pass "zfs can receive raw, recursive send streams"

tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh

@@ -13,6 +13,7 @@
 
 #
 # Copyright (c) 2019 Datto Inc.
+# Copyright (c) 2020 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
@@ -98,7 +99,6 @@ set -A badargs \
 	"-R $POOL/recvfs" \
 	"-p $POOL/recvfs" \
 	"-I $POOL/recvfs" \
-	"-D $POOL/recvfs" \
 	"-h $POOL/recvfs"
 
 while (( i < ${#badargs[*]} ))