From 099fa7e475d30b53ea64b2327fcaee4ba9741690 Mon Sep 17 00:00:00 2001 From: pablofsf <32678034+pablofsf@users.noreply.github.com> Date: Sun, 11 Apr 2021 21:05:35 +0200 Subject: [PATCH] Allow zfs to send replication streams with missing snapshots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tentative implementation and discussion was done in #5285. According to it a send --skip-missing|-s flag has been added. In a replication stream, when there are snapshots missing in the hierarchy, if -s is provided print a warning and ignore dataset (and its children) instead of throwing an error Reviewed-by: Paul Dagnelie Reviewed-by: Brian Behlendorf Signed-off-by: Pablo Correa Gómez Closes #11710 --- cmd/zfs/zfs_main.c | 13 +++- include/libzfs.h | 3 + lib/libzfs/libzfs_sendrecv.c | 36 +++++---- man/man8/zfs-send.8 | 10 ++- tests/runfiles/common.run | 2 +- .../functional/cli_root/zfs_send/Makefile.am | 3 +- .../zfs_send/zfs_send_skip_missing.ksh | 77 +++++++++++++++++++ 7 files changed, 126 insertions(+), 18 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_skip_missing.ksh diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c index 1a5129f794..395fc353fe 100644 --- a/cmd/zfs/zfs_main.c +++ b/cmd/zfs/zfs_main.c @@ -4376,6 +4376,7 @@ zfs_do_send(int argc, char **argv) struct option long_options[] = { {"replicate", no_argument, NULL, 'R'}, + {"skip-missing", no_argument, NULL, 's'}, {"redact", required_argument, NULL, 'd'}, {"props", no_argument, NULL, 'p'}, {"parsable", no_argument, NULL, 'P'}, @@ -4394,7 +4395,7 @@ zfs_do_send(int argc, char **argv) }; /* check options */ - while ((c = getopt_long(argc, argv, ":i:I:RDpvnPLeht:cwbd:S", + while ((c = getopt_long(argc, argv, ":i:I:RsDpvnPLeht:cwbd:S", long_options, NULL)) != -1) { switch (c) { case 'i': @@ -4411,6 +4412,9 @@ zfs_do_send(int argc, char **argv) case 'R': flags.replicate = B_TRUE; break; + case 's': + flags.skipmissing = B_TRUE; + break; case 'd': redactbook = optarg; break; @@ -4575,6 +4579,13 @@ zfs_do_send(int argc, char **argv) resume_token)); } + if (flags.skipmissing && !flags.replicate) { + (void) fprintf(stderr, + gettext("skip-missing flag can only be used in " + "conjunction with replicate\n")); + usage(B_FALSE); + } + /* * For everything except -R and -I, use the new, cleaner code path. */ diff --git a/include/libzfs.h b/include/libzfs.h index 5f0bc03be1..e8e7713827 100644 --- a/include/libzfs.h +++ b/include/libzfs.h @@ -666,6 +666,9 @@ typedef struct sendflags { /* recursive send (ie, -R) */ boolean_t replicate; + /* for recursive send, skip sending missing snapshots */ + boolean_t skipmissing; + /* for incrementals, do all intermediate snapshots */ boolean_t doall; diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index bc887e72a2..ee593f8dbd 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -247,6 +247,7 @@ typedef struct send_data { boolean_t raw; boolean_t doall; boolean_t replicate; + boolean_t skipmissing; boolean_t verbose; boolean_t backup; boolean_t seenfrom; @@ -497,7 +498,8 @@ send_iterate_fs(zfs_handle_t *zhp, void *arg) * - skip sending the current dataset if it was created later than * the parent tosnap * - return error if the current dataset was created earlier than - * the parent tosnap + * the parent tosnap, unless --skip-missing specified. Then + * just print a warning */ if (sd->tosnap != NULL && tosnap_txg == 0) { if (sd->tosnap_txg != 0 && txg > sd->tosnap_txg) { @@ -506,6 +508,11 @@ send_iterate_fs(zfs_handle_t *zhp, void *arg) "skipping dataset %s: snapshot %s does " "not exist\n"), zhp->zfs_name, sd->tosnap); } + } else if (sd->skipmissing) { + (void) fprintf(stderr, dgettext(TEXT_DOMAIN, + "WARNING: skipping dataset %s and its children:" + " snapshot %s does not exist\n"), + zhp->zfs_name, sd->tosnap); } else { (void) fprintf(stderr, dgettext(TEXT_DOMAIN, "cannot send %s@%s%s: snapshot %s@%s does not " @@ -649,8 +656,9 @@ out: static int gather_nvlist(libzfs_handle_t *hdl, const char *fsname, const char *fromsnap, const char *tosnap, boolean_t recursive, boolean_t raw, boolean_t doall, - boolean_t replicate, boolean_t verbose, boolean_t backup, boolean_t holds, - boolean_t props, nvlist_t **nvlp, avl_tree_t **avlp) + boolean_t replicate, boolean_t skipmissing, boolean_t verbose, + boolean_t backup, boolean_t holds, boolean_t props, nvlist_t **nvlp, + avl_tree_t **avlp) { zfs_handle_t *zhp; send_data_t sd = { 0 }; @@ -668,6 +676,7 @@ gather_nvlist(libzfs_handle_t *hdl, const char *fsname, const char *fromsnap, sd.raw = raw; sd.doall = doall; sd.replicate = replicate; + sd.skipmissing = skipmissing; sd.verbose = verbose; sd.backup = backup; sd.holds = holds; @@ -1975,8 +1984,8 @@ send_conclusion_record(int fd, zio_cksum_t *zc) static int send_prelim_records(zfs_handle_t *zhp, const char *from, int fd, boolean_t gather_props, boolean_t recursive, boolean_t verbose, - boolean_t dryrun, boolean_t raw, boolean_t replicate, boolean_t backup, - boolean_t holds, boolean_t props, boolean_t doall, + boolean_t dryrun, boolean_t raw, boolean_t replicate, boolean_t skipmissing, + boolean_t backup, boolean_t holds, boolean_t props, boolean_t doall, nvlist_t **fssp, avl_tree_t **fsavlp) { int err = 0; @@ -2022,8 +2031,8 @@ send_prelim_records(zfs_handle_t *zhp, const char *from, int fd, } if ((err = gather_nvlist(zhp->zfs_hdl, tofs, - from, tosnap, recursive, raw, doall, replicate, verbose, - backup, holds, props, &fss, fsavlp)) != 0) { + from, tosnap, recursive, raw, doall, replicate, skipmissing, + verbose, backup, holds, props, &fss, fsavlp)) != 0) { return (zfs_error(zhp->zfs_hdl, EZFS_BADBACKUP, errbuf)); } @@ -2160,8 +2169,9 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap, err = send_prelim_records(tosnap, fromsnap, outfd, flags->replicate || flags->props || flags->holds, flags->replicate, flags->verbosity > 0, flags->dryrun, - flags->raw, flags->replicate, flags->backup, flags->holds, - flags->props, flags->doall, &fss, &fsavl); + flags->raw, flags->replicate, flags->skipmissing, + flags->backup, flags->holds, flags->props, flags->doall, + &fss, &fsavl); zfs_close(tosnap); if (err != 0) goto err_out; @@ -2464,7 +2474,7 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags, */ err = send_prelim_records(zhp, NULL, fd, B_TRUE, B_FALSE, flags->verbosity > 0, flags->dryrun, flags->raw, - flags->replicate, flags->backup, flags->holds, + flags->replicate, B_FALSE, flags->backup, flags->holds, flags->props, flags->doall, NULL, NULL); if (err != 0) return (err); @@ -3236,7 +3246,7 @@ again: deleted = fnvlist_alloc(); if ((error = gather_nvlist(hdl, tofs, fromsnap, NULL, - recursive, B_TRUE, B_FALSE, recursive, B_FALSE, B_FALSE, + recursive, B_TRUE, B_FALSE, recursive, B_FALSE, B_FALSE, B_FALSE, B_FALSE, B_TRUE, &local_nv, &local_avl)) != 0) return (error); @@ -4729,8 +4739,8 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, */ *cp = '\0'; if (gather_nvlist(hdl, destsnap, NULL, NULL, B_FALSE, B_TRUE, - B_FALSE, B_FALSE, B_FALSE, B_FALSE, B_FALSE, B_TRUE, - &local_nv, &local_avl) == 0) { + B_FALSE, B_FALSE, B_FALSE, B_FALSE, B_FALSE, B_FALSE, + B_TRUE, &local_nv, &local_avl) == 0) { *cp = '@'; fs = fsavl_find(local_avl, drrb->drr_toguid, NULL); fsavl_destroy(local_avl); diff --git a/man/man8/zfs-send.8 b/man/man8/zfs-send.8 index 4156db4f6b..2d64a5a1f7 100644 --- a/man/man8/zfs-send.8 +++ b/man/man8/zfs-send.8 @@ -39,12 +39,12 @@ .Sh SYNOPSIS .Nm zfs .Cm send -.Op Fl DLPRbcehnpvw +.Op Fl DLPRsbcehnpvw .Op Oo Fl I Ns | Ns Fl i Oc Ar snapshot .Ar snapshot .Nm zfs .Cm send -.Op Fl DLPRcenpvw +.Op Fl DLPRscenpvw .Op Fl i Ar snapshot Ns | Ns Ar bookmark .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot .Nm zfs @@ -139,6 +139,12 @@ do not exist on the sending side are destroyed. If the flag is used to send encrypted datasets, then .Fl w must also be specified. +.It Fl s, -skip-missing +Allows sending a replication stream even when there are snapshots missing in the +hierarchy. When a snapshot is missing, instead of throwing an error and aborting +the send, a warning is printed to STDERR and the dataset to which it belongs +and its descendents are skipped. This flag can only be used in conjunction with +.Fl R . .It Fl e, -embed Generate a more compact stream by using .Sy WRITE_EMBEDDED diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 32be9ef057..07c816f52f 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -258,7 +258,7 @@ tags = ['functional', 'cli_root', 'zfs_rollback'] tests = ['zfs_send_001_pos', 'zfs_send_002_pos', 'zfs_send_003_pos', 'zfs_send_004_neg', 'zfs_send_005_pos', 'zfs_send_006_pos', 'zfs_send_007_pos', 'zfs_send_encrypted', 'zfs_send_raw', - 'zfs_send_sparse', 'zfs_send-b'] + 'zfs_send_sparse', 'zfs_send-b', 'zfs_send_skip_missing'] tags = ['functional', 'cli_root', 'zfs_send'] [tests/functional/cli_root/zfs_set] diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_send/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zfs_send/Makefile.am index 9a492f3231..25c7065670 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_send/Makefile.am +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_send/Makefile.am @@ -13,7 +13,8 @@ dist_pkgdata_SCRIPTS = \ zfs_send_encrypted_unloaded.ksh \ zfs_send_raw.ksh \ zfs_send_sparse.ksh \ - zfs_send-b.ksh + zfs_send-b.ksh \ + zfs_send_skip_missing.ksh dist_pkgdata_DATA = \ zfs_send.cfg diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_skip_missing.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_skip_missing.ksh new file mode 100755 index 0000000000..b367cef9c4 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_skip_missing.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2016, loli10K. All rights reserved. +# Copyright (c) 2021, Pablo Correa Gómez. All rights reserved. +# + +. $STF_SUITE/tests/functional/cli_root/cli_common.kshlib +. $STF_SUITE/tests/functional/cli_root/zfs_send/zfs_send.cfg + +# +# DESCRIPTION: +# Verify 'zfs send' will avoid sending replication send +# streams when we're missing snapshots in the dataset +# hierarchy, unless -s|--skip-missing provided +# +# STRATEGY: +# 1. Create a parent and child fs and then only snapshot the parent +# 2. Verify sending with replication will fail +# 3. Verify sending with skip-missing will print a warning but succeed +# + +verify_runnable "both" + +function cleanup +{ + snapexists $SNAP && log_must zfs destroy -f $SNAP + + datasetexists $PARENT && log_must zfs destroy -rf $PARENT + + [[ -e $WARNF ]] && log_must rm -f $WARNF + rm -f $TEST_BASE_DIR/devnull +} + +log_assert "Verify 'zfs send -Rs' works as expected." +log_onexit cleanup + +PARENT=$TESTPOOL/parent +CHILD=$PARENT/child +SNAP=$PARENT@snap +WARNF=$TEST_BASE_DIR/warn.2 + +log_note "Verify 'zfs send -R' fails to generate replication stream"\ + " for datasets created before" + +log_must zfs create $PARENT +log_must zfs create $CHILD +log_must zfs snapshot $SNAP +log_mustnot eval "zfs send -R $SNAP >$TEST_BASE_DIR/devnull" + +log_note "Verify 'zfs send -Rs' warns about missing snapshots, "\ + "but still succeeds" + +log_must eval "zfs send -Rs $SNAP 2> $WARNF >$TEST_BASE_DIR/devnull" +log_must eval "[[ -s $WARNF ]]" + +log_pass "Verify 'zfs send -Rs' works as expected."