From f55d6ee818763a78328345c5bee9f38d18b852a6 Mon Sep 17 00:00:00 2001 From: Alek P Date: Wed, 15 Mar 2023 10:34:10 -0700 Subject: [PATCH] Improve tests and update man page for healing recv Fix the manpage. The "SYNOPSIS" section is incorrectly formatted for receive -c. I also took this opportunity to reword some parts and fix a run-on sentence in the manpage. Add large block testing for corrective recv. This adds a new test that makes sure blocks generated using zfs send -L/--large-block large-block send flag are able to be used for healing. Since with unloaded key and errlog feature enabled corruption is not shown in zpool status #13675 is fixed the zfs_receive_corrective.ksh test no longer sets -o feature@head_errlog=disabled on pool creation so that it can also test for regressions related to head_errlog feature. Note that the zfs_receive_compressed_corrective.ksh and zfs_receive_large_block_corrective.ksh tests are still creating pools with -o feature@head_errlog=disabled. Reviewed-by: Brian Behlendorf Signed-off-by: Alek Pinchuk Closes #14615 --- man/man8/zfs-receive.8 | 15 +- tests/runfiles/common.run | 2 +- tests/zfs-tests/tests/Makefile.am | 1 + .../zfs_receive/zfs_receive_corrective.ksh | 2 +- .../zfs_receive_large_block_corrective.ksh | 194 ++++++++++++++++++ 5 files changed, 204 insertions(+), 10 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh diff --git a/man/man8/zfs-receive.8 b/man/man8/zfs-receive.8 index 22cb567c1c..a15f2cbe94 100644 --- a/man/man8/zfs-receive.8 +++ b/man/man8/zfs-receive.8 @@ -29,7 +29,7 @@ .\" Copyright 2018 Nexenta Systems, Inc. .\" Copyright 2019 Joyent, Inc. .\" -.Dd April 26, 2022 +.Dd March 12, 2023 .Dt ZFS-RECEIVE 8 .Os . @@ -56,8 +56,7 @@ .Cm receive .Fl A .Ar filesystem Ns | Ns Ar volume -. -.Nm +.Nm zfs .Cm receive .Fl c .Op Fl vn @@ -406,15 +405,15 @@ deleting its saved partially received state. .Op Fl vn .Ar filesystem Ns | Ns Ar snapshot .Xc -Attempt to correct data corruption in the specified dataset, +Attempt to repair data corruption in the specified dataset, by using the provided stream as the source of healthy data. This method of healing can only heal data blocks present in the stream. Metadata can not be healed by corrective receive. -Running a scrub is recommended post-healing to ensure all corruption was -healed. +Running a scrub is recommended post-healing to ensure all data corruption was +repaired. .Pp -It's important to consider why corruption has happened in the first place -since if you have slowly failing hardware periodically healing the data +It's important to consider why corruption has happened in the first place. +If you have slowly failing hardware - periodically repairing the data is not going to save you from data loss later on when the hardware fails completely. .El diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index d30bcd804c..e4824457e9 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -242,7 +242,7 @@ tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos', 'zfs_receive_raw', 'zfs_receive_raw_incremental', 'zfs_receive_-e', 'zfs_receive_raw_-d', 'zfs_receive_from_zstd', 'zfs_receive_new_props', 'zfs_receive_-wR-encrypted-mix', 'zfs_receive_corrective', - 'zfs_receive_compressed_corrective'] + 'zfs_receive_compressed_corrective', 'zfs_receive_large_block_corrective'] tags = ['functional', 'cli_root', 'zfs_receive'] [tests/functional/cli_root/zfs_rename] diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index 4b0a28a6ef..9e738227f7 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -774,6 +774,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/zfs_receive/zfs_receive_-wR-encrypted-mix.ksh \ functional/cli_root/zfs_receive/zfs_receive_corrective.ksh \ functional/cli_root/zfs_receive/zfs_receive_compressed_corrective.ksh \ + functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh \ functional/cli_root/zfs_rename/cleanup.ksh \ functional/cli_root/zfs_rename/setup.ksh \ functional/cli_root/zfs_rename/zfs_rename_001_pos.ksh \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_corrective.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_corrective.ksh index b2bbdf2a7b..9ebde1cd9d 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_corrective.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_corrective.ksh @@ -85,7 +85,7 @@ typeset passphrase="password" typeset file="/$TESTPOOL/$TESTFS1/$TESTFILE0" log_must eval "poolexists $TESTPOOL && destroy_pool $TESTPOOL" -log_must zpool create -f -o feature@head_errlog=disabled $TESTPOOL $DISK +log_must zpool create -f $TESTPOOL $DISK log_must eval "echo $passphrase > /$TESTPOOL/pwd" diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh new file mode 100755 index 0000000000..fbcd33f607 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh @@ -0,0 +1,194 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2019 Datto, Inc. All rights reserved. +# Copyright (c) 2022 Axcient. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# OpenZFS should be able to heal data using corrective recv when the send file +# was generated with the --large-block flag +# +# STRATEGY: +# 0. Create a file, checksum the file to be corrupted then compare it's checksum +# with the one obtained after healing under different testing scenarios: +# 1. Test healing (aka corrective) recv from a full send file +# 2. Test healing recv (aka heal recv) from an incremental send file +# 3. Test healing recv when compression on-disk is off but source was compressed +# 4. Test heal recv when compression on-disk is on but source was uncompressed +# 5. Test heal recv when compression doesn't match between send file and on-disk +# 6. Test healing recv of an encrypted dataset using an unencrypted send file +# 7. Test healing recv (on an encrypted dataset) using a raw send file +# 8. Test healing when specifying destination filesystem only (no snapshot) +# 9. Test incremental recv aftear healing recv +# + +verify_runnable "both" + +DISK=${DISKS%% *} + +backup=$TEST_BASE_DIR/backup +raw_backup=$TEST_BASE_DIR/raw_backup +ibackup=$TEST_BASE_DIR/ibackup +unc_backup=$TEST_BASE_DIR/unc_backup + +function cleanup +{ + log_must rm -f $backup $raw_backup $ibackup $unc_backup + + poolexists $TESTPOOL && destroy_pool $TESTPOOL + log_must zpool create -f $TESTPOOL $DISK +} + +function test_corrective_recv +{ + log_must zpool scrub -w $TESTPOOL + log_must zpool status -v $TESTPOOL + log_must eval "zpool status -v $TESTPOOL | \ + grep \"Permanent errors have been detected\"" + + # make sure we will read the corruption from disk by flushing the ARC + log_must zinject -a + + log_must eval "zfs recv -c $1 < $2" + + log_must zpool scrub -w $TESTPOOL + log_must zpool status -v $TESTPOOL + log_mustnot eval "zpool status -v $TESTPOOL | \ + grep \"Permanent errors have been detected\"" + typeset cksum=$(md5digest $file) + [[ "$cksum" == "$checksum" ]] || \ + log_fail "Checksums differ ($cksum != $checksum)" +} + +log_onexit cleanup + +log_assert "ZFS corrective receive should be able to heal data corruption" + +typeset passphrase="password" +typeset file="/$TESTPOOL/$TESTFS1/$TESTFILE0" + +log_must eval "poolexists $TESTPOOL && destroy_pool $TESTPOOL" +log_must zpool create -f -o feature@large_blocks=enabled \ + -o feature@head_errlog=disabled $TESTPOOL $DISK + +log_must eval "echo $passphrase > /$TESTPOOL/pwd" + +log_must zfs create -o recordsize=1m -o primarycache=none \ + -o atime=off -o compression=lz4 $TESTPOOL/$TESTFS1 + +log_must dd if=/dev/urandom of=$file bs=1024 count=1024 oflag=sync +log_must eval "echo 'aaaaaaaa' >> "$file +typeset checksum=$(md5digest $file) + +log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1 + +# create full send file +log_must eval "zfs send -L $TESTPOOL/$TESTFS1@snap1 > $backup" + +log_must dd if=/dev/urandom of=$file"1" bs=1024 count=1024 oflag=sync +log_must eval "echo 'bbbbbbbb' >> "$file"1" +log_must zfs snapshot $TESTPOOL/$TESTFS1@snap2 +# create incremental send file +log_must eval "zfs send -Li $TESTPOOL/$TESTFS1@snap1 \ + $TESTPOOL/$TESTFS1@snap2 > $ibackup" + +corrupt_blocks_at_level $file 0 +# test healing recv from a full send file +test_corrective_recv $TESTPOOL/$TESTFS1@snap1 $backup + +corrupt_blocks_at_level $file"1" 0 +# test healing recv from an incremental send file +test_corrective_recv $TESTPOOL/$TESTFS1@snap2 $ibackup + +# create new uncompressed dataset using our send file +log_must eval "zfs recv -o compression=off -o primarycache=none \ + $TESTPOOL/$TESTFS2 < $backup" +typeset compr=$(get_prop compression $TESTPOOL/$TESTFS2) +[[ "$compr" == "off" ]] || \ + log_fail "Unexpected compression $compr in recved dataset" +corrupt_blocks_at_level "/$TESTPOOL/$TESTFS2/$TESTFILE0" 0 +# test healing recv when compression on-disk is off but source was compressed +test_corrective_recv "$TESTPOOL/$TESTFS2@snap1" $backup + +# create a full sendfile from an uncompressed source +log_must eval "zfs send -L $TESTPOOL/$TESTFS2@snap1 > $unc_backup" +log_must eval "zfs recv -o compression=gzip -o primarycache=none \ + -o recordsize=1m $TESTPOOL/testfs3 < $unc_backup" +typeset compr=$(get_prop compression $TESTPOOL/testfs3) +[[ "$compr" == "gzip" ]] || \ + log_fail "Unexpected compression $compr in recved dataset" +corrupt_blocks_at_level "/$TESTPOOL/testfs3/$TESTFILE0" 0 +# test healing recv when compression on-disk is on but source was uncompressed +test_corrective_recv "$TESTPOOL/testfs3@snap1" $unc_backup + +# create new compressed dataset using our send file +log_must eval "zfs recv -o compression=gzip -o primarycache=none \ + -o recordsize=1m $TESTPOOL/testfs4 < $backup" +typeset compr=$(get_prop compression $TESTPOOL/testfs4) +[[ "$compr" == "gzip" ]] || \ + log_fail "Unexpected compression $compr in recved dataset" +corrupt_blocks_at_level "/$TESTPOOL/testfs4/$TESTFILE0" 0 +# test healing recv when compression doesn't match between send file and on-disk +test_corrective_recv "$TESTPOOL/testfs4@snap1" $backup + +# create new encrypted (and compressed) dataset using our send file +log_must eval "zfs recv -o encryption=aes-256-ccm -o keyformat=passphrase \ + -o recordsize=1m -o keylocation=file:///$TESTPOOL/pwd -o primarycache=none \ + $TESTPOOL/testfs5 < $backup" +typeset encr=$(get_prop encryption $TESTPOOL/testfs5) +[[ "$encr" == "aes-256-ccm" ]] || \ + log_fail "Unexpected encryption $encr in recved dataset" +log_must eval "zfs send -L --raw $TESTPOOL/testfs5@snap1 > $raw_backup" +log_must eval "zfs send -L $TESTPOOL/testfs5@snap1 > $backup" +corrupt_blocks_at_level "/$TESTPOOL/testfs5/$TESTFILE0" 0 +# test healing recv of an encrypted dataset using an unencrypted send file +test_corrective_recv "$TESTPOOL/testfs5@snap1" $backup +corrupt_blocks_at_level "/$TESTPOOL/testfs5/$TESTFILE0" 0 +log_must zfs unmount $TESTPOOL/testfs5 +log_must zfs unload-key $TESTPOOL/testfs5 +# test healing recv (on an encrypted dataset) using a raw send file +test_corrective_recv "$TESTPOOL/testfs5@snap1" $raw_backup +# non raw send file healing an encrypted dataset with an unloaded key will fail +log_mustnot eval "zfs recv -c $TESTPOOL/testfs5@snap1 < $backup" + +log_must zfs rollback -r $TESTPOOL/$TESTFS1@snap1 +corrupt_blocks_at_level $file 0 +# test healing when specifying destination filesystem only (no snapshot) +test_corrective_recv $TESTPOOL/$TESTFS1 $backup +# test incremental recv aftear healing recv +log_must eval "zfs recv -o recordsize=1m $TESTPOOL/$TESTFS1 < $ibackup" + +# test that healing recv can not be combined with incompatible recv options +log_mustnot eval "zfs recv -h -c $TESTPOOL/$TESTFS1@snap1 < $backup" +log_mustnot eval "zfs recv -F -c $TESTPOOL/$TESTFS1@snap1 < $backup" +log_mustnot eval "zfs recv -s -c $TESTPOOL/$TESTFS1@snap1 < $backup" +log_mustnot eval "zfs recv -u -c $TESTPOOL/$TESTFS1@snap1 < $backup" +log_mustnot eval "zfs recv -d -c $TESTPOOL/$TESTFS1@snap1 < $backup" +log_mustnot eval "zfs recv -e -c $TESTPOOL/$TESTFS1@snap1 < $backup" + +# ensure healing recv doesn't work when snap GUIDS don't match +log_mustnot eval "zfs recv -c $TESTPOOL/testfs5@snap2 < $backup" +log_mustnot eval "zfs recv -c $TESTPOOL/testfs5 < $backup" + +# test that healing recv doesn't work on non-existing snapshots +log_mustnot eval "zfs recv -c $TESTPOOL/$TESTFS1@missing < $backup" + +log_pass "OpenZFS corrective recv works for data healing"