Fix issues with truncated files in raw sends
When receiving a raw send stream only reallocated objects whose contents were not freed by the standard indicators should call dmu_free_long_range(). Furthermore, if calling dmu_free_long_range() is required then the objects current block size must be used and not the new block size. Two additional test cases were added to provided realistic test coverage for processing reallocated objects which are part of a raw receive. Reviewed-by: Olaf Faaland <faaland1@llnl.gov> Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov> Signed-off-by: Tom Caputi <tcaputi@datto.com> Closes #8528 Closes #8607
This commit is contained in:
Tom Caputi
Brian Behlendorf
parent
83472fabe5
commit
c2c6eadf29
|
|
@ -1176,6 +1176,7 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
|
|||
1ULL << drro->drr_indblkshift : 0;
|
||||
int nblkptr = deduce_nblkptr(drro->drr_bonustype,
|
||||
drro->drr_bonuslen);
|
||||
boolean_t did_free = B_FALSE;
|
||||
|
||||
object = drro->drr_object;
|
||||
|
||||
|
|
@ -1205,6 +1206,8 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
|
|||
drro->drr_object, 0, DMU_OBJECT_END);
|
||||
if (err != 0)
|
||||
return (SET_ERROR(EINVAL));
|
||||
else
|
||||
did_free = B_TRUE;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1235,11 +1238,15 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
|
|||
* processed. However, for raw receives we manually set the
|
||||
* maxblkid from the drr_maxblkid and so we must first free
|
||||
* everything above that blkid to ensure the DMU is always
|
||||
* consistent with itself.
|
||||
* consistent with itself. We will never free the first block
|
||||
* of the object here because a maxblkid of 0 could indicate
|
||||
* an object with a single block or one with no blocks. This
|
||||
* free may be skipped when dmu_free_long_range() was called
|
||||
* above since it covers the entire object's contents.
|
||||
*/
|
||||
if (rwa->raw) {
|
||||
if (rwa->raw && object != DMU_NEW_OBJECT && !did_free) {
|
||||
err = dmu_free_long_range(rwa->os, drro->drr_object,
|
||||
(drro->drr_maxblkid + 1) * drro->drr_blksz,
|
||||
(drro->drr_maxblkid + 1) * doi.doi_data_block_size,
|
||||
DMU_OBJECT_END);
|
||||
if (err != 0)
|
||||
return (SET_ERROR(EINVAL));
|
||||
|
|
@ -1380,11 +1387,8 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
|
|||
drro->drr_nlevels, tx));
|
||||
|
||||
/*
|
||||
* Set the maxblkid. We will never free the first block of
|
||||
* an object here because a maxblkid of 0 could indicate
|
||||
* an object with a single block or one with no blocks.
|
||||
* This will always succeed because we freed all blocks
|
||||
* beyond the new maxblkid above.
|
||||
* Set the maxblkid. This will always succeed because
|
||||
* we freed all blocks beyond the new maxblkid above.
|
||||
*/
|
||||
VERIFY0(dmu_object_set_maxblkid(rwa->os, drro->drr_object,
|
||||
drro->drr_maxblkid, tx));
|
||||
|
|
|
|||
|
|
@ -804,9 +804,10 @@ tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
|
|||
'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD',
|
||||
'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
|
||||
'send-c_recv_dedup', 'send_encrypted_files', 'send_encrypted_hierarchy',
|
||||
'send_encrypted_props', 'send_freeobjects', 'send_realloc_dnode_size',
|
||||
'send_realloc_files', 'send_holds', 'send_hole_birth', 'send_mixed_raw',
|
||||
'send-wDR_encrypted_zvol']
|
||||
'send_encrypted_props', 'send_encrypted_truncated_files',
|
||||
'send_freeobjects', 'send_realloc_dnode_size', 'send_realloc_files',
|
||||
'send_realloc_encrypted_files', 'send_holds', 'send_hole_birth',
|
||||
'send_mixed_raw', 'send-wDR_encrypted_zvol']
|
||||
tags = ['functional', 'rsend']
|
||||
|
||||
[tests/functional/scrub_mirror]
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ dist_pkgdata_SCRIPTS = \
|
|||
send_encrypted_files.ksh \
|
||||
send_encrypted_hierarchy.ksh \
|
||||
send_encrypted_props.ksh \
|
||||
send_encrypted_truncated_files.ksh \
|
||||
send-cD.ksh \
|
||||
send-c_embedded_blocks.ksh \
|
||||
send-c_incremental.ksh \
|
||||
|
|
@ -42,6 +43,7 @@ dist_pkgdata_SCRIPTS = \
|
|||
send_freeobjects.ksh \
|
||||
send_realloc_dnode_size.ksh \
|
||||
send_realloc_files.ksh \
|
||||
send_realloc_encrypted_files.ksh \
|
||||
send_holds.ksh \
|
||||
send_hole_birth.ksh \
|
||||
send_mixed_raw.ksh \
|
||||
|
|
|
|||
|
|
@ -464,12 +464,14 @@ function rm_files
|
|||
# $1 Number of files to modify
|
||||
# $2 Maximum file size
|
||||
# $3 File system to modify the file on
|
||||
# $4 Enabled xattrs (optional)
|
||||
#
|
||||
function churn_files
|
||||
{
|
||||
nfiles=$1
|
||||
maxsize=$2
|
||||
fs=$3
|
||||
xattrs=${4:-1}
|
||||
|
||||
#
|
||||
# Remove roughly half of the files in order to make it more
|
||||
|
|
@ -514,7 +516,7 @@ function churn_files
|
|||
#
|
||||
if [[ -e $file_name ]]; then
|
||||
value=$((RANDOM % 5))
|
||||
if [ $value -eq 0 ]; then
|
||||
if [ $value -eq 0 -a $xattrs -ne 0 ]; then
|
||||
attrname="testattr$((RANDOM % 3))"
|
||||
attr -qr $attrname $file_name || \
|
||||
log_fail "Failed to remove $attrname"
|
||||
|
|
@ -543,12 +545,15 @@ function churn_files
|
|||
bs=$file_size count=1 >/dev/null 2>&1 || \
|
||||
log_fail "Failed to create $file_name"
|
||||
|
||||
if [ $xattrs -ne 0 ]; then
|
||||
for j in {0..2}; do
|
||||
attrname="testattr$j"
|
||||
attr -qs $attrname -V TestValue $file_name || \
|
||||
log_fail "Failed to set $attrname"
|
||||
attr -qs $attrname -V TestValue \
|
||||
$file_name || log_fail \
|
||||
"Failed to set $attrname"
|
||||
done
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
return 0
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@
|
|||
|
||||
#
|
||||
# DESCRIPTION:
|
||||
#
|
||||
# Verify that a raw zfs send and receive can deal with several different
|
||||
# types of file layouts.
|
||||
#
|
||||
# STRATEGY:
|
||||
# 1. Create a new encrypted filesystem
|
||||
|
|
@ -30,18 +31,14 @@
|
|||
# 3. Add a small 512 byte file to the filesystem
|
||||
# 4. Add a larger 32M file to the filesystem
|
||||
# 5. Add a large sparse file to the filesystem
|
||||
# 6. Add a 3 files that are to be truncated later
|
||||
# 7. Add 1000 empty files to the filesystem
|
||||
# 8. Add a file with a large xattr value
|
||||
# 9. Use xattrtest to create files with random xattrs (with and without xattrs=on)
|
||||
# 10. Take a snapshot of the filesystem
|
||||
# 11. Truncate one of the files from 32M to 128k
|
||||
# 12. Truncate one of the files from 512k to 384k
|
||||
# 13. Truncate one of the files from 512k to 0 to 384k
|
||||
# 14. Remove the 1000 empty files to the filesystem
|
||||
# 15. Take another snapshot of the filesystem
|
||||
# 16. Send and receive both snapshots
|
||||
# 17. Mount the filesystem and check the contents
|
||||
# 6. Add 1000 empty files to the filesystem
|
||||
# 7. Add a file with a large xattr value
|
||||
# 8. Use xattrtest to create files with random xattrs (with and without xattrs=on)
|
||||
# 9. Take a snapshot of the filesystem
|
||||
# 10. Remove the 1000 empty files to the filesystem
|
||||
# 11. Take another snapshot of the filesystem
|
||||
# 12. Send and receive both snapshots
|
||||
# 13. Mount the filesystem and check the contents
|
||||
#
|
||||
|
||||
verify_runnable "both"
|
||||
|
|
@ -68,15 +65,12 @@ log_must eval "echo 'password' > $keyfile"
|
|||
log_must zfs create -o encryption=on -o keyformat=passphrase \
|
||||
-o keylocation=file://$keyfile $TESTPOOL/$TESTFS2
|
||||
|
||||
# Create files with vaired layouts on disk
|
||||
# Create files with varied layouts on disk
|
||||
log_must touch /$TESTPOOL/$TESTFS2/empty
|
||||
log_must mkfile 512 /$TESTPOOL/$TESTFS2/small
|
||||
log_must mkfile 32M /$TESTPOOL/$TESTFS2/full
|
||||
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/sparse \
|
||||
bs=512 count=1 seek=1048576 >/dev/null 2>&1
|
||||
log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated
|
||||
log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2
|
||||
log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3
|
||||
|
||||
log_must mkdir -p /$TESTPOOL/$TESTFS2/dir
|
||||
for i in {1..1000}; do
|
||||
|
|
@ -95,23 +89,10 @@ log_must zfs set compression=on xattr=sa $TESTPOOL/$TESTFS2
|
|||
log_must touch /$TESTPOOL/$TESTFS2/attrs
|
||||
log_must eval "python -c 'print \"a\" * 4096' | \
|
||||
attr -s bigval /$TESTPOOL/$TESTFS2/attrs"
|
||||
log_must zfs set compression=off xattr=on $TESTPOOL/$TESTFS2
|
||||
|
||||
log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1
|
||||
|
||||
#
|
||||
# Truncate files created in the first snapshot. The first tests
|
||||
# truncating a large file to a single block. The second tests
|
||||
# truncating one block off the end of a file without changing
|
||||
# the required nlevels to hold it. The last tests handling
|
||||
# of a maxblkid that is dropped and then raised again.
|
||||
#
|
||||
log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated
|
||||
log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2
|
||||
log_must truncate -s 0 /$TESTPOOL/$TESTFS2/truncated3
|
||||
log_must zpool sync $TESTPOOL
|
||||
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \
|
||||
bs=128k count=3 iflag=fullblock
|
||||
|
||||
# Remove the empty files created in the first snapshot
|
||||
for i in {1..1000}; do
|
||||
log_must rm /$TESTPOOL/$TESTFS2/dir/file-$i
|
||||
|
|
|
|||
|
|
@ -0,0 +1,118 @@
|
|||
#!/bin/ksh -p
|
||||
#
|
||||
# CDDL HEADER START
|
||||
#
|
||||
# This file and its contents are supplied under the terms of the
|
||||
# Common Development and Distribution License ("CDDL"), version 1.0.
|
||||
# You may only use this file in accordance with the terms of version
|
||||
# 1.0 of the CDDL.
|
||||
#
|
||||
# A full copy of the text of the CDDL should have accompanied this
|
||||
# source. A copy of the CDDL is also available via the Internet at
|
||||
# http://www.illumos.org/license/CDDL.
|
||||
#
|
||||
# CDDL HEADER END
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (c) 2018 by Datto Inc. All rights reserved.
|
||||
#
|
||||
|
||||
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
|
||||
|
||||
#
|
||||
# DESCRIPTION:
|
||||
#
|
||||
#
|
||||
# STRATEGY:
|
||||
# 1. Create a new encrypted filesystem
|
||||
# 2. Add a 4 files that are to be truncated later
|
||||
# 3. Take a snapshot of the filesystem
|
||||
# 4. Truncate one of the files from 32M to 128k
|
||||
# 5. Truncate one of the files from 512k to 384k
|
||||
# 6. Truncate one of the files from 512k to 0 to 384k via reallocation
|
||||
# 7. Truncate one of the files from 1k to 0 to 512b via reallocation
|
||||
# 8. Take another snapshot of the filesystem
|
||||
# 9. Send and receive both snapshots
|
||||
# 10. Mount the filesystem and check the contents
|
||||
#
|
||||
|
||||
verify_runnable "both"
|
||||
|
||||
function cleanup
|
||||
{
|
||||
datasetexists $TESTPOOL/$TESTFS2 && \
|
||||
log_must zfs destroy -r $TESTPOOL/$TESTFS2
|
||||
datasetexists $TESTPOOL/recv && \
|
||||
log_must zfs destroy -r $TESTPOOL/recv
|
||||
[[ -f $keyfile ]] && log_must rm $keyfile
|
||||
[[ -f $sendfile ]] && log_must rm $sendfile
|
||||
}
|
||||
log_onexit cleanup
|
||||
|
||||
function recursive_cksum
|
||||
{
|
||||
find $1 -type f -exec sha256sum {} \; | \
|
||||
sort -k 2 | awk '{ print $1 }' | sha256sum
|
||||
}
|
||||
|
||||
log_assert "Verify 'zfs send -w' works with many different file layouts"
|
||||
|
||||
typeset keyfile=/$TESTPOOL/pkey
|
||||
typeset sendfile=/$TESTPOOL/sendfile
|
||||
typeset sendfile2=/$TESTPOOL/sendfile2
|
||||
|
||||
# Create an encrypted dataset
|
||||
log_must eval "echo 'password' > $keyfile"
|
||||
log_must zfs create -o encryption=on -o keyformat=passphrase \
|
||||
-o keylocation=file://$keyfile $TESTPOOL/$TESTFS2
|
||||
|
||||
# Explicitly set the recordsize since the truncation sizes below depend on
|
||||
# this value being 128k. This is currently same as the default recordsize.
|
||||
log_must zfs set recordsize=128k $TESTPOOL/$TESTFS2
|
||||
|
||||
# Create files with varied layouts on disk
|
||||
log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated
|
||||
log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2
|
||||
log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3
|
||||
log_must mkfile 1024 /$TESTPOOL/$TESTFS2/truncated4
|
||||
|
||||
log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1
|
||||
|
||||
#
|
||||
# Truncate files created in the first snapshot. The first tests
|
||||
# truncating a large file to a single block. The second tests
|
||||
# truncating one block off the end of a file without changing
|
||||
# the required nlevels to hold it. The third tests handling
|
||||
# of a maxblkid that is dropped and then raised again. The
|
||||
# fourth tests an object that is truncated from a single block
|
||||
# to a smaller single block.
|
||||
#
|
||||
log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated
|
||||
log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2
|
||||
log_must rm -f /$TESTPOOL/$TESTFS2/truncated3
|
||||
log_must rm -f /$TESTPOOL/$TESTFS2/truncated4
|
||||
log_must zpool sync $TESTPOOL
|
||||
log_must zfs umount $TESTPOOL/$TESTFS2
|
||||
log_must zfs mount $TESTPOOL/$TESTFS2
|
||||
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \
|
||||
bs=128k count=3 iflag=fullblock
|
||||
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated4 \
|
||||
bs=512 count=1 iflag=fullblock
|
||||
|
||||
log_must zfs snapshot $TESTPOOL/$TESTFS2@snap2
|
||||
expected_cksum=$(recursive_cksum /$TESTPOOL/$TESTFS2)
|
||||
|
||||
log_must eval "zfs send -wp $TESTPOOL/$TESTFS2@snap1 > $sendfile"
|
||||
log_must eval "zfs send -wp -i @snap1 $TESTPOOL/$TESTFS2@snap2 > $sendfile2"
|
||||
|
||||
log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile"
|
||||
log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile2"
|
||||
log_must zfs load-key $TESTPOOL/recv
|
||||
|
||||
log_must zfs mount -a
|
||||
actual_cksum=$(recursive_cksum /$TESTPOOL/recv)
|
||||
[[ "$expected_cksum" != "$actual_cksum" ]] && \
|
||||
log_fail "Recursive checksums differ ($expected_cksum != $actual_cksum)"
|
||||
|
||||
log_pass "Verified 'zfs send -w' works with many different file layouts"
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
#!/bin/ksh
|
||||
|
||||
#
|
||||
# This file and its contents are supplied under the terms of the
|
||||
# Common Development and Distribution License ("CDDL"), version 1.0.
|
||||
# You may only use this file in accordance with the terms of version
|
||||
# 1.0 of the CDDL.
|
||||
#
|
||||
# A full copy of the text of the CDDL should have accompanied this
|
||||
# source. A copy of the CDDL is also available via the Internet at
|
||||
# http://www.illumos.org/license/CDDL.
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (c) 2019 by Lawrence Livermore National Security, LLC.
|
||||
#
|
||||
|
||||
. $STF_SUITE/include/libtest.shlib
|
||||
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
|
||||
|
||||
#
|
||||
# Description:
|
||||
# Verify encrypted raw incremental receives handle dnode reallocation.
|
||||
|
||||
# Strategy:
|
||||
# 1. Create a pool containing an encrypted filesystem.
|
||||
# 2. Use 'zfs send -wp' to perform a raw send of the initial filesystem.
|
||||
# 3. Repeat the followings steps N times to verify raw incremental receives.
|
||||
# a) Randomly change several key dataset properties.
|
||||
# b) Modify the contents of the filesystem such that dnode reallocation
|
||||
# is likely during the 'zfs receive', and receive_object() exercises
|
||||
# as much of its functionality as possible.
|
||||
# c) Create a new snapshot and generate an raw incremental stream.
|
||||
# d) Receive the raw incremental stream and verify the received contents.
|
||||
# e) Destroy the incremental stream and old snapshot.
|
||||
#
|
||||
|
||||
verify_runnable "both"
|
||||
|
||||
log_assert "Verify encrypted raw incremental receive handles reallocation"
|
||||
|
||||
function cleanup
|
||||
{
|
||||
rm -f $BACKDIR/fs@*
|
||||
rm -f $keyfile
|
||||
destroy_dataset $POOL/fs "-rR"
|
||||
destroy_dataset $POOL/newfs "-rR"
|
||||
}
|
||||
|
||||
log_onexit cleanup
|
||||
|
||||
typeset keyfile=/$TESTPOOL/pkey
|
||||
|
||||
# Create an encrypted dataset
|
||||
log_must eval "echo 'password' > $keyfile"
|
||||
log_must zfs create -o encryption=on -o keyformat=passphrase \
|
||||
-o keylocation=file://$keyfile $POOL/fs
|
||||
|
||||
last_snap=1
|
||||
log_must zfs snapshot $POOL/fs@snap${last_snap}
|
||||
log_must eval "zfs send -wp $POOL/fs@snap${last_snap} \
|
||||
>$BACKDIR/fs@snap${last_snap}"
|
||||
log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs@snap${last_snap}"
|
||||
|
||||
# Set atime=off to prevent the recursive_cksum from modifying newfs.
|
||||
log_must zfs set atime=off $POOL/newfs
|
||||
|
||||
for i in {1..5}; do
|
||||
# Randomly modify several dataset properties in order to generate
|
||||
# more interesting incremental send streams.
|
||||
rand_set_prop $POOL/fs checksum "off" "fletcher4" "sha256"
|
||||
rand_set_prop $POOL/fs compression "off" "lzjb" "gzip" "lz4"
|
||||
rand_set_prop $POOL/fs recordsize "32K" "128K"
|
||||
rand_set_prop $POOL/fs dnodesize "legacy" "auto" "4k"
|
||||
rand_set_prop $POOL/fs xattr "on" "sa"
|
||||
|
||||
# Churn the filesystem in such a way that we're likely to be both
|
||||
# allocating and reallocating objects in the incremental stream.
|
||||
#
|
||||
# Disable xattrs until the following spill block issue is resolved:
|
||||
# https://github.com/openzfs/openzfs/pull/705
|
||||
#
|
||||
log_must churn_files 1000 524288 $POOL/fs 0
|
||||
expected_cksum=$(recursive_cksum /$fs)
|
||||
|
||||
# Create a snapshot and use it to send an incremental stream.
|
||||
this_snap=$((last_snap + 1))
|
||||
log_must zfs snapshot $POOL/fs@snap${this_snap}
|
||||
log_must eval "zfs send -wp -i $POOL/fs@snap${last_snap} \
|
||||
$POOL/fs@snap${this_snap} > $BACKDIR/fs@snap${this_snap}"
|
||||
|
||||
# Receive the incremental stream and verify the received contents.
|
||||
log_must eval "zfs recv -Fu $POOL/newfs < $BACKDIR/fs@snap${this_snap}"
|
||||
|
||||
log_must zfs load-key $POOL/newfs
|
||||
log_must zfs mount $POOL/newfs
|
||||
actual_cksum=$(recursive_cksum /$POOL/newfs)
|
||||
log_must zfs umount $POOL/newfs
|
||||
log_must zfs unload-key $POOL/newfs
|
||||
|
||||
if [[ "$expected_cksum" != "$actual_cksum" ]]; then
|
||||
log_fail "Checksums differ ($expected_cksum != $actual_cksum)"
|
||||
fi
|
||||
|
||||
# Destroy the incremental stream and old snapshot.
|
||||
rm -f $BACKDIR/fs@snap${last_snap}
|
||||
log_must zfs destroy $POOL/fs@snap${last_snap}
|
||||
log_must zfs destroy $POOL/newfs@snap${last_snap}
|
||||
last_snap=$this_snap
|
||||
done
|
||||
|
||||
log_pass "Verify encrypted raw incremental receive handles reallocation"
|
||||
Loading…
Reference in New Issue