Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity

Revert "Fix issues with truncated files in raw sends" 

This partially reverts commit 5dbf8b4ed.  This change resolved
the issues observed with truncated files in raw sends.  However,
the required changes to dnode_allocate() introduced a regression
for non-raw streams which needs to be understood.

The additional debugging improvements from the original patch
were not reverted.

Reviewed-by: Tom Caputi <tcaputi@datto.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Issue #7378
Issue #8528
Issue #8540
Issue #8565
Close #8584
This commit is contained in:
Brian Behlendorf 2019-04-05 17:32:56 -07:00 committed by GitHub
parent 944a37248a
commit d93d4b1acd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 46 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
module/zfs/dmu.c
View File

@ -1737,7 +1737,6 @@ dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset, arc_buf_t *buf,
/* compressed bufs must always be assignable to their dbuf */
ASSERT3U(arc_get_compression(buf), ==, ZIO_COMPRESS_OFF);
ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED));
ASSERT(!arc_is_encrypted(buf));
dbuf_rele(db, FTAG);
dmu_write(os, object, offset, blksz, buf->b_data, tx);

15
module/zfs/dmu_recv.c
View File

@ -1235,13 +1235,11 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
* processed. However, for raw receives we manually set the
* maxblkid from the drr_maxblkid and so we must first free
* everything above that blkid to ensure the DMU is always
* consistent with itself. We will never free the first block
* of the object here because a maxblkid of 0 could indicate
* an object with a single block or one with no blocks.
* consistent with itself.
*/
if (rwa->raw && object != DMU_NEW_OBJECT) {
if (rwa->raw) {
err = dmu_free_long_range(rwa->os, drro->drr_object,
(drro->drr_maxblkid + 1) * doi.doi_data_block_size,
(drro->drr_maxblkid + 1) * drro->drr_blksz,
DMU_OBJECT_END);
if (err != 0)
return (SET_ERROR(EINVAL));
@ -1377,8 +1375,11 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
drro->drr_nlevels, tx));
/*
* Set the maxblkid. This will always succeed because
* we freed all blocks beyond the new maxblkid above.
* Set the maxblkid. We will never free the first block of
* an object here because a maxblkid of 0 could indicate
* an object with a single block or one with no blocks.
* This will always succeed because we freed all blocks
* beyond the new maxblkid above.
*/
VERIFY0(dmu_object_set_maxblkid(rwa->os, drro->drr_object,
drro->drr_maxblkid, tx));

11
module/zfs/dnode.c
View File

@ -689,9 +689,12 @@ dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize,
rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
dnode_setdirty(dn, tx);
if (dn->dn_datablksz != blocksize) {
ASSERT0(dn->dn_maxblkid);
ASSERT(BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) ||
dnode_block_freed(dn, 0));
/* change blocksize */
ASSERT(dn->dn_maxblkid == 0 &&
(BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) ||
dnode_block_freed(dn, 0)));
dnode_setdblksz(dn, blocksize);
dn->dn_next_blksz[tx->tx_txg&TXG_MASK] = blocksize;
}
if (dn->dn_bonuslen != bonuslen)
dn->dn_next_bonuslen[tx->tx_txg&TXG_MASK] = bonuslen;
@ -712,8 +715,6 @@ dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize,
}
rw_exit(&dn->dn_struct_rwlock);
VERIFY0(dnode_set_blksz(dn, blocksize, 0, tx));
/* change type */
dn->dn_type = ot;

3
tests/runfiles/linux.run
View File

@ -803,8 +803,7 @@ tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
'send-c_lz4_disabled', 'send-c_recv_lz4_disabled',
'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD',
'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
'send-c_recv_dedup', 'send_encrypted_files',
'send_encrypted_truncated_files', 'send_encrypted_heirarchy',
'send-c_recv_dedup', 'send_encrypted_files', 'send_encrypted_heirarchy',
'send_encrypted_props', 'send_freeobjects', 'send_realloc_dnode_size',
'send_holds', 'send_hole_birth', 'send_mixed_raw',
'send-wDR_encrypted_zvol']

1
tests/zfs-tests/tests/functional/rsend/Makefile.am
View File

@ -22,7 +22,6 @@ dist_pkgdata_SCRIPTS = \
rsend_022_pos.ksh \
rsend_024_pos.ksh \
send_encrypted_files.ksh \
send_encrypted_truncated_files.ksh \
send_encrypted_heirarchy.ksh \
send_encrypted_props.ksh \
send-cD.ksh \

43
tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh
View File

@ -22,8 +22,7 @@
#
# DESCRIPTION:
# Verify that a raw zfs send and receive can deal with several different
# types of file layouts.
#
#
# STRATEGY:
# 1. Create a new encrypted filesystem
@ -31,14 +30,18 @@
# 3. Add a small 512 byte file to the filesystem
# 4. Add a larger 32M file to the filesystem
# 5. Add a large sparse file to the filesystem
# 6. Add 1000 empty files to the filesystem
# 7. Add a file with a large xattr value
# 8. Use xattrtest to create files with random xattrs (with and without xattrs=on)
# 9. Take a snapshot of the filesystem
# 10. Remove the 1000 empty files to the filesystem
# 11. Take another snapshot of the filesystem
# 12. Send and receive both snapshots
# 13. Mount the filesystem and check the contents
# 6. Add a 3 files that are to be truncated later
# 7. Add 1000 empty files to the filesystem
# 8. Add a file with a large xattr value
# 9. Use xattrtest to create files with random xattrs (with and without xattrs=on)
# 10. Take a snapshot of the filesystem
# 11. Truncate one of the files from 32M to 128k
# 12. Truncate one of the files from 512k to 384k
# 13. Truncate one of the files from 512k to 0 to 384k
# 14. Remove the 1000 empty files to the filesystem
# 15. Take another snapshot of the filesystem
# 16. Send and receive both snapshots
# 17. Mount the filesystem and check the contents
#
verify_runnable "both"
@ -71,12 +74,15 @@ log_must eval "echo 'password' > $keyfile"
log_must zfs create -o encryption=on -o keyformat=passphrase \
-o keylocation=file://$keyfile $TESTPOOL/$TESTFS2
# Create files with varied layouts on disk
# Create files with vaired layouts on disk
log_must touch /$TESTPOOL/$TESTFS2/empty
log_must mkfile 512 /$TESTPOOL/$TESTFS2/small
log_must mkfile 32M /$TESTPOOL/$TESTFS2/full
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/sparse \
bs=512 count=1 seek=1048576 >/dev/null 2>&1
log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated
log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2
log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3
log_must mkdir -p /$TESTPOOL/$TESTFS2/dir
for i in {1..1000}; do
@ -95,10 +101,23 @@ log_must zfs set compression=on xattr=sa $TESTPOOL/$TESTFS2
log_must touch /$TESTPOOL/$TESTFS2/attrs
log_must eval "python -c 'print \"a\" * 4096' | \
attr -s bigval /$TESTPOOL/$TESTFS2/attrs"
log_must zfs set compression=off xattr=on $TESTPOOL/$TESTFS2
log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1
#
# Truncate files created in the first snapshot. The first tests
# truncating a large file to a single block. The second tests
# truncating one block off the end of a file without changing
# the required nlevels to hold it. The last tests handling
# of a maxblkid that is dropped and then raised again.
#
log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated
log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2
log_must truncate -s 0 /$TESTPOOL/$TESTFS2/truncated3
log_must zpool sync $TESTPOOL
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \
bs=128k count=3 iflag=fullblock
# Remove the empty files created in the first snapshot
for i in {1..1000}; do
log_must rm /$TESTPOOL/$TESTFS2/dir/file-$i

114
tests/zfs-tests/tests/functional/rsend/send_encrypted_truncated_files.ksh
View File

@ -1,114 +0,0 @@
#!/bin/ksh -p
#
# CDDL HEADER START
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
# CDDL HEADER END
#
#
# Copyright (c) 2018 by Datto Inc. All rights reserved.
#
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
#
# DESCRIPTION:
#
#
# STRATEGY:
# 1. Create a new encrypted filesystem
# 2. Add a 4 files that are to be truncated later
# 3. Take a snapshot of the filesystem
# 4. Truncate one of the files from 32M to 128k
# 5. Truncate one of the files from 512k to 384k
# 6. Truncate one of the files from 512k to 0 to 384k via reallocation
# 7. Truncate one of the files from 1k to 0 to 512b via reallocation
# 8. Take another snapshot of the filesystem
# 9. Send and receive both snapshots
# 10. Mount the filesystem and check the contents
#
verify_runnable "both"
function cleanup
{
datasetexists $TESTPOOL/$TESTFS2 && \
log_must zfs destroy -r $TESTPOOL/$TESTFS2
datasetexists $TESTPOOL/recv && \
log_must zfs destroy -r $TESTPOOL/recv
[[ -f $keyfile ]] && log_must rm $keyfile
[[ -f $sendfile ]] && log_must rm $sendfile
}
log_onexit cleanup
function recursive_cksum
{
find $1 -type f -exec sha256sum {} \; | \
sort -k 2 | awk '{ print $1 }' | sha256sum
}
log_assert "Verify 'zfs send -w' works with many different file layouts"
typeset keyfile=/$TESTPOOL/pkey
typeset sendfile=/$TESTPOOL/sendfile
typeset sendfile2=/$TESTPOOL/sendfile2
# Create an encrypted dataset
log_must eval "echo 'password' > $keyfile"
log_must zfs create -o encryption=on -o keyformat=passphrase \
-o keylocation=file://$keyfile $TESTPOOL/$TESTFS2
# Create files with varied layouts on disk
log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated
log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2
log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3
log_must mkfile 1024 /$TESTPOOL/$TESTFS2/truncated4
log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1
#
# Truncate files created in the first snapshot. The first tests
# truncating a large file to a single block. The second tests
# truncating one block off the end of a file without changing
# the required nlevels to hold it. The third tests handling
# of a maxblkid that is dropped and then raised again. The
# fourth tests an object that is truncated from a single block
# to a smaller single block.
#
log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated
log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2
log_must rm -f /$TESTPOOL/$TESTFS2/truncated3
log_must rm -f /$TESTPOOL/$TESTFS2/truncated4
log_must zpool sync $TESTPOOL
log_must zfs umount $TESTPOOL/$TESTFS2
log_must zfs mount $TESTPOOL/$TESTFS2
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \
bs=128k count=3 iflag=fullblock
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated4 \
bs=512 count=1 iflag=fullblock
log_must zfs snapshot $TESTPOOL/$TESTFS2@snap2
expected_cksum=$(recursive_cksum /$TESTPOOL/$TESTFS2)
log_must eval "zfs send -wp $TESTPOOL/$TESTFS2@snap1 > $sendfile"
log_must eval "zfs send -wp -i @snap1 $TESTPOOL/$TESTFS2@snap2 > $sendfile2"
log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile"
log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile2"
log_must zfs load-key $TESTPOOL/recv
log_must zfs mount -a
actual_cksum=$(recursive_cksum /$TESTPOOL/recv)
[[ "$expected_cksum" != "$actual_cksum" ]] && \
log_fail "Recursive checksums differ ($expected_cksum != $actual_cksum)"
log_pass "Verified 'zfs send -w' works with many different file layouts"