Revert "Fix issues with truncated files in raw sends"
This partially reverts commit 5dbf8b4ed
. This change resolved
the issues observed with truncated files in raw sends. However,
the required changes to dnode_allocate() introduced a regression
for non-raw streams which needs to be understood.
The additional debugging improvements from the original patch
were not reverted.
Reviewed-by: Tom Caputi <tcaputi@datto.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Issue #7378
Issue #8528
Issue #8540
Issue #8565
Close #8584
This commit is contained in:
parent
944a37248a
commit
d93d4b1acd
|
@ -1737,7 +1737,6 @@ dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset, arc_buf_t *buf,
|
||||||
/* compressed bufs must always be assignable to their dbuf */
|
/* compressed bufs must always be assignable to their dbuf */
|
||||||
ASSERT3U(arc_get_compression(buf), ==, ZIO_COMPRESS_OFF);
|
ASSERT3U(arc_get_compression(buf), ==, ZIO_COMPRESS_OFF);
|
||||||
ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED));
|
ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED));
|
||||||
ASSERT(!arc_is_encrypted(buf));
|
|
||||||
|
|
||||||
dbuf_rele(db, FTAG);
|
dbuf_rele(db, FTAG);
|
||||||
dmu_write(os, object, offset, blksz, buf->b_data, tx);
|
dmu_write(os, object, offset, blksz, buf->b_data, tx);
|
||||||
|
|
|
@ -1235,13 +1235,11 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
|
||||||
* processed. However, for raw receives we manually set the
|
* processed. However, for raw receives we manually set the
|
||||||
* maxblkid from the drr_maxblkid and so we must first free
|
* maxblkid from the drr_maxblkid and so we must first free
|
||||||
* everything above that blkid to ensure the DMU is always
|
* everything above that blkid to ensure the DMU is always
|
||||||
* consistent with itself. We will never free the first block
|
* consistent with itself.
|
||||||
* of the object here because a maxblkid of 0 could indicate
|
|
||||||
* an object with a single block or one with no blocks.
|
|
||||||
*/
|
*/
|
||||||
if (rwa->raw && object != DMU_NEW_OBJECT) {
|
if (rwa->raw) {
|
||||||
err = dmu_free_long_range(rwa->os, drro->drr_object,
|
err = dmu_free_long_range(rwa->os, drro->drr_object,
|
||||||
(drro->drr_maxblkid + 1) * doi.doi_data_block_size,
|
(drro->drr_maxblkid + 1) * drro->drr_blksz,
|
||||||
DMU_OBJECT_END);
|
DMU_OBJECT_END);
|
||||||
if (err != 0)
|
if (err != 0)
|
||||||
return (SET_ERROR(EINVAL));
|
return (SET_ERROR(EINVAL));
|
||||||
|
@ -1377,8 +1375,11 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
|
||||||
drro->drr_nlevels, tx));
|
drro->drr_nlevels, tx));
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Set the maxblkid. This will always succeed because
|
* Set the maxblkid. We will never free the first block of
|
||||||
* we freed all blocks beyond the new maxblkid above.
|
* an object here because a maxblkid of 0 could indicate
|
||||||
|
* an object with a single block or one with no blocks.
|
||||||
|
* This will always succeed because we freed all blocks
|
||||||
|
* beyond the new maxblkid above.
|
||||||
*/
|
*/
|
||||||
VERIFY0(dmu_object_set_maxblkid(rwa->os, drro->drr_object,
|
VERIFY0(dmu_object_set_maxblkid(rwa->os, drro->drr_object,
|
||||||
drro->drr_maxblkid, tx));
|
drro->drr_maxblkid, tx));
|
||||||
|
|
|
@ -689,9 +689,12 @@ dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize,
|
||||||
rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
|
rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
|
||||||
dnode_setdirty(dn, tx);
|
dnode_setdirty(dn, tx);
|
||||||
if (dn->dn_datablksz != blocksize) {
|
if (dn->dn_datablksz != blocksize) {
|
||||||
ASSERT0(dn->dn_maxblkid);
|
/* change blocksize */
|
||||||
ASSERT(BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) ||
|
ASSERT(dn->dn_maxblkid == 0 &&
|
||||||
dnode_block_freed(dn, 0));
|
(BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) ||
|
||||||
|
dnode_block_freed(dn, 0)));
|
||||||
|
dnode_setdblksz(dn, blocksize);
|
||||||
|
dn->dn_next_blksz[tx->tx_txg&TXG_MASK] = blocksize;
|
||||||
}
|
}
|
||||||
if (dn->dn_bonuslen != bonuslen)
|
if (dn->dn_bonuslen != bonuslen)
|
||||||
dn->dn_next_bonuslen[tx->tx_txg&TXG_MASK] = bonuslen;
|
dn->dn_next_bonuslen[tx->tx_txg&TXG_MASK] = bonuslen;
|
||||||
|
@ -712,8 +715,6 @@ dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize,
|
||||||
}
|
}
|
||||||
rw_exit(&dn->dn_struct_rwlock);
|
rw_exit(&dn->dn_struct_rwlock);
|
||||||
|
|
||||||
VERIFY0(dnode_set_blksz(dn, blocksize, 0, tx));
|
|
||||||
|
|
||||||
/* change type */
|
/* change type */
|
||||||
dn->dn_type = ot;
|
dn->dn_type = ot;
|
||||||
|
|
||||||
|
|
|
@ -803,8 +803,7 @@ tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
|
||||||
'send-c_lz4_disabled', 'send-c_recv_lz4_disabled',
|
'send-c_lz4_disabled', 'send-c_recv_lz4_disabled',
|
||||||
'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD',
|
'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD',
|
||||||
'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
|
'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
|
||||||
'send-c_recv_dedup', 'send_encrypted_files',
|
'send-c_recv_dedup', 'send_encrypted_files', 'send_encrypted_heirarchy',
|
||||||
'send_encrypted_truncated_files', 'send_encrypted_heirarchy',
|
|
||||||
'send_encrypted_props', 'send_freeobjects', 'send_realloc_dnode_size',
|
'send_encrypted_props', 'send_freeobjects', 'send_realloc_dnode_size',
|
||||||
'send_holds', 'send_hole_birth', 'send_mixed_raw',
|
'send_holds', 'send_hole_birth', 'send_mixed_raw',
|
||||||
'send-wDR_encrypted_zvol']
|
'send-wDR_encrypted_zvol']
|
||||||
|
|
|
@ -22,7 +22,6 @@ dist_pkgdata_SCRIPTS = \
|
||||||
rsend_022_pos.ksh \
|
rsend_022_pos.ksh \
|
||||||
rsend_024_pos.ksh \
|
rsend_024_pos.ksh \
|
||||||
send_encrypted_files.ksh \
|
send_encrypted_files.ksh \
|
||||||
send_encrypted_truncated_files.ksh \
|
|
||||||
send_encrypted_heirarchy.ksh \
|
send_encrypted_heirarchy.ksh \
|
||||||
send_encrypted_props.ksh \
|
send_encrypted_props.ksh \
|
||||||
send-cD.ksh \
|
send-cD.ksh \
|
||||||
|
|
|
@ -22,8 +22,7 @@
|
||||||
|
|
||||||
#
|
#
|
||||||
# DESCRIPTION:
|
# DESCRIPTION:
|
||||||
# Verify that a raw zfs send and receive can deal with several different
|
#
|
||||||
# types of file layouts.
|
|
||||||
#
|
#
|
||||||
# STRATEGY:
|
# STRATEGY:
|
||||||
# 1. Create a new encrypted filesystem
|
# 1. Create a new encrypted filesystem
|
||||||
|
@ -31,14 +30,18 @@
|
||||||
# 3. Add a small 512 byte file to the filesystem
|
# 3. Add a small 512 byte file to the filesystem
|
||||||
# 4. Add a larger 32M file to the filesystem
|
# 4. Add a larger 32M file to the filesystem
|
||||||
# 5. Add a large sparse file to the filesystem
|
# 5. Add a large sparse file to the filesystem
|
||||||
# 6. Add 1000 empty files to the filesystem
|
# 6. Add a 3 files that are to be truncated later
|
||||||
# 7. Add a file with a large xattr value
|
# 7. Add 1000 empty files to the filesystem
|
||||||
# 8. Use xattrtest to create files with random xattrs (with and without xattrs=on)
|
# 8. Add a file with a large xattr value
|
||||||
# 9. Take a snapshot of the filesystem
|
# 9. Use xattrtest to create files with random xattrs (with and without xattrs=on)
|
||||||
# 10. Remove the 1000 empty files to the filesystem
|
# 10. Take a snapshot of the filesystem
|
||||||
# 11. Take another snapshot of the filesystem
|
# 11. Truncate one of the files from 32M to 128k
|
||||||
# 12. Send and receive both snapshots
|
# 12. Truncate one of the files from 512k to 384k
|
||||||
# 13. Mount the filesystem and check the contents
|
# 13. Truncate one of the files from 512k to 0 to 384k
|
||||||
|
# 14. Remove the 1000 empty files to the filesystem
|
||||||
|
# 15. Take another snapshot of the filesystem
|
||||||
|
# 16. Send and receive both snapshots
|
||||||
|
# 17. Mount the filesystem and check the contents
|
||||||
#
|
#
|
||||||
|
|
||||||
verify_runnable "both"
|
verify_runnable "both"
|
||||||
|
@ -71,12 +74,15 @@ log_must eval "echo 'password' > $keyfile"
|
||||||
log_must zfs create -o encryption=on -o keyformat=passphrase \
|
log_must zfs create -o encryption=on -o keyformat=passphrase \
|
||||||
-o keylocation=file://$keyfile $TESTPOOL/$TESTFS2
|
-o keylocation=file://$keyfile $TESTPOOL/$TESTFS2
|
||||||
|
|
||||||
# Create files with varied layouts on disk
|
# Create files with vaired layouts on disk
|
||||||
log_must touch /$TESTPOOL/$TESTFS2/empty
|
log_must touch /$TESTPOOL/$TESTFS2/empty
|
||||||
log_must mkfile 512 /$TESTPOOL/$TESTFS2/small
|
log_must mkfile 512 /$TESTPOOL/$TESTFS2/small
|
||||||
log_must mkfile 32M /$TESTPOOL/$TESTFS2/full
|
log_must mkfile 32M /$TESTPOOL/$TESTFS2/full
|
||||||
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/sparse \
|
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/sparse \
|
||||||
bs=512 count=1 seek=1048576 >/dev/null 2>&1
|
bs=512 count=1 seek=1048576 >/dev/null 2>&1
|
||||||
|
log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated
|
||||||
|
log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2
|
||||||
|
log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3
|
||||||
|
|
||||||
log_must mkdir -p /$TESTPOOL/$TESTFS2/dir
|
log_must mkdir -p /$TESTPOOL/$TESTFS2/dir
|
||||||
for i in {1..1000}; do
|
for i in {1..1000}; do
|
||||||
|
@ -95,10 +101,23 @@ log_must zfs set compression=on xattr=sa $TESTPOOL/$TESTFS2
|
||||||
log_must touch /$TESTPOOL/$TESTFS2/attrs
|
log_must touch /$TESTPOOL/$TESTFS2/attrs
|
||||||
log_must eval "python -c 'print \"a\" * 4096' | \
|
log_must eval "python -c 'print \"a\" * 4096' | \
|
||||||
attr -s bigval /$TESTPOOL/$TESTFS2/attrs"
|
attr -s bigval /$TESTPOOL/$TESTFS2/attrs"
|
||||||
log_must zfs set compression=off xattr=on $TESTPOOL/$TESTFS2
|
|
||||||
|
|
||||||
log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1
|
log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1
|
||||||
|
|
||||||
|
#
|
||||||
|
# Truncate files created in the first snapshot. The first tests
|
||||||
|
# truncating a large file to a single block. The second tests
|
||||||
|
# truncating one block off the end of a file without changing
|
||||||
|
# the required nlevels to hold it. The last tests handling
|
||||||
|
# of a maxblkid that is dropped and then raised again.
|
||||||
|
#
|
||||||
|
log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated
|
||||||
|
log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2
|
||||||
|
log_must truncate -s 0 /$TESTPOOL/$TESTFS2/truncated3
|
||||||
|
log_must zpool sync $TESTPOOL
|
||||||
|
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \
|
||||||
|
bs=128k count=3 iflag=fullblock
|
||||||
|
|
||||||
# Remove the empty files created in the first snapshot
|
# Remove the empty files created in the first snapshot
|
||||||
for i in {1..1000}; do
|
for i in {1..1000}; do
|
||||||
log_must rm /$TESTPOOL/$TESTFS2/dir/file-$i
|
log_must rm /$TESTPOOL/$TESTFS2/dir/file-$i
|
||||||
|
|
|
@ -1,114 +0,0 @@
|
||||||
#!/bin/ksh -p
|
|
||||||
#
|
|
||||||
# CDDL HEADER START
|
|
||||||
#
|
|
||||||
# This file and its contents are supplied under the terms of the
|
|
||||||
# Common Development and Distribution License ("CDDL"), version 1.0.
|
|
||||||
# You may only use this file in accordance with the terms of version
|
|
||||||
# 1.0 of the CDDL.
|
|
||||||
#
|
|
||||||
# A full copy of the text of the CDDL should have accompanied this
|
|
||||||
# source. A copy of the CDDL is also available via the Internet at
|
|
||||||
# http://www.illumos.org/license/CDDL.
|
|
||||||
#
|
|
||||||
# CDDL HEADER END
|
|
||||||
#
|
|
||||||
|
|
||||||
#
|
|
||||||
# Copyright (c) 2018 by Datto Inc. All rights reserved.
|
|
||||||
#
|
|
||||||
|
|
||||||
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
|
|
||||||
|
|
||||||
#
|
|
||||||
# DESCRIPTION:
|
|
||||||
#
|
|
||||||
#
|
|
||||||
# STRATEGY:
|
|
||||||
# 1. Create a new encrypted filesystem
|
|
||||||
# 2. Add a 4 files that are to be truncated later
|
|
||||||
# 3. Take a snapshot of the filesystem
|
|
||||||
# 4. Truncate one of the files from 32M to 128k
|
|
||||||
# 5. Truncate one of the files from 512k to 384k
|
|
||||||
# 6. Truncate one of the files from 512k to 0 to 384k via reallocation
|
|
||||||
# 7. Truncate one of the files from 1k to 0 to 512b via reallocation
|
|
||||||
# 8. Take another snapshot of the filesystem
|
|
||||||
# 9. Send and receive both snapshots
|
|
||||||
# 10. Mount the filesystem and check the contents
|
|
||||||
#
|
|
||||||
|
|
||||||
verify_runnable "both"
|
|
||||||
|
|
||||||
function cleanup
|
|
||||||
{
|
|
||||||
datasetexists $TESTPOOL/$TESTFS2 && \
|
|
||||||
log_must zfs destroy -r $TESTPOOL/$TESTFS2
|
|
||||||
datasetexists $TESTPOOL/recv && \
|
|
||||||
log_must zfs destroy -r $TESTPOOL/recv
|
|
||||||
[[ -f $keyfile ]] && log_must rm $keyfile
|
|
||||||
[[ -f $sendfile ]] && log_must rm $sendfile
|
|
||||||
}
|
|
||||||
log_onexit cleanup
|
|
||||||
|
|
||||||
function recursive_cksum
|
|
||||||
{
|
|
||||||
find $1 -type f -exec sha256sum {} \; | \
|
|
||||||
sort -k 2 | awk '{ print $1 }' | sha256sum
|
|
||||||
}
|
|
||||||
|
|
||||||
log_assert "Verify 'zfs send -w' works with many different file layouts"
|
|
||||||
|
|
||||||
typeset keyfile=/$TESTPOOL/pkey
|
|
||||||
typeset sendfile=/$TESTPOOL/sendfile
|
|
||||||
typeset sendfile2=/$TESTPOOL/sendfile2
|
|
||||||
|
|
||||||
# Create an encrypted dataset
|
|
||||||
log_must eval "echo 'password' > $keyfile"
|
|
||||||
log_must zfs create -o encryption=on -o keyformat=passphrase \
|
|
||||||
-o keylocation=file://$keyfile $TESTPOOL/$TESTFS2
|
|
||||||
|
|
||||||
# Create files with varied layouts on disk
|
|
||||||
log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated
|
|
||||||
log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2
|
|
||||||
log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3
|
|
||||||
log_must mkfile 1024 /$TESTPOOL/$TESTFS2/truncated4
|
|
||||||
|
|
||||||
log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1
|
|
||||||
|
|
||||||
#
|
|
||||||
# Truncate files created in the first snapshot. The first tests
|
|
||||||
# truncating a large file to a single block. The second tests
|
|
||||||
# truncating one block off the end of a file without changing
|
|
||||||
# the required nlevels to hold it. The third tests handling
|
|
||||||
# of a maxblkid that is dropped and then raised again. The
|
|
||||||
# fourth tests an object that is truncated from a single block
|
|
||||||
# to a smaller single block.
|
|
||||||
#
|
|
||||||
log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated
|
|
||||||
log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2
|
|
||||||
log_must rm -f /$TESTPOOL/$TESTFS2/truncated3
|
|
||||||
log_must rm -f /$TESTPOOL/$TESTFS2/truncated4
|
|
||||||
log_must zpool sync $TESTPOOL
|
|
||||||
log_must zfs umount $TESTPOOL/$TESTFS2
|
|
||||||
log_must zfs mount $TESTPOOL/$TESTFS2
|
|
||||||
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \
|
|
||||||
bs=128k count=3 iflag=fullblock
|
|
||||||
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated4 \
|
|
||||||
bs=512 count=1 iflag=fullblock
|
|
||||||
|
|
||||||
log_must zfs snapshot $TESTPOOL/$TESTFS2@snap2
|
|
||||||
expected_cksum=$(recursive_cksum /$TESTPOOL/$TESTFS2)
|
|
||||||
|
|
||||||
log_must eval "zfs send -wp $TESTPOOL/$TESTFS2@snap1 > $sendfile"
|
|
||||||
log_must eval "zfs send -wp -i @snap1 $TESTPOOL/$TESTFS2@snap2 > $sendfile2"
|
|
||||||
|
|
||||||
log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile"
|
|
||||||
log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile2"
|
|
||||||
log_must zfs load-key $TESTPOOL/recv
|
|
||||||
|
|
||||||
log_must zfs mount -a
|
|
||||||
actual_cksum=$(recursive_cksum /$TESTPOOL/recv)
|
|
||||||
[[ "$expected_cksum" != "$actual_cksum" ]] && \
|
|
||||||
log_fail "Recursive checksums differ ($expected_cksum != $actual_cksum)"
|
|
||||||
|
|
||||||
log_pass "Verified 'zfs send -w' works with many different file layouts"
|
|
Loading…
Reference in New Issue