diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c index 3d8423af33..2197032312 100644 --- a/module/zfs/dmu.c +++ b/module/zfs/dmu.c @@ -1737,6 +1737,7 @@ dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset, arc_buf_t *buf, /* compressed bufs must always be assignable to their dbuf */ ASSERT3U(arc_get_compression(buf), ==, ZIO_COMPRESS_OFF); ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED)); + ASSERT(!arc_is_encrypted(buf)); dbuf_rele(db, FTAG); dmu_write(os, object, offset, blksz, buf->b_data, tx); diff --git a/module/zfs/dmu_recv.c b/module/zfs/dmu_recv.c index e49a0f4aa4..e534540cb5 100644 --- a/module/zfs/dmu_recv.c +++ b/module/zfs/dmu_recv.c @@ -1235,11 +1235,13 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro, * processed. However, for raw receives we manually set the * maxblkid from the drr_maxblkid and so we must first free * everything above that blkid to ensure the DMU is always - * consistent with itself. + * consistent with itself. We will never free the first block + * of the object here because a maxblkid of 0 could indicate + * an object with a single block or one with no blocks. */ - if (rwa->raw) { + if (rwa->raw && object != DMU_NEW_OBJECT) { err = dmu_free_long_range(rwa->os, drro->drr_object, - (drro->drr_maxblkid + 1) * drro->drr_blksz, + (drro->drr_maxblkid + 1) * doi.doi_data_block_size, DMU_OBJECT_END); if (err != 0) return (SET_ERROR(EINVAL)); @@ -1375,11 +1377,8 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro, drro->drr_nlevels, tx)); /* - * Set the maxblkid. We will never free the first block of - * an object here because a maxblkid of 0 could indicate - * an object with a single block or one with no blocks. - * This will always succeed because we freed all blocks - * beyond the new maxblkid above. + * Set the maxblkid. This will always succeed because + * we freed all blocks beyond the new maxblkid above. */ VERIFY0(dmu_object_set_maxblkid(rwa->os, drro->drr_object, drro->drr_maxblkid, tx)); @@ -2185,7 +2184,7 @@ dprintf_drr(struct receive_record_arg *rrd, int err) { struct drr_write *drrw = &rrd->header.drr_u.drr_write; dprintf("drr_type = WRITE obj = %llu type = %u offset = %llu " - "lsize = %llu cksumtype = %u cksumflags = %u " + "lsize = %llu cksumtype = %u flags = %u " "compress = %u psize = %llu err = %d\n", drrw->drr_object, drrw->drr_type, drrw->drr_offset, drrw->drr_logical_size, drrw->drr_checksumtype, @@ -2200,7 +2199,7 @@ dprintf_drr(struct receive_record_arg *rrd, int err) dprintf("drr_type = WRITE_BYREF obj = %llu offset = %llu " "length = %llu toguid = %llx refguid = %llx " "refobject = %llu refoffset = %llu cksumtype = %u " - "cksumflags = %u err = %d\n", + "flags = %u err = %d\n", drrwbr->drr_object, drrwbr->drr_offset, drrwbr->drr_length, drrwbr->drr_toguid, drrwbr->drr_refguid, drrwbr->drr_refobject, @@ -2236,6 +2235,16 @@ dprintf_drr(struct receive_record_arg *rrd, int err) "err = %d\n", drrs->drr_object, drrs->drr_length, err); break; } + case DRR_OBJECT_RANGE: + { + struct drr_object_range *drror = + &rrd->header.drr_u.drr_object_range; + dprintf("drr_type = OBJECT_RANGE firstobj = %llu " + "numslots = %llu flags = %u err = %d\n", + drror->drr_firstobj, drror->drr_numslots, + drror->drr_flags, err); + break; + } default: return; } @@ -2319,10 +2328,11 @@ receive_process_record(struct receive_writer_arg *rwa, { struct drr_object_range *drror = &rrd->header.drr_u.drr_object_range; - return (receive_object_range(rwa, drror)); + err = receive_object_range(rwa, drror); + break; } default: - return (SET_ERROR(EINVAL)); + err = (SET_ERROR(EINVAL)); } if (err != 0) diff --git a/module/zfs/dnode.c b/module/zfs/dnode.c index 2903bc78d8..952ec95ae5 100644 --- a/module/zfs/dnode.c +++ b/module/zfs/dnode.c @@ -689,12 +689,9 @@ dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, rw_enter(&dn->dn_struct_rwlock, RW_WRITER); dnode_setdirty(dn, tx); if (dn->dn_datablksz != blocksize) { - /* change blocksize */ - ASSERT(dn->dn_maxblkid == 0 && - (BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) || - dnode_block_freed(dn, 0))); - dnode_setdblksz(dn, blocksize); - dn->dn_next_blksz[tx->tx_txg&TXG_MASK] = blocksize; + ASSERT0(dn->dn_maxblkid); + ASSERT(BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) || + dnode_block_freed(dn, 0)); } if (dn->dn_bonuslen != bonuslen) dn->dn_next_bonuslen[tx->tx_txg&TXG_MASK] = bonuslen; @@ -715,6 +712,8 @@ dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, } rw_exit(&dn->dn_struct_rwlock); + VERIFY0(dnode_set_blksz(dn, blocksize, 0, tx)); + /* change type */ dn->dn_type = ot; diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 9537798a92..211daaf27a 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -792,7 +792,8 @@ tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'send-c_lz4_disabled', 'send-c_recv_lz4_disabled', 'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD', 'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize', - 'send-c_recv_dedup', 'send_encrypted_files', 'send_encrypted_heirarchy', + 'send-c_recv_dedup', 'send_encrypted_files', + 'send_encrypted_truncated_files', 'send_encrypted_heirarchy', 'send_encrypted_props', 'send_freeobjects', 'send_realloc_dnode_size', 'send_holds', 'send_hole_birth', 'send_mixed_raw', 'send-wDR_encrypted_zvol'] diff --git a/tests/zfs-tests/tests/functional/rsend/Makefile.am b/tests/zfs-tests/tests/functional/rsend/Makefile.am index b0c68c5be5..322f1521f2 100644 --- a/tests/zfs-tests/tests/functional/rsend/Makefile.am +++ b/tests/zfs-tests/tests/functional/rsend/Makefile.am @@ -22,6 +22,7 @@ dist_pkgdata_SCRIPTS = \ rsend_022_pos.ksh \ rsend_024_pos.ksh \ send_encrypted_files.ksh \ + send_encrypted_truncated_files.ksh \ send_encrypted_heirarchy.ksh \ send_encrypted_props.ksh \ send-cD.ksh \ diff --git a/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh b/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh index d981aa3fd2..0156bc589a 100755 --- a/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh +++ b/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh @@ -22,7 +22,8 @@ # # DESCRIPTION: -# +# Verify that a raw zfs send and receive can deal with several different +# types of file layouts. # # STRATEGY: # 1. Create a new encrypted filesystem @@ -30,18 +31,14 @@ # 3. Add a small 512 byte file to the filesystem # 4. Add a larger 32M file to the filesystem # 5. Add a large sparse file to the filesystem -# 6. Add a 3 files that are to be truncated later -# 7. Add 1000 empty files to the filesystem -# 8. Add a file with a large xattr value -# 9. Use xattrtest to create files with random xattrs (with and without xattrs=on) -# 10. Take a snapshot of the filesystem -# 11. Truncate one of the files from 32M to 128k -# 12. Truncate one of the files from 512k to 384k -# 13. Truncate one of the files from 512k to 0 to 384k -# 14. Remove the 1000 empty files to the filesystem -# 15. Take another snapshot of the filesystem -# 16. Send and receive both snapshots -# 17. Mount the filesystem and check the contents +# 6. Add 1000 empty files to the filesystem +# 7. Add a file with a large xattr value +# 8. Use xattrtest to create files with random xattrs (with and without xattrs=on) +# 9. Take a snapshot of the filesystem +# 10. Remove the 1000 empty files to the filesystem +# 11. Take another snapshot of the filesystem +# 12. Send and receive both snapshots +# 13. Mount the filesystem and check the contents # verify_runnable "both" @@ -74,15 +71,12 @@ log_must eval "echo 'password' > $keyfile" log_must zfs create -o encryption=on -o keyformat=passphrase \ -o keylocation=file://$keyfile $TESTPOOL/$TESTFS2 -# Create files with vaired layouts on disk +# Create files with varied layouts on disk log_must touch /$TESTPOOL/$TESTFS2/empty log_must mkfile 512 /$TESTPOOL/$TESTFS2/small log_must mkfile 32M /$TESTPOOL/$TESTFS2/full log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/sparse \ bs=512 count=1 seek=1048576 >/dev/null 2>&1 -log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated -log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2 -log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3 log_must mkdir -p /$TESTPOOL/$TESTFS2/dir for i in {1..1000}; do @@ -101,23 +95,10 @@ log_must zfs set compression=on xattr=sa $TESTPOOL/$TESTFS2 log_must touch /$TESTPOOL/$TESTFS2/attrs log_must eval "python -c 'print \"a\" * 4096' | \ attr -s bigval /$TESTPOOL/$TESTFS2/attrs" +log_must zfs set compression=off xattr=on $TESTPOOL/$TESTFS2 log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1 -# -# Truncate files created in the first snapshot. The first tests -# truncating a large file to a single block. The second tests -# truncating one block off the end of a file without changing -# the required nlevels to hold it. The last tests handling -# of a maxblkid that is dropped and then raised again. -# -log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated -log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2 -log_must truncate -s 0 /$TESTPOOL/$TESTFS2/truncated3 -log_must zpool sync $TESTPOOL -log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \ - bs=128k count=3 iflag=fullblock - # Remove the empty files created in the first snapshot for i in {1..1000}; do log_must rm /$TESTPOOL/$TESTFS2/dir/file-$i diff --git a/tests/zfs-tests/tests/functional/rsend/send_encrypted_truncated_files.ksh b/tests/zfs-tests/tests/functional/rsend/send_encrypted_truncated_files.ksh new file mode 100755 index 0000000000..8578d57688 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send_encrypted_truncated_files.ksh @@ -0,0 +1,114 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Datto Inc. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# +# +# STRATEGY: +# 1. Create a new encrypted filesystem +# 2. Add a 4 files that are to be truncated later +# 3. Take a snapshot of the filesystem +# 4. Truncate one of the files from 32M to 128k +# 5. Truncate one of the files from 512k to 384k +# 6. Truncate one of the files from 512k to 0 to 384k via reallocation +# 7. Truncate one of the files from 1k to 0 to 512b via reallocation +# 8. Take another snapshot of the filesystem +# 9. Send and receive both snapshots +# 10. Mount the filesystem and check the contents +# + +verify_runnable "both" + +function cleanup +{ + datasetexists $TESTPOOL/$TESTFS2 && \ + log_must zfs destroy -r $TESTPOOL/$TESTFS2 + datasetexists $TESTPOOL/recv && \ + log_must zfs destroy -r $TESTPOOL/recv + [[ -f $keyfile ]] && log_must rm $keyfile + [[ -f $sendfile ]] && log_must rm $sendfile +} +log_onexit cleanup + +function recursive_cksum +{ + find $1 -type f -exec sha256sum {} \; | \ + sort -k 2 | awk '{ print $1 }' | sha256sum +} + +log_assert "Verify 'zfs send -w' works with many different file layouts" + +typeset keyfile=/$TESTPOOL/pkey +typeset sendfile=/$TESTPOOL/sendfile +typeset sendfile2=/$TESTPOOL/sendfile2 + +# Create an encrypted dataset +log_must eval "echo 'password' > $keyfile" +log_must zfs create -o encryption=on -o keyformat=passphrase \ + -o keylocation=file://$keyfile $TESTPOOL/$TESTFS2 + +# Create files with varied layouts on disk +log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated +log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2 +log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3 +log_must mkfile 1024 /$TESTPOOL/$TESTFS2/truncated4 + +log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1 + +# +# Truncate files created in the first snapshot. The first tests +# truncating a large file to a single block. The second tests +# truncating one block off the end of a file without changing +# the required nlevels to hold it. The third tests handling +# of a maxblkid that is dropped and then raised again. The +# fourth tests an object that is truncated from a single block +# to a smaller single block. +# +log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated +log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2 +log_must rm -f /$TESTPOOL/$TESTFS2/truncated3 +log_must rm -f /$TESTPOOL/$TESTFS2/truncated4 +log_must zpool sync $TESTPOOL +log_must zfs umount $TESTPOOL/$TESTFS2 +log_must zfs mount $TESTPOOL/$TESTFS2 +log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \ + bs=128k count=3 iflag=fullblock +log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated4 \ + bs=512 count=1 iflag=fullblock + +log_must zfs snapshot $TESTPOOL/$TESTFS2@snap2 +expected_cksum=$(recursive_cksum /$TESTPOOL/$TESTFS2) + +log_must eval "zfs send -wp $TESTPOOL/$TESTFS2@snap1 > $sendfile" +log_must eval "zfs send -wp -i @snap1 $TESTPOOL/$TESTFS2@snap2 > $sendfile2" + +log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile" +log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile2" +log_must zfs load-key $TESTPOOL/recv + +log_must zfs mount -a +actual_cksum=$(recursive_cksum /$TESTPOOL/recv) +[[ "$expected_cksum" != "$actual_cksum" ]] && \ + log_fail "Recursive checksums differ ($expected_cksum != $actual_cksum)" + +log_pass "Verified 'zfs send -w' works with many different file layouts"