2008-11-20 20:01:55 +00:00
|
|
|
/*
|
|
|
|
* CDDL HEADER START
|
|
|
|
*
|
|
|
|
* The contents of this file are subject to the terms of the
|
|
|
|
* Common Development and Distribution License (the "License").
|
|
|
|
* You may not use this file except in compliance with the License.
|
|
|
|
*
|
|
|
|
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
|
|
|
* or http://www.opensolaris.org/os/licensing.
|
|
|
|
* See the License for the specific language governing permissions
|
|
|
|
* and limitations under the License.
|
|
|
|
*
|
|
|
|
* When distributing Covered Code, include this CDDL HEADER in each
|
|
|
|
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
|
|
|
* If applicable, add the following below this CDDL HEADER, with the
|
|
|
|
* fields enclosed by brackets "[]" replaced with your own identifying
|
|
|
|
* information: Portions Copyright [yyyy] [name of copyright owner]
|
|
|
|
*
|
|
|
|
* CDDL HEADER END
|
|
|
|
*/
|
|
|
|
/*
|
2010-05-28 20:45:14 +00:00
|
|
|
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
|
Improve zfs receive performance with lightweight write
The performance of `zfs receive` can be bottlenecked on the CPU consumed
by the `receive_writer` thread, especially when receiving streams with
small compressed block sizes. Much of the CPU is spent creating and
destroying dbuf's and arc buf's, one for each `WRITE` record in the send
stream.
This commit introduces the concept of "lightweight writes", which allows
`zfs receive` to write to the DMU by providing an ABD, and instantiating
only a new type of `dbuf_dirty_record_t`. The dbuf and arc buf for this
"dirty leaf block" are not instantiated.
Because there is no dbuf with the dirty data, this mechanism doesn't
support reading from "lightweight-dirty" blocks (they would see the
on-disk state rather than the dirty data). Since the dedup-receive code
has been removed, `zfs receive` is write-only, so this works fine.
Because there are no arc bufs for the received data, the received data
is no longer cached in the ARC.
Testing a receive of a stream with average compressed block size of 4KB,
this commit improves performance by 50%, while also reducing CPU usage
by 50% of a CPU. On a per-block basis, CPU consumed by receive_writer()
and dbuf_evict() is now 1/7th (14%) of what it was.
Baseline: 450MB/s, CPU in receive_writer() 40% + dbuf_evict() 35%
New: 670MB/s, CPU in receive_writer() 17% + dbuf_evict() 0%
The code is also restructured in a few ways:
Added a `dr_dnode` field to the dbuf_dirty_record_t. This simplifies
some existing code that no longer needs `DB_DNODE_ENTER()` and related
routines. The new field is needed by the lightweight-type dirty record.
To ensure that the `dr_dnode` field remains valid until the dirty record
is freed, we have to ensure that the `dnode_move()` doesn't relocate the
dnode_t. To do this we keep a hold on the dnode until it's zio's have
completed. This is already done by the user-accounting code
(`userquota_updates_task()`), this commit extends that so that it always
keeps the dnode hold until zio completion (see `dnode_rele_task()`).
`dn_dirty_txg` was previously zeroed when the dnode was synced. This
was not necessary, since its meaning can be "when was this dnode last
dirtied". This change simplifies the new `dnode_rele_task()` code.
Removed some dead code related to `DRR_WRITE_BYREF` (dedup receive).
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Paul Dagnelie <pcd@delphix.com>
Reviewed-by: George Wilson <gwilson@delphix.com>
Signed-off-by: Matthew Ahrens <mahrens@delphix.com>
Closes #11105
2020-12-11 18:26:02 +00:00
|
|
|
* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
|
2013-08-01 20:02:10 +00:00
|
|
|
* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
|
2015-04-02 03:44:32 +00:00
|
|
|
* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
|
2008-11-20 20:01:55 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _SYS_DBUF_H
|
|
|
|
#define _SYS_DBUF_H
|
|
|
|
|
|
|
|
#include <sys/dmu.h>
|
|
|
|
#include <sys/spa.h>
|
|
|
|
#include <sys/txg.h>
|
|
|
|
#include <sys/zio.h>
|
|
|
|
#include <sys/arc.h>
|
|
|
|
#include <sys/zfs_context.h>
|
2020-07-29 23:35:33 +00:00
|
|
|
#include <sys/zfs_refcount.h>
|
2010-08-26 21:24:34 +00:00
|
|
|
#include <sys/zrlock.h>
|
2016-06-02 04:04:53 +00:00
|
|
|
#include <sys/multilist.h>
|
2008-11-20 20:01:55 +00:00
|
|
|
|
|
|
|
#ifdef __cplusplus
|
|
|
|
extern "C" {
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#define IN_DMU_SYNC 2
|
|
|
|
|
|
|
|
/*
|
|
|
|
* define flags for dbuf_read
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define DB_RF_MUST_SUCCEED (1 << 0)
|
|
|
|
#define DB_RF_CANFAIL (1 << 1)
|
|
|
|
#define DB_RF_HAVESTRUCT (1 << 2)
|
|
|
|
#define DB_RF_NOPREFETCH (1 << 3)
|
|
|
|
#define DB_RF_NEVERWAIT (1 << 4)
|
|
|
|
#define DB_RF_CACHED (1 << 5)
|
Native Encryption for ZFS on Linux
This change incorporates three major pieces:
The first change is a keystore that manages wrapping
and encryption keys for encrypted datasets. These
commands mostly involve manipulating the new
DSL Crypto Key ZAP Objects that live in the MOS. Each
encrypted dataset has its own DSL Crypto Key that is
protected with a user's key. This level of indirection
allows users to change their keys without re-encrypting
their entire datasets. The change implements the new
subcommands "zfs load-key", "zfs unload-key" and
"zfs change-key" which allow the user to manage their
encryption keys and settings. In addition, several new
flags and properties have been added to allow dataset
creation and to make mounting and unmounting more
convenient.
The second piece of this patch provides the ability to
encrypt, decyrpt, and authenticate protected datasets.
Each object set maintains a Merkel tree of Message
Authentication Codes that protect the lower layers,
similarly to how checksums are maintained. This part
impacts the zio layer, which handles the actual
encryption and generation of MACs, as well as the ARC
and DMU, which need to be able to handle encrypted
buffers and protected data.
The last addition is the ability to do raw, encrypted
sends and receives. The idea here is to send raw
encrypted and compressed data and receive it exactly
as is on a backup system. This means that the dataset
on the receiving system is protected using the same
user key that is in use on the sending side. By doing
so, datasets can be efficiently backed up to an
untrusted system without fear of data being
compromised.
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Jorgen Lundman <lundman@lundman.net>
Signed-off-by: Tom Caputi <tcaputi@datto.com>
Closes #494
Closes #5769
2017-08-14 17:36:48 +00:00
|
|
|
#define DB_RF_NO_DECRYPT (1 << 6)
|
2008-11-20 20:01:55 +00:00
|
|
|
|
|
|
|
/*
|
2008-12-03 20:09:06 +00:00
|
|
|
* The simplified state transition diagram for dbufs looks like:
|
2008-11-20 20:01:55 +00:00
|
|
|
*
|
|
|
|
* +----> READ ----+
|
|
|
|
* | |
|
|
|
|
* | V
|
|
|
|
* (alloc)-->UNCACHED CACHED-->EVICTING-->(free)
|
2008-12-03 20:09:06 +00:00
|
|
|
* | ^ ^
|
|
|
|
* | | |
|
|
|
|
* +----> FILL ----+ |
|
|
|
|
* | |
|
|
|
|
* | |
|
|
|
|
* +--------> NOFILL -------+
|
2015-04-01 15:10:58 +00:00
|
|
|
*
|
|
|
|
* DB_SEARCH is an invalid state for a dbuf. It is used by dbuf_free_range
|
|
|
|
* to find all dbufs in a range of a dnode and must be less than any other
|
|
|
|
* dbuf_states_t (see comment on dn_dbufs in dnode.h).
|
2008-11-20 20:01:55 +00:00
|
|
|
*/
|
|
|
|
typedef enum dbuf_states {
|
2015-04-01 15:10:58 +00:00
|
|
|
DB_SEARCH = -1,
|
2008-11-20 20:01:55 +00:00
|
|
|
DB_UNCACHED,
|
|
|
|
DB_FILL,
|
2008-12-03 20:09:06 +00:00
|
|
|
DB_NOFILL,
|
2008-11-20 20:01:55 +00:00
|
|
|
DB_READ,
|
|
|
|
DB_CACHED,
|
|
|
|
DB_EVICTING
|
|
|
|
} dbuf_states_t;
|
|
|
|
|
2018-07-10 17:49:50 +00:00
|
|
|
typedef enum dbuf_cached_state {
|
|
|
|
DB_NO_CACHE = -1,
|
|
|
|
DB_DBUF_CACHE,
|
|
|
|
DB_DBUF_METADATA_CACHE,
|
|
|
|
DB_CACHE_MAX
|
|
|
|
} dbuf_cached_state_t;
|
|
|
|
|
2008-11-20 20:01:55 +00:00
|
|
|
struct dnode;
|
|
|
|
struct dmu_tx;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* level = 0 means the user data
|
|
|
|
* level = 1 means the single indirect block
|
|
|
|
* etc.
|
|
|
|
*/
|
|
|
|
|
|
|
|
struct dmu_buf_impl;
|
|
|
|
|
|
|
|
typedef enum override_states {
|
|
|
|
DR_NOT_OVERRIDDEN,
|
|
|
|
DR_IN_DMU_SYNC,
|
|
|
|
DR_OVERRIDDEN
|
|
|
|
} override_states_t;
|
|
|
|
|
2019-07-08 20:18:50 +00:00
|
|
|
typedef enum db_lock_type {
|
|
|
|
DLT_NONE,
|
|
|
|
DLT_PARENT,
|
|
|
|
DLT_OBJSET
|
|
|
|
} db_lock_type_t;
|
|
|
|
|
2008-11-20 20:01:55 +00:00
|
|
|
typedef struct dbuf_dirty_record {
|
|
|
|
/* link on our parents dirty list */
|
|
|
|
list_node_t dr_dirty_node;
|
|
|
|
|
|
|
|
/* transaction group this data will sync in */
|
|
|
|
uint64_t dr_txg;
|
|
|
|
|
|
|
|
/* zio of outstanding write IO */
|
|
|
|
zio_t *dr_zio;
|
|
|
|
|
|
|
|
/* pointer back to our dbuf */
|
|
|
|
struct dmu_buf_impl *dr_dbuf;
|
|
|
|
|
2020-02-05 19:07:19 +00:00
|
|
|
/* list link for dbuf dirty records */
|
|
|
|
list_node_t dr_dbuf_node;
|
2008-11-20 20:01:55 +00:00
|
|
|
|
Improve zfs receive performance with lightweight write
The performance of `zfs receive` can be bottlenecked on the CPU consumed
by the `receive_writer` thread, especially when receiving streams with
small compressed block sizes. Much of the CPU is spent creating and
destroying dbuf's and arc buf's, one for each `WRITE` record in the send
stream.
This commit introduces the concept of "lightweight writes", which allows
`zfs receive` to write to the DMU by providing an ABD, and instantiating
only a new type of `dbuf_dirty_record_t`. The dbuf and arc buf for this
"dirty leaf block" are not instantiated.
Because there is no dbuf with the dirty data, this mechanism doesn't
support reading from "lightweight-dirty" blocks (they would see the
on-disk state rather than the dirty data). Since the dedup-receive code
has been removed, `zfs receive` is write-only, so this works fine.
Because there are no arc bufs for the received data, the received data
is no longer cached in the ARC.
Testing a receive of a stream with average compressed block size of 4KB,
this commit improves performance by 50%, while also reducing CPU usage
by 50% of a CPU. On a per-block basis, CPU consumed by receive_writer()
and dbuf_evict() is now 1/7th (14%) of what it was.
Baseline: 450MB/s, CPU in receive_writer() 40% + dbuf_evict() 35%
New: 670MB/s, CPU in receive_writer() 17% + dbuf_evict() 0%
The code is also restructured in a few ways:
Added a `dr_dnode` field to the dbuf_dirty_record_t. This simplifies
some existing code that no longer needs `DB_DNODE_ENTER()` and related
routines. The new field is needed by the lightweight-type dirty record.
To ensure that the `dr_dnode` field remains valid until the dirty record
is freed, we have to ensure that the `dnode_move()` doesn't relocate the
dnode_t. To do this we keep a hold on the dnode until it's zio's have
completed. This is already done by the user-accounting code
(`userquota_updates_task()`), this commit extends that so that it always
keeps the dnode hold until zio completion (see `dnode_rele_task()`).
`dn_dirty_txg` was previously zeroed when the dnode was synced. This
was not necessary, since its meaning can be "when was this dnode last
dirtied". This change simplifies the new `dnode_rele_task()` code.
Removed some dead code related to `DRR_WRITE_BYREF` (dedup receive).
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Paul Dagnelie <pcd@delphix.com>
Reviewed-by: George Wilson <gwilson@delphix.com>
Signed-off-by: Matthew Ahrens <mahrens@delphix.com>
Closes #11105
2020-12-11 18:26:02 +00:00
|
|
|
/*
|
|
|
|
* The dnode we are part of. Note that the dnode can not be moved or
|
|
|
|
* evicted due to the hold that's added by dnode_setdirty() or
|
|
|
|
* dmu_objset_sync_dnodes(), and released by dnode_rele_task() or
|
|
|
|
* userquota_updates_task(). This hold is necessary for
|
|
|
|
* dirty_lightweight_leaf-type dirty records, which don't have a hold
|
|
|
|
* on a dbuf.
|
|
|
|
*/
|
|
|
|
dnode_t *dr_dnode;
|
|
|
|
|
2008-11-20 20:01:55 +00:00
|
|
|
/* pointer to parent dirty record */
|
|
|
|
struct dbuf_dirty_record *dr_parent;
|
|
|
|
|
Illumos #4045 write throttle & i/o scheduler performance work
4045 zfs write throttle & i/o scheduler performance work
1. The ZFS i/o scheduler (vdev_queue.c) now divides i/os into 5 classes: sync
read, sync write, async read, async write, and scrub/resilver. The scheduler
issues a number of concurrent i/os from each class to the device. Once a class
has been selected, an i/o is selected from this class using either an elevator
algorithem (async, scrub classes) or FIFO (sync classes). The number of
concurrent async write i/os is tuned dynamically based on i/o load, to achieve
good sync i/o latency when there is not a high load of writes, and good write
throughput when there is. See the block comment in vdev_queue.c (reproduced
below) for more details.
2. The write throttle (dsl_pool_tempreserve_space() and
txg_constrain_throughput()) is rewritten to produce much more consistent delays
when under constant load. The new write throttle is based on the amount of
dirty data, rather than guesses about future performance of the system. When
there is a lot of dirty data, each transaction (e.g. write() syscall) will be
delayed by the same small amount. This eliminates the "brick wall of wait"
that the old write throttle could hit, causing all transactions to wait several
seconds until the next txg opens. One of the keys to the new write throttle is
decrementing the amount of dirty data as i/o completes, rather than at the end
of spa_sync(). Note that the write throttle is only applied once the i/o
scheduler is issuing the maximum number of outstanding async writes. See the
block comments in dsl_pool.c and above dmu_tx_delay() (reproduced below) for
more details.
This diff has several other effects, including:
* the commonly-tuned global variable zfs_vdev_max_pending has been removed;
use per-class zfs_vdev_*_max_active values or zfs_vdev_max_active instead.
* the size of each txg (meaning the amount of dirty data written, and thus the
time it takes to write out) is now controlled differently. There is no longer
an explicit time goal; the primary determinant is amount of dirty data.
Systems that are under light or medium load will now often see that a txg is
always syncing, but the impact to performance (e.g. read latency) is minimal.
Tune zfs_dirty_data_max and zfs_dirty_data_sync to control this.
* zio_taskq_batch_pct = 75 -- Only use 75% of all CPUs for compression,
checksum, etc. This improves latency by not allowing these CPU-intensive tasks
to consume all CPU (on machines with at least 4 CPU's; the percentage is
rounded up).
--matt
APPENDIX: problems with the current i/o scheduler
The current ZFS i/o scheduler (vdev_queue.c) is deadline based. The problem
with this is that if there are always i/os pending, then certain classes of
i/os can see very long delays.
For example, if there are always synchronous reads outstanding, then no async
writes will be serviced until they become "past due". One symptom of this
situation is that each pass of the txg sync takes at least several seconds
(typically 3 seconds).
If many i/os become "past due" (their deadline is in the past), then we must
service all of these overdue i/os before any new i/os. This happens when we
enqueue a batch of async writes for the txg sync, with deadlines 2.5 seconds in
the future. If we can't complete all the i/os in 2.5 seconds (e.g. because
there were always reads pending), then these i/os will become past due. Now we
must service all the "async" writes (which could be hundreds of megabytes)
before we service any reads, introducing considerable latency to synchronous
i/os (reads or ZIL writes).
Notes on porting to ZFS on Linux:
- zio_t gained new members io_physdone and io_phys_children. Because
object caches in the Linux port call the constructor only once at
allocation time, objects may contain residual data when retrieved
from the cache. Therefore zio_create() was updated to zero out the two
new fields.
- vdev_mirror_pending() relied on the depth of the per-vdev pending queue
(vq->vq_pending_tree) to select the least-busy leaf vdev to read from.
This tree has been replaced by vq->vq_active_tree which is now used
for the same purpose.
- vdev_queue_init() used the value of zfs_vdev_max_pending to determine
the number of vdev I/O buffers to pre-allocate. That global no longer
exists, so we instead use the sum of the *_max_active values for each of
the five I/O classes described above.
- The Illumos implementation of dmu_tx_delay() delays a transaction by
sleeping in condition variable embedded in the thread
(curthread->t_delay_cv). We do not have an equivalent CV to use in
Linux, so this change replaced the delay logic with a wrapper called
zfs_sleep_until(). This wrapper could be adopted upstream and in other
downstream ports to abstract away operating system-specific delay logic.
- These tunables are added as module parameters, and descriptions added
to the zfs-module-parameters.5 man page.
spa_asize_inflation
zfs_deadman_synctime_ms
zfs_vdev_max_active
zfs_vdev_async_write_active_min_dirty_percent
zfs_vdev_async_write_active_max_dirty_percent
zfs_vdev_async_read_max_active
zfs_vdev_async_read_min_active
zfs_vdev_async_write_max_active
zfs_vdev_async_write_min_active
zfs_vdev_scrub_max_active
zfs_vdev_scrub_min_active
zfs_vdev_sync_read_max_active
zfs_vdev_sync_read_min_active
zfs_vdev_sync_write_max_active
zfs_vdev_sync_write_min_active
zfs_dirty_data_max_percent
zfs_delay_min_dirty_percent
zfs_dirty_data_max_max_percent
zfs_dirty_data_max
zfs_dirty_data_max_max
zfs_dirty_data_sync
zfs_delay_scale
The latter four have type unsigned long, whereas they are uint64_t in
Illumos. This accommodates Linux's module_param() supported types, but
means they may overflow on 32-bit architectures.
The values zfs_dirty_data_max and zfs_dirty_data_max_max are the most
likely to overflow on 32-bit systems, since they express physical RAM
sizes in bytes. In fact, Illumos initializes zfs_dirty_data_max_max to
2^32 which does overflow. To resolve that, this port instead initializes
it in arc_init() to 25% of physical RAM, and adds the tunable
zfs_dirty_data_max_max_percent to override that percentage. While this
solution doesn't completely avoid the overflow issue, it should be a
reasonable default for most systems, and the minority of affected
systems can work around the issue by overriding the defaults.
- Fixed reversed logic in comment above zfs_delay_scale declaration.
- Clarified comments in vdev_queue.c regarding when per-queue minimums take
effect.
- Replaced dmu_tx_write_limit in the dmu_tx kstat file
with dmu_tx_dirty_delay and dmu_tx_dirty_over_max. The first counts
how many times a transaction has been delayed because the pool dirty
data has exceeded zfs_delay_min_dirty_percent. The latter counts how
many times the pool dirty data has exceeded zfs_dirty_data_max (which
we expect to never happen).
- The original patch would have regressed the bug fixed in
zfsonlinux/zfs@c418410, which prevented users from setting the
zfs_vdev_aggregation_limit tuning larger than SPA_MAXBLOCKSIZE.
A similar fix is added to vdev_queue_aggregate().
- In vdev_queue_io_to_issue(), dynamically allocate 'zio_t search' on the
heap instead of the stack. In Linux we can't afford such large
structures on the stack.
Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Adam Leventhal <ahl@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Reviewed by: Ned Bass <bass6@llnl.gov>
Reviewed by: Brendan Gregg <brendan.gregg@joyent.com>
Approved by: Robert Mustacchi <rm@joyent.com>
References:
http://www.illumos.org/issues/4045
illumos/illumos-gate@69962b5647e4a8b9b14998733b765925381b727e
Ported-by: Ned Bass <bass6@llnl.gov>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #1913
2013-08-29 03:01:20 +00:00
|
|
|
/* How much space was changed to dsl_pool_dirty_space() for this? */
|
|
|
|
unsigned int dr_accounted;
|
|
|
|
|
2016-04-21 18:23:37 +00:00
|
|
|
/* A copy of the bp that points to us */
|
|
|
|
blkptr_t dr_bp_copy;
|
|
|
|
|
2008-11-20 20:01:55 +00:00
|
|
|
union dirty_types {
|
|
|
|
struct dirty_indirect {
|
|
|
|
|
|
|
|
/* protect access to list */
|
|
|
|
kmutex_t dr_mtx;
|
|
|
|
|
|
|
|
/* Our list of dirty children */
|
|
|
|
list_t dr_children;
|
|
|
|
} di;
|
|
|
|
struct dirty_leaf {
|
|
|
|
|
|
|
|
/*
|
|
|
|
* dr_data is set when we dirty the buffer
|
|
|
|
* so that we can retain the pointer even if it
|
|
|
|
* gets COW'd in a subsequent transaction group.
|
|
|
|
*/
|
|
|
|
arc_buf_t *dr_data;
|
|
|
|
blkptr_t dr_overridden_by;
|
|
|
|
override_states_t dr_override_state;
|
2010-05-28 20:45:14 +00:00
|
|
|
uint8_t dr_copies;
|
2013-05-10 19:47:54 +00:00
|
|
|
boolean_t dr_nopwrite;
|
2018-04-17 18:06:54 +00:00
|
|
|
boolean_t dr_has_raw_params;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If dr_has_raw_params is set, the following crypt
|
|
|
|
* params will be set on the BP that's written.
|
|
|
|
*/
|
|
|
|
boolean_t dr_byteorder;
|
|
|
|
uint8_t dr_salt[ZIO_DATA_SALT_LEN];
|
|
|
|
uint8_t dr_iv[ZIO_DATA_IV_LEN];
|
|
|
|
uint8_t dr_mac[ZIO_DATA_MAC_LEN];
|
2008-11-20 20:01:55 +00:00
|
|
|
} dl;
|
Improve zfs receive performance with lightweight write
The performance of `zfs receive` can be bottlenecked on the CPU consumed
by the `receive_writer` thread, especially when receiving streams with
small compressed block sizes. Much of the CPU is spent creating and
destroying dbuf's and arc buf's, one for each `WRITE` record in the send
stream.
This commit introduces the concept of "lightweight writes", which allows
`zfs receive` to write to the DMU by providing an ABD, and instantiating
only a new type of `dbuf_dirty_record_t`. The dbuf and arc buf for this
"dirty leaf block" are not instantiated.
Because there is no dbuf with the dirty data, this mechanism doesn't
support reading from "lightweight-dirty" blocks (they would see the
on-disk state rather than the dirty data). Since the dedup-receive code
has been removed, `zfs receive` is write-only, so this works fine.
Because there are no arc bufs for the received data, the received data
is no longer cached in the ARC.
Testing a receive of a stream with average compressed block size of 4KB,
this commit improves performance by 50%, while also reducing CPU usage
by 50% of a CPU. On a per-block basis, CPU consumed by receive_writer()
and dbuf_evict() is now 1/7th (14%) of what it was.
Baseline: 450MB/s, CPU in receive_writer() 40% + dbuf_evict() 35%
New: 670MB/s, CPU in receive_writer() 17% + dbuf_evict() 0%
The code is also restructured in a few ways:
Added a `dr_dnode` field to the dbuf_dirty_record_t. This simplifies
some existing code that no longer needs `DB_DNODE_ENTER()` and related
routines. The new field is needed by the lightweight-type dirty record.
To ensure that the `dr_dnode` field remains valid until the dirty record
is freed, we have to ensure that the `dnode_move()` doesn't relocate the
dnode_t. To do this we keep a hold on the dnode until it's zio's have
completed. This is already done by the user-accounting code
(`userquota_updates_task()`), this commit extends that so that it always
keeps the dnode hold until zio completion (see `dnode_rele_task()`).
`dn_dirty_txg` was previously zeroed when the dnode was synced. This
was not necessary, since its meaning can be "when was this dnode last
dirtied". This change simplifies the new `dnode_rele_task()` code.
Removed some dead code related to `DRR_WRITE_BYREF` (dedup receive).
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Paul Dagnelie <pcd@delphix.com>
Reviewed-by: George Wilson <gwilson@delphix.com>
Signed-off-by: Matthew Ahrens <mahrens@delphix.com>
Closes #11105
2020-12-11 18:26:02 +00:00
|
|
|
struct dirty_lightweight_leaf {
|
|
|
|
/*
|
|
|
|
* This dirty record refers to a leaf (level=0)
|
|
|
|
* block, whose dbuf has not been instantiated for
|
|
|
|
* performance reasons.
|
|
|
|
*/
|
|
|
|
uint64_t dr_blkid;
|
|
|
|
abd_t *dr_abd;
|
|
|
|
zio_prop_t dr_props;
|
|
|
|
enum zio_flag dr_flags;
|
|
|
|
} dll;
|
2008-11-20 20:01:55 +00:00
|
|
|
} dt;
|
|
|
|
} dbuf_dirty_record_t;
|
|
|
|
|
|
|
|
typedef struct dmu_buf_impl {
|
|
|
|
/*
|
|
|
|
* The following members are immutable, with the exception of
|
|
|
|
* db.db_data, which is protected by db_mtx.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* the publicly visible structure */
|
|
|
|
dmu_buf_t db;
|
|
|
|
|
|
|
|
/* the objset we belong to */
|
2010-05-28 20:45:14 +00:00
|
|
|
struct objset *db_objset;
|
2008-11-20 20:01:55 +00:00
|
|
|
|
|
|
|
/*
|
2010-08-26 21:24:34 +00:00
|
|
|
* handle to safely access the dnode we belong to (NULL when evicted)
|
2008-11-20 20:01:55 +00:00
|
|
|
*/
|
2010-08-26 21:24:34 +00:00
|
|
|
struct dnode_handle *db_dnode_handle;
|
2008-11-20 20:01:55 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* our parent buffer; if the dnode points to us directly,
|
2010-08-26 21:24:34 +00:00
|
|
|
* db_parent == db_dnode_handle->dnh_dnode->dn_dbuf
|
2008-11-20 20:01:55 +00:00
|
|
|
* only accessed by sync thread ???
|
|
|
|
* (NULL when evicted)
|
2010-08-26 21:24:34 +00:00
|
|
|
* May change from NULL to non-NULL under the protection of db_mtx
|
|
|
|
* (see dbuf_check_blkptr())
|
2008-11-20 20:01:55 +00:00
|
|
|
*/
|
|
|
|
struct dmu_buf_impl *db_parent;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* link for hash table of all dmu_buf_impl_t's
|
|
|
|
*/
|
|
|
|
struct dmu_buf_impl *db_hash_next;
|
|
|
|
|
2020-02-05 19:08:44 +00:00
|
|
|
/*
|
|
|
|
* Our link on the owner dnodes's dn_dbufs list.
|
|
|
|
* Protected by its dn_dbufs_mtx. Should be on the same cache line
|
|
|
|
* as db_level and db_blkid for the best avl_add() performance.
|
|
|
|
*/
|
|
|
|
avl_node_t db_link;
|
|
|
|
|
2008-11-20 20:01:55 +00:00
|
|
|
/* our block number */
|
|
|
|
uint64_t db_blkid;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Pointer to the blkptr_t which points to us. May be NULL if we
|
|
|
|
* don't have one yet. (NULL when evicted)
|
|
|
|
*/
|
|
|
|
blkptr_t *db_blkptr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Our indirection level. Data buffers have db_level==0.
|
|
|
|
* Indirect buffers which point to data buffers have
|
|
|
|
* db_level==1. etc. Buffers which contain dnodes have
|
|
|
|
* db_level==0, since the dnodes are stored in a file.
|
|
|
|
*/
|
|
|
|
uint8_t db_level;
|
|
|
|
|
2019-07-08 20:18:50 +00:00
|
|
|
/*
|
|
|
|
* Protects db_buf's contents if they contain an indirect block or data
|
|
|
|
* block of the meta-dnode. We use this lock to protect the structure of
|
|
|
|
* the block tree. This means that when modifying this dbuf's data, we
|
|
|
|
* grab its rwlock. When modifying its parent's data (including the
|
|
|
|
* blkptr to this dbuf), we grab the parent's rwlock. The lock ordering
|
|
|
|
* for this lock is:
|
|
|
|
* 1) dn_struct_rwlock
|
|
|
|
* 2) db_rwlock
|
|
|
|
* We don't currently grab multiple dbufs' db_rwlocks at once.
|
|
|
|
*/
|
|
|
|
krwlock_t db_rwlock;
|
|
|
|
|
|
|
|
/* buffer holding our data */
|
|
|
|
arc_buf_t *db_buf;
|
|
|
|
|
2008-11-20 20:01:55 +00:00
|
|
|
/* db_mtx protects the members below */
|
|
|
|
kmutex_t db_mtx;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Current state of the buffer
|
|
|
|
*/
|
|
|
|
dbuf_states_t db_state;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Refcount accessed by dmu_buf_{hold,rele}.
|
|
|
|
* If nonzero, the buffer can't be destroyed.
|
|
|
|
* Protected by db_mtx.
|
|
|
|
*/
|
2018-09-26 17:29:26 +00:00
|
|
|
zfs_refcount_t db_holds;
|
2008-11-20 20:01:55 +00:00
|
|
|
|
|
|
|
kcondvar_t db_changed;
|
|
|
|
dbuf_dirty_record_t *db_data_pending;
|
|
|
|
|
2020-02-05 19:07:19 +00:00
|
|
|
/* List of dirty records for the buffer sorted newest to oldest. */
|
|
|
|
list_t db_dirty_records;
|
2008-11-20 20:01:55 +00:00
|
|
|
|
2018-07-10 17:49:50 +00:00
|
|
|
/* Link in dbuf_cache or dbuf_metadata_cache */
|
2016-06-02 04:04:53 +00:00
|
|
|
multilist_node_t db_cache_link;
|
|
|
|
|
2018-07-10 17:49:50 +00:00
|
|
|
/* Tells us which dbuf cache this dbuf is in, if any */
|
|
|
|
dbuf_cached_state_t db_caching_status;
|
|
|
|
|
2008-11-20 20:01:55 +00:00
|
|
|
/* Data which is unique to data (leaf) blocks: */
|
|
|
|
|
2015-04-02 03:44:32 +00:00
|
|
|
/* User callback information. */
|
|
|
|
dmu_buf_user_t *db_user;
|
2008-11-20 20:01:55 +00:00
|
|
|
|
2015-10-13 21:09:45 +00:00
|
|
|
/*
|
|
|
|
* Evict user data as soon as the dirty and reference
|
|
|
|
* counts are equal.
|
|
|
|
*/
|
|
|
|
uint8_t db_user_immediate_evict;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This block was freed while a read or write was
|
|
|
|
* active.
|
|
|
|
*/
|
2008-11-20 20:01:55 +00:00
|
|
|
uint8_t db_freed_in_flight;
|
|
|
|
|
2015-10-13 21:09:45 +00:00
|
|
|
/*
|
|
|
|
* dnode_evict_dbufs() or dnode_evict_bonus() tried to
|
|
|
|
* evict this dbuf, but couldn't due to outstanding
|
|
|
|
* references. Evict once the refcount drops to 0.
|
|
|
|
*/
|
|
|
|
uint8_t db_pending_evict;
|
|
|
|
|
2008-11-20 20:01:55 +00:00
|
|
|
uint8_t db_dirtycnt;
|
|
|
|
} dmu_buf_impl_t;
|
|
|
|
|
|
|
|
/* Note: the dbuf hash table is exposed only for the mdb module */
|
2021-07-01 15:30:31 +00:00
|
|
|
#define DBUF_MUTEXES 2048
|
2008-11-20 20:01:55 +00:00
|
|
|
#define DBUF_HASH_MUTEX(h, idx) (&(h)->hash_mutexes[(idx) & (DBUF_MUTEXES-1)])
|
|
|
|
typedef struct dbuf_hash_table {
|
|
|
|
uint64_t hash_table_mask;
|
|
|
|
dmu_buf_impl_t **hash_table;
|
2021-07-01 15:30:31 +00:00
|
|
|
kmutex_t hash_mutexes[DBUF_MUTEXES] ____cacheline_aligned;
|
2008-11-20 20:01:55 +00:00
|
|
|
} dbuf_hash_table_t;
|
|
|
|
|
2020-09-28 00:08:38 +00:00
|
|
|
typedef void (*dbuf_prefetch_fn)(void *, boolean_t);
|
|
|
|
|
2016-08-31 08:12:08 +00:00
|
|
|
uint64_t dbuf_whichblock(const struct dnode *di, const int64_t level,
|
|
|
|
const uint64_t offset);
|
2008-11-20 20:01:55 +00:00
|
|
|
|
|
|
|
void dbuf_create_bonus(struct dnode *dn);
|
2010-05-28 20:45:14 +00:00
|
|
|
int dbuf_spill_set_blksz(dmu_buf_t *db, uint64_t blksz, dmu_tx_t *tx);
|
|
|
|
|
|
|
|
void dbuf_rm_spill(struct dnode *dn, dmu_tx_t *tx);
|
2008-11-20 20:01:55 +00:00
|
|
|
|
|
|
|
dmu_buf_impl_t *dbuf_hold(struct dnode *dn, uint64_t blkid, void *tag);
|
|
|
|
dmu_buf_impl_t *dbuf_hold_level(struct dnode *dn, int level, uint64_t blkid,
|
|
|
|
void *tag);
|
2015-12-22 01:31:57 +00:00
|
|
|
int dbuf_hold_impl(struct dnode *dn, uint8_t level, uint64_t blkid,
|
|
|
|
boolean_t fail_sparse, boolean_t fail_uncached,
|
2008-11-20 20:01:55 +00:00
|
|
|
void *tag, dmu_buf_impl_t **dbp);
|
|
|
|
|
2020-09-28 00:08:38 +00:00
|
|
|
int dbuf_prefetch_impl(struct dnode *dn, int64_t level, uint64_t blkid,
|
|
|
|
zio_priority_t prio, arc_flags_t aflags, dbuf_prefetch_fn cb,
|
|
|
|
void *arg);
|
|
|
|
int dbuf_prefetch(struct dnode *dn, int64_t level, uint64_t blkid,
|
2015-12-22 01:31:57 +00:00
|
|
|
zio_priority_t prio, arc_flags_t aflags);
|
2008-11-20 20:01:55 +00:00
|
|
|
|
|
|
|
void dbuf_add_ref(dmu_buf_impl_t *db, void *tag);
|
2015-04-02 11:59:15 +00:00
|
|
|
boolean_t dbuf_try_add_ref(dmu_buf_t *db, objset_t *os, uint64_t obj,
|
|
|
|
uint64_t blkid, void *tag);
|
2008-11-20 20:01:55 +00:00
|
|
|
uint64_t dbuf_refcount(dmu_buf_impl_t *db);
|
|
|
|
|
|
|
|
void dbuf_rele(dmu_buf_impl_t *db, void *tag);
|
2018-07-31 21:51:15 +00:00
|
|
|
void dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag, boolean_t evicting);
|
2008-11-20 20:01:55 +00:00
|
|
|
|
2015-04-02 11:59:15 +00:00
|
|
|
dmu_buf_impl_t *dbuf_find(struct objset *os, uint64_t object, uint8_t level,
|
|
|
|
uint64_t blkid);
|
2008-11-20 20:01:55 +00:00
|
|
|
|
|
|
|
int dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags);
|
2008-12-03 20:09:06 +00:00
|
|
|
void dmu_buf_will_not_fill(dmu_buf_t *db, dmu_tx_t *tx);
|
2008-11-20 20:01:55 +00:00
|
|
|
void dmu_buf_will_fill(dmu_buf_t *db, dmu_tx_t *tx);
|
|
|
|
void dmu_buf_fill_done(dmu_buf_t *db, dmu_tx_t *tx);
|
2009-07-02 22:44:48 +00:00
|
|
|
void dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx);
|
2008-11-20 20:01:55 +00:00
|
|
|
dbuf_dirty_record_t *dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
|
Improve zfs receive performance with lightweight write
The performance of `zfs receive` can be bottlenecked on the CPU consumed
by the `receive_writer` thread, especially when receiving streams with
small compressed block sizes. Much of the CPU is spent creating and
destroying dbuf's and arc buf's, one for each `WRITE` record in the send
stream.
This commit introduces the concept of "lightweight writes", which allows
`zfs receive` to write to the DMU by providing an ABD, and instantiating
only a new type of `dbuf_dirty_record_t`. The dbuf and arc buf for this
"dirty leaf block" are not instantiated.
Because there is no dbuf with the dirty data, this mechanism doesn't
support reading from "lightweight-dirty" blocks (they would see the
on-disk state rather than the dirty data). Since the dedup-receive code
has been removed, `zfs receive` is write-only, so this works fine.
Because there are no arc bufs for the received data, the received data
is no longer cached in the ARC.
Testing a receive of a stream with average compressed block size of 4KB,
this commit improves performance by 50%, while also reducing CPU usage
by 50% of a CPU. On a per-block basis, CPU consumed by receive_writer()
and dbuf_evict() is now 1/7th (14%) of what it was.
Baseline: 450MB/s, CPU in receive_writer() 40% + dbuf_evict() 35%
New: 670MB/s, CPU in receive_writer() 17% + dbuf_evict() 0%
The code is also restructured in a few ways:
Added a `dr_dnode` field to the dbuf_dirty_record_t. This simplifies
some existing code that no longer needs `DB_DNODE_ENTER()` and related
routines. The new field is needed by the lightweight-type dirty record.
To ensure that the `dr_dnode` field remains valid until the dirty record
is freed, we have to ensure that the `dnode_move()` doesn't relocate the
dnode_t. To do this we keep a hold on the dnode until it's zio's have
completed. This is already done by the user-accounting code
(`userquota_updates_task()`), this commit extends that so that it always
keeps the dnode hold until zio completion (see `dnode_rele_task()`).
`dn_dirty_txg` was previously zeroed when the dnode was synced. This
was not necessary, since its meaning can be "when was this dnode last
dirtied". This change simplifies the new `dnode_rele_task()` code.
Removed some dead code related to `DRR_WRITE_BYREF` (dedup receive).
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Paul Dagnelie <pcd@delphix.com>
Reviewed-by: George Wilson <gwilson@delphix.com>
Signed-off-by: Matthew Ahrens <mahrens@delphix.com>
Closes #11105
2020-12-11 18:26:02 +00:00
|
|
|
dbuf_dirty_record_t *dbuf_dirty_lightweight(dnode_t *dn, uint64_t blkid,
|
|
|
|
dmu_tx_t *tx);
|
2010-05-28 20:45:14 +00:00
|
|
|
arc_buf_t *dbuf_loan_arcbuf(dmu_buf_impl_t *db);
|
2014-06-05 21:19:08 +00:00
|
|
|
void dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data,
|
|
|
|
bp_embedded_type_t etype, enum zio_compress comp,
|
|
|
|
int uncompressed_size, int compressed_size, int byteorder, dmu_tx_t *tx);
|
2008-11-20 20:01:55 +00:00
|
|
|
|
Improve zfs receive performance with lightweight write
The performance of `zfs receive` can be bottlenecked on the CPU consumed
by the `receive_writer` thread, especially when receiving streams with
small compressed block sizes. Much of the CPU is spent creating and
destroying dbuf's and arc buf's, one for each `WRITE` record in the send
stream.
This commit introduces the concept of "lightweight writes", which allows
`zfs receive` to write to the DMU by providing an ABD, and instantiating
only a new type of `dbuf_dirty_record_t`. The dbuf and arc buf for this
"dirty leaf block" are not instantiated.
Because there is no dbuf with the dirty data, this mechanism doesn't
support reading from "lightweight-dirty" blocks (they would see the
on-disk state rather than the dirty data). Since the dedup-receive code
has been removed, `zfs receive` is write-only, so this works fine.
Because there are no arc bufs for the received data, the received data
is no longer cached in the ARC.
Testing a receive of a stream with average compressed block size of 4KB,
this commit improves performance by 50%, while also reducing CPU usage
by 50% of a CPU. On a per-block basis, CPU consumed by receive_writer()
and dbuf_evict() is now 1/7th (14%) of what it was.
Baseline: 450MB/s, CPU in receive_writer() 40% + dbuf_evict() 35%
New: 670MB/s, CPU in receive_writer() 17% + dbuf_evict() 0%
The code is also restructured in a few ways:
Added a `dr_dnode` field to the dbuf_dirty_record_t. This simplifies
some existing code that no longer needs `DB_DNODE_ENTER()` and related
routines. The new field is needed by the lightweight-type dirty record.
To ensure that the `dr_dnode` field remains valid until the dirty record
is freed, we have to ensure that the `dnode_move()` doesn't relocate the
dnode_t. To do this we keep a hold on the dnode until it's zio's have
completed. This is already done by the user-accounting code
(`userquota_updates_task()`), this commit extends that so that it always
keeps the dnode hold until zio completion (see `dnode_rele_task()`).
`dn_dirty_txg` was previously zeroed when the dnode was synced. This
was not necessary, since its meaning can be "when was this dnode last
dirtied". This change simplifies the new `dnode_rele_task()` code.
Removed some dead code related to `DRR_WRITE_BYREF` (dedup receive).
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Paul Dagnelie <pcd@delphix.com>
Reviewed-by: George Wilson <gwilson@delphix.com>
Signed-off-by: Matthew Ahrens <mahrens@delphix.com>
Closes #11105
2020-12-11 18:26:02 +00:00
|
|
|
int dmu_lightweight_write_by_dnode(dnode_t *dn, uint64_t offset, abd_t *abd,
|
|
|
|
const struct zio_prop *zp, enum zio_flag flags, dmu_tx_t *tx);
|
|
|
|
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
void dmu_buf_redact(dmu_buf_t *dbuf, dmu_tx_t *tx);
|
2016-06-02 04:04:53 +00:00
|
|
|
void dbuf_destroy(dmu_buf_impl_t *db);
|
2008-11-20 20:01:55 +00:00
|
|
|
|
|
|
|
void dbuf_unoverride(dbuf_dirty_record_t *dr);
|
2015-07-02 16:23:20 +00:00
|
|
|
void dbuf_sync_list(list_t *list, int level, dmu_tx_t *tx);
|
2010-05-28 20:45:14 +00:00
|
|
|
void dbuf_release_bp(dmu_buf_impl_t *db);
|
2019-07-08 20:18:50 +00:00
|
|
|
db_lock_type_t dmu_buf_lock_parent(dmu_buf_impl_t *db, krw_t rw, void *tag);
|
|
|
|
void dmu_buf_unlock_parent(dmu_buf_impl_t *db, db_lock_type_t type, void *tag);
|
2008-11-20 20:01:55 +00:00
|
|
|
|
2008-12-03 20:09:06 +00:00
|
|
|
void dbuf_free_range(struct dnode *dn, uint64_t start, uint64_t end,
|
2008-11-20 20:01:55 +00:00
|
|
|
struct dmu_tx *);
|
|
|
|
|
|
|
|
void dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx);
|
|
|
|
|
2013-10-03 00:11:19 +00:00
|
|
|
void dbuf_stats_init(dbuf_hash_table_t *hash);
|
|
|
|
void dbuf_stats_destroy(void);
|
|
|
|
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
int dbuf_dnode_findbp(dnode_t *dn, uint64_t level, uint64_t blkid,
|
|
|
|
blkptr_t *bp, uint16_t *datablkszsec, uint8_t *indblkshift);
|
|
|
|
|
2010-08-26 21:24:34 +00:00
|
|
|
#define DB_DNODE(_db) ((_db)->db_dnode_handle->dnh_dnode)
|
|
|
|
#define DB_DNODE_LOCK(_db) ((_db)->db_dnode_handle->dnh_zrlock)
|
|
|
|
#define DB_DNODE_ENTER(_db) (zrl_add(&DB_DNODE_LOCK(_db)))
|
|
|
|
#define DB_DNODE_EXIT(_db) (zrl_remove(&DB_DNODE_LOCK(_db)))
|
|
|
|
#define DB_DNODE_HELD(_db) (!zrl_is_zero(&DB_DNODE_LOCK(_db)))
|
|
|
|
|
2008-11-20 20:01:55 +00:00
|
|
|
void dbuf_init(void);
|
|
|
|
void dbuf_fini(void);
|
|
|
|
|
2010-08-26 21:24:34 +00:00
|
|
|
boolean_t dbuf_is_metadata(dmu_buf_impl_t *db);
|
|
|
|
|
2020-02-05 19:07:19 +00:00
|
|
|
static inline dbuf_dirty_record_t *
|
|
|
|
dbuf_find_dirty_lte(dmu_buf_impl_t *db, uint64_t txg)
|
|
|
|
{
|
|
|
|
dbuf_dirty_record_t *dr;
|
|
|
|
|
|
|
|
for (dr = list_head(&db->db_dirty_records);
|
|
|
|
dr != NULL && dr->dr_txg > txg;
|
|
|
|
dr = list_next(&db->db_dirty_records, dr))
|
|
|
|
continue;
|
|
|
|
return (dr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline dbuf_dirty_record_t *
|
|
|
|
dbuf_find_dirty_eq(dmu_buf_impl_t *db, uint64_t txg)
|
|
|
|
{
|
|
|
|
dbuf_dirty_record_t *dr;
|
|
|
|
|
|
|
|
dr = dbuf_find_dirty_lte(db, txg);
|
|
|
|
if (dr && dr->dr_txg == txg)
|
|
|
|
return (dr);
|
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
|
2010-08-26 21:24:34 +00:00
|
|
|
#define DBUF_GET_BUFC_TYPE(_db) \
|
2013-09-04 12:00:57 +00:00
|
|
|
(dbuf_is_metadata(_db) ? ARC_BUFC_METADATA : ARC_BUFC_DATA)
|
2008-12-03 20:09:06 +00:00
|
|
|
|
2010-08-26 21:24:34 +00:00
|
|
|
#define DBUF_IS_CACHEABLE(_db) \
|
|
|
|
((_db)->db_objset->os_primary_cache == ZFS_CACHE_ALL || \
|
2013-09-04 12:00:57 +00:00
|
|
|
(dbuf_is_metadata(_db) && \
|
2010-08-26 21:24:34 +00:00
|
|
|
((_db)->db_objset->os_primary_cache == ZFS_CACHE_METADATA)))
|
2008-12-03 20:09:06 +00:00
|
|
|
|
2010-08-26 21:24:34 +00:00
|
|
|
#define DBUF_IS_L2CACHEABLE(_db) \
|
|
|
|
((_db)->db_objset->os_secondary_cache == ZFS_CACHE_ALL || \
|
2013-09-04 12:00:57 +00:00
|
|
|
(dbuf_is_metadata(_db) && \
|
2010-08-26 21:24:34 +00:00
|
|
|
((_db)->db_objset->os_secondary_cache == ZFS_CACHE_METADATA)))
|
2008-11-20 20:01:55 +00:00
|
|
|
|
2017-11-02 15:01:56 +00:00
|
|
|
#define DNODE_LEVEL_IS_L2CACHEABLE(_dn, _level) \
|
|
|
|
((_dn)->dn_objset->os_secondary_cache == ZFS_CACHE_ALL || \
|
|
|
|
(((_level) > 0 || \
|
|
|
|
DMU_OT_IS_METADATA((_dn)->dn_handle->dnh_dnode->dn_type)) && \
|
|
|
|
((_dn)->dn_objset->os_secondary_cache == ZFS_CACHE_METADATA)))
|
|
|
|
|
2008-11-20 20:01:55 +00:00
|
|
|
#ifdef ZFS_DEBUG
|
|
|
|
|
|
|
|
/*
|
|
|
|
* There should be a ## between the string literal and fmt, to make it
|
|
|
|
* clear that we're joining two strings together, but gcc does not
|
|
|
|
* support that preprocessor token.
|
|
|
|
*/
|
|
|
|
#define dprintf_dbuf(dbuf, fmt, ...) do { \
|
|
|
|
if (zfs_flags & ZFS_DEBUG_DPRINTF) { \
|
|
|
|
char __db_buf[32]; \
|
|
|
|
uint64_t __db_obj = (dbuf)->db.db_object; \
|
|
|
|
if (__db_obj == DMU_META_DNODE_OBJECT) \
|
|
|
|
(void) strcpy(__db_buf, "mdn"); \
|
|
|
|
else \
|
|
|
|
(void) snprintf(__db_buf, sizeof (__db_buf), "%lld", \
|
|
|
|
(u_longlong_t)__db_obj); \
|
|
|
|
dprintf_ds((dbuf)->db_objset->os_dsl_dataset, \
|
|
|
|
"obj=%s lvl=%u blkid=%lld " fmt, \
|
|
|
|
__db_buf, (dbuf)->db_level, \
|
|
|
|
(u_longlong_t)(dbuf)->db_blkid, __VA_ARGS__); \
|
|
|
|
} \
|
|
|
|
_NOTE(CONSTCOND) } while (0)
|
|
|
|
|
2014-11-21 00:09:39 +00:00
|
|
|
#define dprintf_dbuf_bp(db, bp, fmt, ...) do { \
|
|
|
|
if (zfs_flags & ZFS_DEBUG_DPRINTF) { \
|
|
|
|
char *__blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_SLEEP); \
|
2013-12-09 18:37:51 +00:00
|
|
|
snprintf_blkptr(__blkbuf, BP_SPRINTF_LEN, bp); \
|
2014-11-21 00:09:39 +00:00
|
|
|
dprintf_dbuf(db, fmt " %s\n", __VA_ARGS__, __blkbuf); \
|
|
|
|
kmem_free(__blkbuf, BP_SPRINTF_LEN); \
|
|
|
|
} \
|
2008-11-20 20:01:55 +00:00
|
|
|
_NOTE(CONSTCOND) } while (0)
|
|
|
|
|
|
|
|
#define DBUF_VERIFY(db) dbuf_verify(db)
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
#define dprintf_dbuf(db, fmt, ...)
|
|
|
|
#define dprintf_dbuf_bp(db, bp, fmt, ...)
|
|
|
|
#define DBUF_VERIFY(db)
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
#ifdef __cplusplus
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#endif /* _SYS_DBUF_H */
|