Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity

Assorted fixes for the performance tests 

- Bail out early if we're running the perf tests and forget to
  specify disks.
- Allow perf tests to run with any number of disks.
- Remove weekly vs. nightly settings
- Move variables with common values to perf.shlib
- Use zinject to clear the ARC over export/import
- Fix dbuf cache size calculation

When the meaning of `dbuf_cache_max_bytes` changed, the performance
test that covers the dbuf cache started to fail. The test would try to
write files for the test using the max possible size of the cache,
inevitably filling the pool and failing. This change uses
`dbuf_cache_shift` to correctly calculate the dbuf cache size.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Tony Nguyen <tony.nguyen@delphix.com>
Reviewed-by: George Melikov <mail@gmelikov.ru>
Signed-off-by: John Kennedy <john.kennedy@delphix.com>
Closes #12408
This commit is contained in:
John Wren Kennedy 2021-07-26 15:47:08 -06:00 committed by GitHub
parent d8381f50d6
commit bdd2bfd02c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 128 additions and 269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
scripts/zfs-tests.sh
View File

@ -567,18 +567,17 @@ fi
. "$STF_SUITE/include/default.cfg" . "$STF_SUITE/include/default.cfg"
msg
msg "--- Configuration ---"
msg "Runfiles: $RUNFILES"
msg "STF_TOOLS: $STF_TOOLS"
msg "STF_SUITE: $STF_SUITE"
msg "STF_PATH: $STF_PATH"
# #
# No DISKS have been provided so a basic file or loopback based devices # No DISKS have been provided so a basic file or loopback based devices
# must be created for the test suite to use. # must be created for the test suite to use.
# #
if [ -z "${DISKS}" ]; then if [ -z "${DISKS}" ]; then
#
# If this is a performance run, prevent accidental use of
# loopback devices.
#
[ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
# #
# Create sparse files for the test suite. These may be used # Create sparse files for the test suite. These may be used
# directory or have loopback devices layered on them. # directory or have loopback devices layered on them.
@ -619,8 +618,14 @@ if [ -z "${DISKS}" ]; then
fi fi
fi fi
#
# It may be desirable to test with fewer disks than the default when running
# the performance tests, but the functional tests require at least three.
#
NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}') NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)" if [ "$TAGS" != "perf" ]; then
[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
fi
# #
# Disable SELinux until the ZFS Test Suite has been updated accordingly. # Disable SELinux until the ZFS Test Suite has been updated accordingly.
@ -637,6 +642,12 @@ if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg" sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
fi fi
msg
msg "--- Configuration ---"
msg "Runfiles: $RUNFILES"
msg "STF_TOOLS: $STF_TOOLS"
msg "STF_SUITE: $STF_SUITE"
msg "STF_PATH: $STF_PATH"
msg "FILEDIR: $FILEDIR" msg "FILEDIR: $FILEDIR"
msg "FILES: $FILES" msg "FILES: $FILES"
msg "LOOPBACKS: $LOOPBACKS" msg "LOOPBACKS: $LOOPBACKS"

2
tests/zfs-tests/include/tunables.cfg
View File

@ -27,7 +27,7 @@ COMPRESSED_ARC_ENABLED compressed_arc_enabled zfs_compressed_arc_enabled
CONDENSE_INDIRECT_COMMIT_ENTRY_DELAY_MS condense.indirect_commit_entry_delay_ms zfs_condense_indirect_commit_entry_delay_ms CONDENSE_INDIRECT_COMMIT_ENTRY_DELAY_MS condense.indirect_commit_entry_delay_ms zfs_condense_indirect_commit_entry_delay_ms
CONDENSE_INDIRECT_OBSOLETE_PCT condense.indirect_obsolete_pct zfs_condense_indirect_obsolete_pct CONDENSE_INDIRECT_OBSOLETE_PCT condense.indirect_obsolete_pct zfs_condense_indirect_obsolete_pct
CONDENSE_MIN_MAPPING_BYTES condense.min_mapping_bytes zfs_condense_min_mapping_bytes CONDENSE_MIN_MAPPING_BYTES condense.min_mapping_bytes zfs_condense_min_mapping_bytes
DBUF_CACHE_MAX_BYTES dbuf_cache.max_bytes dbuf_cache_max_bytes DBUF_CACHE_SHIFT dbuf.cache_shift dbuf_cache_shift
DEADMAN_CHECKTIME_MS deadman.checktime_ms zfs_deadman_checktime_ms DEADMAN_CHECKTIME_MS deadman.checktime_ms zfs_deadman_checktime_ms
DEADMAN_FAILMODE deadman.failmode zfs_deadman_failmode DEADMAN_FAILMODE deadman.failmode zfs_deadman_failmode
DEADMAN_SYNCTIME_MS deadman.synctime_ms zfs_deadman_synctime_ms DEADMAN_SYNCTIME_MS deadman.synctime_ms zfs_deadman_synctime_ms

61
tests/zfs-tests/tests/perf/perf.shlib
View File

@ -10,18 +10,18 @@
# #
# #
# Copyright (c) 2015, 2016 by Delphix. All rights reserved. # Copyright (c) 2015, 2021 by Delphix. All rights reserved.
# Copyright (c) 2016, Intel Corporation. # Copyright (c) 2016, Intel Corporation.
# #
. $STF_SUITE/include/libtest.shlib . $STF_SUITE/include/libtest.shlib
# If neither is specified, do a nightly run. # Defaults common to all the tests in the regression group
[[ -z $PERF_REGRESSION_WEEKLY ]] && export PERF_REGRESSION_NIGHTLY=1 export PERF_RUNTIME=${PERF_RUNTIME:-'180'}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
# Default runtime for each type of test run. export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_RUNTIME_WEEKLY=$((30 * 60)) export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTIME_NIGHTLY=$((10 * 60)) export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
# Default to JSON for fio output # Default to JSON for fio output
export PERF_FIO_FORMAT=${PERF_FIO_FORMAT:-'json'} export PERF_FIO_FORMAT=${PERF_FIO_FORMAT:-'json'}
@ -90,8 +90,7 @@ function do_fio_run_impl
if $clear_cache; then if $clear_cache; then
# Clear the ARC # Clear the ARC
zpool export $PERFPOOL log_must zinject -a
zpool import $PERFPOOL
fi fi
if [[ -n $ZINJECT_DELAYS ]]; then if [[ -n $ZINJECT_DELAYS ]]; then
@ -159,8 +158,6 @@ function do_fio_run_impl
# set before launching zfstest to override the defaults. # set before launching zfstest to override the defaults.
# #
# PERF_RUNTIME: The time in seconds each fio invocation should run. # PERF_RUNTIME: The time in seconds each fio invocation should run.
# PERF_RUNTYPE: A human readable tag that appears in logs. The defaults are
# nightly and weekly.
# PERF_NTHREADS: A list of how many threads each fio invocation will use. # PERF_NTHREADS: A list of how many threads each fio invocation will use.
# PERF_SYNC_TYPES: Whether to use (O_SYNC) or not. 1 is sync IO, 0 is async IO. # PERF_SYNC_TYPES: Whether to use (O_SYNC) or not. 1 is sync IO, 0 is async IO.
# PERF_IOSIZES: A list of blocksizes in which each fio invocation will do IO. # PERF_IOSIZES: A list of blocksizes in which each fio invocation will do IO.
@ -424,22 +421,44 @@ function get_max_arc_size
echo $max_arc_size echo $max_arc_size
} }
function get_max_dbuf_cache_size function get_arc_target
{ {
typeset -l max_dbuf_cache_size typeset -l arc_c
if is_freebsd; then
arc_c=$(sysctl -n kstat.zfs.misc.arcstats.c)
elif is_illumos; then
arc_c=$(dtrace -qn 'BEGIN {
printf("%u\n", `arc_stats.arcstat_c.value.ui64);
exit(0);
}')
elif is_linux; then
arc_c=`awk '$1 == "c" { print $3 }' \
/proc/spl/kstat/zfs/arcstats`
fi
[[ $? -eq 0 ]] || log_fail "get_arc_target failed"
echo $arc_c
}
function get_dbuf_cache_size
{
typeset -l dbuf_cache_size dbuf_cache_shift
if is_illumos; then if is_illumos; then
max_dbuf_cache_size=$(dtrace -qn 'BEGIN { dbuf_cache_size=$(dtrace -qn 'BEGIN {
printf("%u\n", `dbuf_cache_max_bytes); printf("%u\n", `dbuf_cache_max_bytes);
exit(0); exit(0);
}') }')
else else
max_dbuf_cache_size=$(get_tunable DBUF_CACHE_MAX_BYTES) dbuf_cache_shift=$(get_tunable DBUF_CACHE_SHIFT)
dbuf_cache_size=$(($(get_arc_target) / 2**dbuf_cache_shift))
fi fi
[[ $? -eq 0 ]] || log_fail "get_max_dbuf_cache_size failed" [[ $? -eq 0 ]] || log_fail "get_dbuf_cache_size failed"
echo $max_dbuf_cache_size echo $dbuf_cache_size
} }
# Create a file with some information about how this system is configured. # Create a file with some information about how this system is configured.
@ -569,6 +588,14 @@ function pool_to_lun_list
echo $lun_list echo $lun_list
} }
function print_perf_settings
{
echo "PERF_NTHREADS: $PERF_NTHREADS"
echo "PERF_NTHREADS_PER_FS: $PERF_NTHREADS_PER_FS"
echo "PERF_SYNC_TYPES: $PERF_SYNC_TYPES"
echo "PERF_IOSIZES: $PERF_IOSIZES"
}
# Create a perf_data directory to hold performance statistics and # Create a perf_data directory to hold performance statistics and
# configuration information. # configuration information.
export PERF_DATA_DIR=$(get_perf_output_dir) export PERF_DATA_DIR=$(get_perf_output_dir)

30
tests/zfs-tests/tests/perf/regression/random_reads.ksh
View File

@ -12,7 +12,7 @@
# #
# #
# Copyright (c) 2015, 2020 by Delphix. All rights reserved. # Copyright (c) 2015, 2021 by Delphix. All rights reserved.
# #
# #
@ -55,28 +55,10 @@ populate_perf_filesystems
# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio. # Aim to fill the pool to 50% capacity while accounting for a 3x compressratio.
export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2)) export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
# Variables for use by fio. # Variables specific to this test for use by fio.
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'16 32'}
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'} export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'16 32'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
fi
# Layout the files to be used by the read tests. Create as many files as the # Layout the files to be used by the read tests. Create as many files as the
# largest number of threads. An fio run with fewer threads will use a subset # largest number of threads. An fio run with fewer threads will use a subset
@ -109,6 +91,6 @@ else
) )
fi fi
log_note "Random reads with $PERF_RUNTYPE settings" log_note "Random reads with settings: $(print_perf_settings)"
do_fio_run random_reads.fio false true do_fio_run random_reads.fio false true
log_pass "Measure IO stats during random read load" log_pass "Measure IO stats during random read load"

30
tests/zfs-tests/tests/perf/regression/random_readwrite.ksh
View File

@ -12,7 +12,7 @@
# #
# #
# Copyright (c) 2015, 2020 by Delphix. All rights reserved. # Copyright (c) 2015, 2021 by Delphix. All rights reserved.
# #
# #
@ -55,28 +55,10 @@ populate_perf_filesystems
# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio. # Aim to fill the pool to 50% capacity while accounting for a 3x compressratio.
export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2)) export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
# Variables for use by fio. # Variables specific to this test for use by fio.
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'32 64'}
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'} export PERF_IOSIZES='' # bssplit used instead
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'4 8 16 64'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
export PERF_IOSIZES='' # bssplit used instead
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'32 64'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES='' # bssplit used instead
fi
# Layout the files to be used by the readwrite tests. Create as many files # Layout the files to be used by the readwrite tests. Create as many files
# as the largest number of threads. An fio run with fewer threads will use # as the largest number of threads. An fio run with fewer threads will use
@ -109,6 +91,6 @@ else
) )
fi fi
log_note "Random reads and writes with $PERF_RUNTYPE settings" log_note "Random reads and writes with settings: $(print_perf_settings)"
do_fio_run random_readwrite.fio false true do_fio_run random_readwrite.fio false true
log_pass "Measure IO stats during random read and write load" log_pass "Measure IO stats during random read and write load"

30
tests/zfs-tests/tests/perf/regression/random_readwrite_fixed.ksh
View File

@ -11,7 +11,7 @@
# #
# #
# Copyright (c) 2017, 2020 by Delphix. All rights reserved. # Copyright (c) 2017, 2021 by Delphix. All rights reserved.
# #
# #
@ -45,28 +45,10 @@ populate_perf_filesystems
# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio. # Aim to fill the pool to 50% capacity while accounting for a 3x compressratio.
export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2)) export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
# Variables for use by fio. # Variables specific to this test for use by fio.
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'} export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
export PERF_IOSIZES='8k 64k'
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES='8k'
fi
# Layout the files to be used by the readwrite tests. Create as many files # Layout the files to be used by the readwrite tests. Create as many files
# as the largest number of threads. An fio run with fewer threads will use # as the largest number of threads. An fio run with fewer threads will use
@ -101,6 +83,6 @@ else
) )
fi fi
log_note "Random reads and writes with $PERF_RUNTYPE settings" log_note "Random reads and writes with settings: $(print_perf_settings)"
do_fio_run random_readwrite_fixed.fio false true do_fio_run random_readwrite_fixed.fio false true
log_pass "Measure IO stats during random read and write load" log_pass "Measure IO stats during random read and write load"

30
tests/zfs-tests/tests/perf/regression/random_writes.ksh
View File

@ -12,7 +12,7 @@
# #
# #
# Copyright (c) 2015, 2020 by Delphix. All rights reserved. # Copyright (c) 2015, 2021 by Delphix. All rights reserved.
# #
# #
@ -54,28 +54,10 @@ populate_perf_filesystems
# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio. # Aim to fill the pool to 50% capacity while accounting for a 3x compressratio.
export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2)) export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
# Variables for use by fio. # Variables specific to this test for use by fio.
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'32 128'}
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'} export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 8 16 32 64 128'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 256k'}
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'32 128'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
fi
# Set up the scripts and output files that will log performance data. # Set up the scripts and output files that will log performance data.
lun_list=$(pool_to_lun_list $PERFPOOL) lun_list=$(pool_to_lun_list $PERFPOOL)
@ -100,6 +82,6 @@ else
) )
fi fi
log_note "Random writes with $PERF_RUNTYPE settings" log_note "Random writes with settings: $(print_perf_settings)"
do_fio_run random_writes.fio true false do_fio_run random_writes.fio true false
log_pass "Measure IO stats during random write load" log_pass "Measure IO stats during random write load"

31
tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh
View File

@ -12,7 +12,7 @@
# #
# #
# Copyright (c) 2015, 2020 by Delphix. All rights reserved. # Copyright (c) 2015, 2021 by Delphix. All rights reserved.
# #
. $STF_SUITE/include/libtest.shlib . $STF_SUITE/include/libtest.shlib
@ -43,28 +43,10 @@ recreate_perf_pool
# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio. # Aim to fill the pool to 50% capacity while accounting for a 3x compressratio.
export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2)) export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then # Variables specific to this test for use by fio.
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 16 64'}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0 1'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'} export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'1 2 4 8 16 32 64 128'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0 1'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 16 64'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0 1'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
fi
# Until the performance tests over NFS can deal with multiple file systems, # Until the performance tests over NFS can deal with multiple file systems,
# force the use of only one file system when testing over NFS. # force the use of only one file system when testing over NFS.
@ -95,6 +77,7 @@ else
"dtrace -s $PERF_SCRIPTS/offcpu-profile.d" "offcpu-profile" "dtrace -s $PERF_SCRIPTS/offcpu-profile.d" "offcpu-profile"
) )
fi fi
log_note "ZIL specific random write workload with $PERF_RUNTYPE settings" log_note \
"ZIL specific random write workload with settings: $(print_perf_settings)"
do_fio_run random_writes.fio true false do_fio_run random_writes.fio true false
log_pass "Measure IO stats during ZIL specific random write workload" log_pass "Measure IO stats during ZIL specific random write workload"

30
tests/zfs-tests/tests/perf/regression/sequential_reads.ksh
View File

@ -12,7 +12,7 @@
# #
# #
# Copyright (c) 2015, 2020 by Delphix. All rights reserved. # Copyright (c) 2015, 2021 by Delphix. All rights reserved.
# #
# #
@ -55,28 +55,10 @@ populate_perf_filesystems
# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio. # Aim to fill the pool to 50% capacity while accounting for a 3x compressratio.
export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2)) export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
# Variables for use by fio. # Variables specific to this test for use by fio.
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'8 16'}
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'} export PERF_IOSIZES=${PERF_IOSIZES:-'128k 1m'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'8 16'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'128k 1m'}
fi
# Layout the files to be used by the read tests. Create as many files as the # Layout the files to be used by the read tests. Create as many files as the
# largest number of threads. An fio run with fewer threads will use a subset # largest number of threads. An fio run with fewer threads will use a subset
@ -111,6 +93,6 @@ else
) )
fi fi
log_note "Sequential reads with $PERF_RUNTYPE settings" log_note "Sequential reads with settings: $(print_perf_settings)"
do_fio_run sequential_reads.fio false true do_fio_run sequential_reads.fio false true
log_pass "Measure IO stats during sequential read load" log_pass "Measure IO stats during sequential read load"

30
tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh
View File

@ -12,7 +12,7 @@
# #
# #
# Copyright (c) 2015, 2020 by Delphix. All rights reserved. # Copyright (c) 2015, 2021 by Delphix. All rights reserved.
# #
# #
@ -45,28 +45,10 @@ populate_perf_filesystems
# Make sure the working set can be cached in the arc. Aim for 1/2 of arc. # Make sure the working set can be cached in the arc. Aim for 1/2 of arc.
export TOTAL_SIZE=$(($(get_max_arc_size) / 2)) export TOTAL_SIZE=$(($(get_max_arc_size) / 2))
# Variables for use by fio. # Variables specific to this test for use by fio.
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'} export PERF_IOSIZES=${PERF_IOSIZES:-'128k 1m'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'128k 1m'}
fi
# Layout the files to be used by the read tests. Create as many files as the # Layout the files to be used by the read tests. Create as many files as the
# largest number of threads. An fio run with fewer threads will use a subset # largest number of threads. An fio run with fewer threads will use a subset
@ -101,6 +83,6 @@ else
) )
fi fi
log_note "Sequential cached reads with $PERF_RUNTYPE settings" log_note "Sequential cached reads with settings: $(print_perf_settings)"
do_fio_run sequential_reads.fio false false do_fio_run sequential_reads.fio false false
log_pass "Measure IO stats during sequential cached read load" log_pass "Measure IO stats during sequential cached read load"

31
tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh
View File

@ -12,7 +12,7 @@
# #
# #
# Copyright (c) 2015, 2020 by Delphix. All rights reserved. # Copyright (c) 2015, 2021 by Delphix. All rights reserved.
# #
# #
@ -51,28 +51,10 @@ populate_perf_filesystems
# Make sure the working set can be cached in the arc. Aim for 1/2 of arc. # Make sure the working set can be cached in the arc. Aim for 1/2 of arc.
export TOTAL_SIZE=$(($(get_max_arc_size) / 2)) export TOTAL_SIZE=$(($(get_max_arc_size) / 2))
# Variables for use by fio. # Variables specific to this test for use by fio.
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'} export PERF_IOSIZES=${PERF_IOSIZES:-'128k 1m'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'128k 1m'}
fi
# Layout the files to be used by the read tests. Create as many files as the # Layout the files to be used by the read tests. Create as many files as the
# largest number of threads. An fio run with fewer threads will use a subset # largest number of threads. An fio run with fewer threads will use a subset
@ -127,6 +109,7 @@ else
) )
fi fi
log_note "Sequential cached reads from $DIRECTORY with $PERF_RUNTYPE settings" log_note "Sequential cached reads from $DIRECTORY with " \
"ettings: $(print_perf_settings)"
do_fio_run sequential_reads.fio false false do_fio_run sequential_reads.fio false false
log_pass "Measure IO stats during sequential cached read load" log_pass "Measure IO stats during sequential cached read load"

32
tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh
View File

@ -12,7 +12,7 @@
# #
# #
# Copyright (c) 2016, 2020 by Delphix. All rights reserved. # Copyright (c) 2016, 2021 by Delphix. All rights reserved.
# #
# #
@ -47,30 +47,12 @@ recreate_perf_pool
populate_perf_filesystems populate_perf_filesystems
# Ensure the working set can be cached in the dbuf cache. # Ensure the working set can be cached in the dbuf cache.
export TOTAL_SIZE=$(($(get_max_dbuf_cache_size) * 3 / 4)) export TOTAL_SIZE=$(($(get_dbuf_cache_size) * 3 / 4))
# Variables for use by fio. # Variables specific to this test for use by fio.
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'64'}
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'} export PERF_IOSIZES=${PERF_IOSIZES:-'64k'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'}
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'64'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'64k'}
fi
# Layout the files to be used by the read tests. Create as many files as the # Layout the files to be used by the read tests. Create as many files as the
# largest number of threads. An fio run with fewer threads will use a subset # largest number of threads. An fio run with fewer threads will use a subset
@ -107,6 +89,6 @@ else
) )
fi fi
log_note "Sequential cached reads with $PERF_RUNTYPE settings" log_note "Sequential cached reads with settings: $(print_perf_settings)"
do_fio_run sequential_reads.fio false false do_fio_run sequential_reads.fio false false
log_pass "Measure IO stats during sequential cached read load" log_pass "Measure IO stats during sequential cached read load"

30
tests/zfs-tests/tests/perf/regression/sequential_writes.ksh
View File

@ -12,7 +12,7 @@
# #
# #
# Copyright (c) 2015, 2020 by Delphix. All rights reserved. # Copyright (c) 2015, 2021 by Delphix. All rights reserved.
# #
# #
@ -54,28 +54,10 @@ populate_perf_filesystems
# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio. # Aim to fill the pool to 50% capacity while accounting for a 3x compressratio.
export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2)) export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2))
# Variables for use by fio. # Variables specific to this test for use by fio.
if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'16 32'}
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'} export PERF_IOSIZES=${PERF_IOSIZES:-'8k 128k 1m'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 8 16 32 64 128'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 256k'}
elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
export PERF_RANDSEED=${PERF_RANDSEED:-'1234'}
export PERF_COMPPERCENT=${PERF_COMPPERCENT:-'66'}
export PERF_COMPCHUNK=${PERF_COMPCHUNK:-'4096'}
export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
export PERF_NTHREADS=${PERF_NTHREADS:-'16 32'}
export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'}
export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
export PERF_IOSIZES=${PERF_IOSIZES:-'8k 128k 1m'}
fi
# Set up the scripts and output files that will log performance data. # Set up the scripts and output files that will log performance data.
lun_list=$(pool_to_lun_list $PERFPOOL) lun_list=$(pool_to_lun_list $PERFPOOL)
@ -100,6 +82,6 @@ else
) )
fi fi
log_note "Sequential writes with $PERF_RUNTYPE settings" log_note "Sequential writes with settings: $(print_perf_settings)"
do_fio_run sequential_writes.fio true false do_fio_run sequential_writes.fio true false
log_pass "Measure IO stats during sequential write load" log_pass "Measure IO stats during sequential write load"

3
tests/zfs-tests/tests/perf/regression/setup.ksh
View File

@ -12,12 +12,11 @@
# #
# #
# Copyright (c) 2015 by Delphix. All rights reserved. # Copyright (c) 2015, 2021 by Delphix. All rights reserved.
# #
. $STF_SUITE/include/libtest.shlib . $STF_SUITE/include/libtest.shlib
verify_runnable "global" verify_runnable "global"
verify_disk_count "$DISKS" 3
log_pass log_pass