From 96342996577aac97fc9eaa0c0a9f74377dc9e336 Mon Sep 17 00:00:00 2001 From: Ahmed Ghanem Date: Sun, 11 Feb 2018 16:11:59 -0700 Subject: [PATCH] OpenZFS 9185 - Enable testing over NFS in ZFS performance tests This change makes additions to the ZFS test suite that allows the performance tests to run over NFS. The test is run and performance data collected from the server side, while IO is generated on the NFS client. This has been tested with Linux and illumos NFS clients. Authored by: Ahmed Ghanem Reviewed by: Dan Kimmel Reviewed by: John Kennedy Reviewed by: Kevin Greene Reviewed-by: Richard Elling Reviewed-by: Brian Behlendorf Ported-by: John Kennedy Signed-off-by: John Kennedy OpenZFS-issue: https://www.illumos.org/issues/9185 Closes #8367 --- scripts/zfs-tests.sh | 9 +++- tests/zfs-tests/include/commands.cfg | 2 + tests/zfs-tests/tests/perf/Makefile.am | 4 +- .../zfs-tests/tests/perf/fio/random_reads.fio | 1 + .../tests/perf/fio/random_readwrite.fio | 1 + .../tests/perf/fio/random_readwrite_fixed.fio | 1 + .../tests/perf/fio/random_writes.fio | 1 + .../tests/perf/fio/sequential_reads.fio | 1 + .../tests/perf/fio/sequential_writes.fio | 1 + tests/zfs-tests/tests/perf/nfs-sample.cfg | 46 ++++++++++++++++++ tests/zfs-tests/tests/perf/perf.shlib | 47 ++++++++++++++++++- .../perf/regression/random_writes_zil.ksh | 4 ++ 12 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 tests/zfs-tests/tests/perf/nfs-sample.cfg diff --git a/scripts/zfs-tests.sh b/scripts/zfs-tests.sh index f00a284847..7c5286ba70 100755 --- a/scripts/zfs-tests.sh +++ b/scripts/zfs-tests.sh @@ -273,6 +273,7 @@ OPTIONS: -f Use files only, disables block device tests -S Enable stack tracer (negative performance impact) -c Only create and populate constrained path + -n NFSFILE Use the nfsfile to determine the NFS configuration -I NUM Number of iterations -d DIR Use DIR for files and loopback devices -s SIZE Use vdevs of SIZE (default: 4G) @@ -295,7 +296,7 @@ $0 -x EOF } -while getopts 'hvqxkfScd:s:r:?t:T:u:I:' OPTION; do +while getopts 'hvqxkfScn:d:s:r:?t:T:u:I:' OPTION; do case $OPTION in h) usage @@ -324,6 +325,12 @@ while getopts 'hvqxkfScd:s:r:?t:T:u:I:' OPTION; do constrain_path exit ;; + n) + nfsfile=$OPTARG + [[ -f $nfsfile ]] || fail "Cannot read file: $nfsfile" + export NFS=1 + . "$nfsfile" + ;; d) FILEDIR="$OPTARG" ;; diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg index 78ba2488d4..127a1477d4 100644 --- a/tests/zfs-tests/include/commands.cfg +++ b/tests/zfs-tests/include/commands.cfg @@ -100,6 +100,7 @@ export SYSTEM_FILES='arp readlink rm rmdir + scp sed seq setenforce @@ -110,6 +111,7 @@ export SYSTEM_FILES='arp shuf sleep sort + ssh stat strings su diff --git a/tests/zfs-tests/tests/perf/Makefile.am b/tests/zfs-tests/tests/perf/Makefile.am index d31bba0e22..68dd31ec12 100644 --- a/tests/zfs-tests/tests/perf/Makefile.am +++ b/tests/zfs-tests/tests/perf/Makefile.am @@ -1,5 +1,7 @@ pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/perf -dist_pkgdata_DATA = perf.shlib +dist_pkgdata_SCRIPTS = \ + nfs-sample.cfg \ + perf.shlib SUBDIRS = \ fio \ diff --git a/tests/zfs-tests/tests/perf/fio/random_reads.fio b/tests/zfs-tests/tests/perf/fio/random_reads.fio index 79610f9b28..e6e7034e0a 100644 --- a/tests/zfs-tests/tests/perf/fio/random_reads.fio +++ b/tests/zfs-tests/tests/perf/fio/random_reads.fio @@ -26,6 +26,7 @@ runtime=${RUNTIME} bs=${BLOCKSIZE} ioengine=psync sync=${SYNC_TYPE} +direct=${DIRECT} numjobs=${NUMJOBS} [job] diff --git a/tests/zfs-tests/tests/perf/fio/random_readwrite.fio b/tests/zfs-tests/tests/perf/fio/random_readwrite.fio index 7d01c38ada..852d4bed69 100644 --- a/tests/zfs-tests/tests/perf/fio/random_readwrite.fio +++ b/tests/zfs-tests/tests/perf/fio/random_readwrite.fio @@ -28,6 +28,7 @@ runtime=${RUNTIME} bssplit=4k/50:8k/30:128k/10:1m/10 ioengine=psync sync=${SYNC_TYPE} +direct=${DIRECT} numjobs=${NUMJOBS} buffer_compress_percentage=66 buffer_compress_chunk=4096 diff --git a/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio b/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio index ed44955532..67b88c09d7 100644 --- a/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio +++ b/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio @@ -28,6 +28,7 @@ runtime=${RUNTIME} bs=${BLOCKSIZE} ioengine=psync sync=${SYNC_TYPE} +direct=${DIRECT} numjobs=${NUMJOBS} buffer_compress_percentage=66 buffer_compress_chunk=4096 diff --git a/tests/zfs-tests/tests/perf/fio/random_writes.fio b/tests/zfs-tests/tests/perf/fio/random_writes.fio index 5e2cb30026..90db5ce3bf 100644 --- a/tests/zfs-tests/tests/perf/fio/random_writes.fio +++ b/tests/zfs-tests/tests/perf/fio/random_writes.fio @@ -25,6 +25,7 @@ runtime=${RUNTIME} bs=${BLOCKSIZE} ioengine=psync sync=${SYNC_TYPE} +direct=${DIRECT} numjobs=${NUMJOBS} filesize=${FILESIZE} buffer_compress_percentage=66 diff --git a/tests/zfs-tests/tests/perf/fio/sequential_reads.fio b/tests/zfs-tests/tests/perf/fio/sequential_reads.fio index 33a9a1d893..b4b45e0841 100644 --- a/tests/zfs-tests/tests/perf/fio/sequential_reads.fio +++ b/tests/zfs-tests/tests/perf/fio/sequential_reads.fio @@ -26,6 +26,7 @@ runtime=${RUNTIME} bs=${BLOCKSIZE} ioengine=psync sync=${SYNC_TYPE} +direct=${DIRECT} numjobs=${NUMJOBS} [job] diff --git a/tests/zfs-tests/tests/perf/fio/sequential_writes.fio b/tests/zfs-tests/tests/perf/fio/sequential_writes.fio index 65a65910fd..714993e92f 100644 --- a/tests/zfs-tests/tests/perf/fio/sequential_writes.fio +++ b/tests/zfs-tests/tests/perf/fio/sequential_writes.fio @@ -25,6 +25,7 @@ runtime=${RUNTIME} bs=${BLOCKSIZE} ioengine=psync sync=${SYNC_TYPE} +direct=${DIRECT} numjobs=${NUMJOBS} filesize=${FILESIZE} buffer_compress_percentage=66 diff --git a/tests/zfs-tests/tests/perf/nfs-sample.cfg b/tests/zfs-tests/tests/perf/nfs-sample.cfg new file mode 100644 index 0000000000..f7ac2dae35 --- /dev/null +++ b/tests/zfs-tests/tests/perf/nfs-sample.cfg @@ -0,0 +1,46 @@ +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2016 by Delphix. All rights reserved. +# + +# +# This file is a sample NFS configuration for the performance tests. To use the +# performance tests over NFS you must have: +# - a client machine with fio and sudo installed +# - passwordless SSH set up from this host +# for delphix and root users to the client +# - passwordless sudo for the user on the client +# + + +# The IP address for the server +export NFS_SERVER=127.0.0.1 + +# The IP address for the client +export NFS_CLIENT=127.0.0.1 + +# The mountpoint to use inside the client +export NFS_MOUNT=/var/tmp/nfs + +# The user to run the tests as on the client +export NFS_USER=delphix + +# Common NFS client mount options +export NFS_OPTIONS="-o rw,nosuid,bg,hard,rsize=1048576,wsize=1048576," +NFS_OPTIONS+="nointr,timeo=600,proto=tcp,actimeo=0,port=2049" + +# illumos NFS client mount options +# export NFS_OPTIONS="$NFS_OPTIONS,vers=3" + +# Linux NFS client mount options +export NFS_OPTIONS="-t nfs $NFS_OPTIONS,noacl,nfsvers=3" diff --git a/tests/zfs-tests/tests/perf/perf.shlib b/tests/zfs-tests/tests/perf/perf.shlib index 7165df759b..69e61e9fd1 100644 --- a/tests/zfs-tests/tests/perf/perf.shlib +++ b/tests/zfs-tests/tests/perf/perf.shlib @@ -112,6 +112,16 @@ function do_fio_run_impl export BLOCKSIZE=$iosize sync + # When running locally, we want to keep the default behavior of + # DIRECT == 0, so only set it when we're running over NFS to + # disable client cache for reads. + if [[ $NFS -eq 1 ]]; then + export DIRECT=1 + do_setup_nfs $script + else + export DIRECT=0 + fi + # This will be part of the output filename. typeset suffix=$(get_suffix $threads $sync $iosize) @@ -124,7 +134,14 @@ function do_fio_run_impl typeset outfile="$logbase.fio.$suffix" # Start the load - log_must fio --output $outfile $FIO_SCRIPTS/$script + if [[ $NFS -eq 1 ]]; then + log_must ssh -t $NFS_USER@$NFS_CLIENT " + fio --output /tmp/fio.out /tmp/test.fio + " + log_must scp $NFS_USER@$NFS_CLIENT:/tmp/fio.out $outfile + else + log_must fio --output $outfile $FIO_SCRIPTS/$script + fi } # @@ -166,6 +183,34 @@ function do_fio_run done } +# This function sets NFS mount on the client and make sure all correct +# permissions are in place +# +function do_setup_nfs +{ + typeset script=$1 + zfs set sharenfs=on $TESTFS + log_must chmod -R 777 /$TESTFS + + ssh -t $NFS_USER@$NFS_CLIENT "mkdir -m 777 -p $NFS_MOUNT" + ssh -t $NFS_USER@$NFS_CLIENT "sudo -S umount $NFS_MOUNT" + log_must ssh -t $NFS_USER@$NFS_CLIENT " + sudo -S mount $NFS_OPTIONS $NFS_SERVER:/$TESTFS $NFS_MOUNT + " + # + # The variables in the fio script are only available in our current + # shell session, so we have to evaluate them here before copying + # the resulting script over to the target machine. + # + export jobnum='$jobnum' + while read line; do + eval echo "$line" + done < $FIO_SCRIPTS/$script > /tmp/test.fio + log_must sed -i -e "s%directory.*%directory=$NFS_MOUNT%" /tmp/test.fio + log_must scp /tmp/test.fio $NFS_USER@$NFS_CLIENT:/tmp + log_must rm /tmp/test.fio +} + # # This function iterates through the value pairs in $PERF_COLLECT_SCRIPTS. # The script at index N is launched in the background, with its output diff --git a/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh b/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh index 0f3256351c..e0b2532009 100755 --- a/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh +++ b/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh @@ -60,6 +60,10 @@ elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then export PERF_IOSIZES=${PERF_IOSIZES:-'8k'} fi +# Until the performance tests over NFS can deal with multiple file systems, +# force the use of only one file system when testing over NFS. +[[ $NFS -eq 1 ]] && PERF_NTHREADS_PER_FS='0' + lun_list=$(pool_to_lun_list $PERFPOOL) log_note "Collecting backend IO stats with lun list $lun_list" if is_linux; then