Allow test-runner to filter test groups by tag

Enable test-runner to accept a list of tags to identify
which test groups the user wishes to run.

Also allow test-runner to perform multiple iterations
of a test run.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: John Wren Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Melikov <mail@gmelikov.ru>
Signed-off-by: Giuseppe Di Natale <dinatale2@llnl.gov>
Closes #6788

scripts/zfs-tests.sh

@@ -41,6 +41,8 @@ FILEDIR=${FILEDIR:-/var/tmp}
 DISKS=${DISKS:-""}
 SINGLETEST=()
 SINGLETESTUSER="root"
+TAGS="functional"
+ITERATIONS=1
 ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
 ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
 ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
@@ -265,10 +267,12 @@ OPTIONS:
 	-k          Disable cleanup after test failure
 	-f          Use files only, disables block device tests
 	-c          Only create and populate constrained path
+	-I NUM      Number of iterations
 	-d DIR      Use DIR for files and loopback devices
 	-s SIZE     Use vdevs of SIZE (default: 4G)
 	-r RUNFILE  Run tests in RUNFILE (default: linux.run)
 	-t PATH     Run single test at PATH relative to test suite
+	-T TAGS     Comma separated list of tags
 	-u USER     Run single test as USER (default: root)
 
 EXAMPLES:
@@ -285,7 +289,7 @@ $0 -x
 EOF
 }
 
-while getopts 'hvqxkfcd:s:r:?t:u:' OPTION; do
+while getopts 'hvqxkfcd:s:r:?t:T:u:I:' OPTION; do
 	case $OPTION in
 	h)
 		usage
@@ -314,6 +318,12 @@ while getopts 'hvqxkfcd:s:r:?t:u:' OPTION; do
 	d)
 		FILEDIR="$OPTARG"
 		;;
+	I)
+		ITERATIONS="$OPTARG"
+		if [ "$ITERATIONS" -le 0 ]; then
+			fail "Iterations must be greater than 0."
+		fi
+		;;
 	s)
 		FILESIZE="$OPTARG"
 		;;
@@ -326,6 +336,9 @@ while getopts 'hvqxkfcd:s:r:?t:u:' OPTION; do
 		fi
 		SINGLETEST+=("$OPTARG")
 		;;
+	T)
+		TAGS="$OPTARG"
+		;;
 	u)
 		SINGLETESTUSER="$OPTARG"
 		;;
@@ -529,6 +542,8 @@ msg "LOOPBACKS:       $LOOPBACKS"
 msg "DISKS:           $DISKS"
 msg "NUM_DISKS:       $NUM_DISKS"
 msg "FILESIZE:        $FILESIZE"
+msg "ITERATIONS:      $ITERATIONS"
+msg "TAGS:            $TAGS"
 msg "Keep pool(s):    $KEEP"
 msg "Missing util(s): $STF_MISSING_BIN"
 msg ""
@@ -543,8 +558,10 @@ export __ZFS_POOL_EXCLUDE
 export TESTFAIL_CALLBACKS
 export PATH=$STF_PATH
 
-msg "${TEST_RUNNER} ${QUIET} -c ${RUNFILE} -i ${STF_SUITE}"
+msg "${TEST_RUNNER} ${QUIET} -c ${RUNFILE} -T ${TAGS} -i ${STF_SUITE}" \
-${TEST_RUNNER} ${QUIET} -c "${RUNFILE}" -i "${STF_SUITE}"
+    "-I ${ITERATIONS}"
+${TEST_RUNNER} ${QUIET} -c "${RUNFILE}" -T "${TAGS}" -i "${STF_SUITE}" \
+    -I "${ITERATIONS}"
 RESULT=$?
 echo
 

tests/runfiles/linux.run

@@ -18,29 +18,35 @@ timeout = 600
 post_user = root
 post = cleanup
 outputdir = /var/tmp/test_results
+tags = ['functional']
 
 # Update to use ZFS_ACL_* variables and user_run helper.
 # posix_001_pos
 # posix_002_pos
 [tests/functional/acl/posix]
 tests = ['posix_003_pos']
+tags = ['functional', 'acl', 'posix']
 
 [tests/functional/atime]
 tests = ['atime_001_pos', 'atime_002_neg', 'atime_003_pos']
+tags = ['functional', 'atime']
 
 [tests/functional/bootfs]
 tests = ['bootfs_001_pos', 'bootfs_002_neg', 'bootfs_003_pos',
     'bootfs_004_neg', 'bootfs_005_neg', 'bootfs_006_pos', 'bootfs_007_pos',
     'bootfs_008_pos']
+tags = ['functional', 'bootfs']
 
 [tests/functional/cache]
 tests = ['cache_001_pos', 'cache_002_pos', 'cache_003_pos', 'cache_004_neg',
     'cache_005_neg', 'cache_006_pos', 'cache_007_neg', 'cache_008_neg',
     'cache_009_pos', 'cache_010_neg', 'cache_011_pos']
+tags = ['functional', 'cache']
 
 [tests/functional/cachefile]
 tests = ['cachefile_001_pos', 'cachefile_002_pos', 'cachefile_003_pos',
     'cachefile_004_pos']
+tags = ['functional', 'cachefile']
 
 # 'sensitive_none_lookup', 'sensitive_none_delete',
 # 'sensitive_formd_lookup', 'sensitive_formd_delete',
@@ -50,43 +56,53 @@ tests = ['cachefile_001_pos', 'cachefile_002_pos', 'cachefile_003_pos',
 # 'mixed_formd_lookup', 'mixed_formd_lookup_ci', 'mixed_formd_delete']
 [tests/functional/casenorm]
 tests = ['case_all_values', 'norm_all_values']
+tags = ['functional', 'casenorm']
 
 [tests/functional/chattr]
 tests = ['chattr_001_pos', 'chattr_002_neg']
+tags = ['functional', 'chattr']
 
 [tests/functional/checksum]
 tests = ['run_edonr_test', 'run_sha2_test', 'run_skein_test', 'filetest_001_pos']
+tags = ['functional', 'checksum']
 
 [tests/functional/clean_mirror]
 tests = [ 'clean_mirror_001_pos', 'clean_mirror_002_pos',
     'clean_mirror_003_pos', 'clean_mirror_004_pos']
+tags = ['functional', 'clean_mirror']
 
 [tests/functional/cli_root/zdb]
 tests = ['zdb_001_neg', 'zdb_002_pos', 'zdb_003_pos', 'zdb_004_pos',
     'zdb_005_pos']
 pre =
 post =
+tags = ['functional', 'cli_root', 'zdb']
 
 [tests/functional/cli_root/zfs]
 tests = ['zfs_001_neg', 'zfs_002_pos', 'zfs_003_neg']
+tags = ['functional', 'cli_root', 'zfs']
 
 [tests/functional/cli_root/zfs_bookmark]
 tests = ['zfs_bookmark_cliargs']
+tags = ['functional', 'cli_root', 'zfs_bookmark']
 
 [tests/functional/cli_root/zfs_change-key]
 tests = ['zfs_change-key', 'zfs_change-key_child', 'zfs_change-key_format',
     'zfs_change-key_inherit', 'zfs_change-key_load', 'zfs_change-key_location',
     'zfs_change-key_pbkdf2iters']
+tags = ['functional', 'cli_root', 'zfs_change-key']
 
 [tests/functional/cli_root/zfs_clone]
 tests = ['zfs_clone_001_neg', 'zfs_clone_002_pos', 'zfs_clone_003_pos',
     'zfs_clone_004_pos', 'zfs_clone_005_pos', 'zfs_clone_006_pos',
     'zfs_clone_007_pos', 'zfs_clone_008_neg', 'zfs_clone_009_neg',
     'zfs_clone_010_pos', 'zfs_clone_encrypted']
+tags = ['functional', 'cli_root', 'zfs_clone']
 
 [tests/functional/cli_root/zfs_copies]
 tests = ['zfs_copies_001_pos', 'zfs_copies_002_pos', 'zfs_copies_003_pos',
     'zfs_copies_004_neg', 'zfs_copies_005_neg', 'zfs_copies_006_pos']
+tags = ['functional', 'cli_root', 'zfs_copies']
 
 [tests/functional/cli_root/zfs_create]
 tests = ['zfs_create_001_pos', 'zfs_create_002_pos', 'zfs_create_003_pos',
@@ -95,6 +111,7 @@ tests = ['zfs_create_001_pos', 'zfs_create_002_pos', 'zfs_create_003_pos',
     'zfs_create_010_neg', 'zfs_create_011_pos', 'zfs_create_012_pos',
     'zfs_create_013_pos', 'zfs_create_014_pos', 'zfs_create_encrypted',
     'zfs_create_crypt_combos']
+tags = ['functional', 'cli_root', 'zfs_create']
 
 [tests/functional/cli_root/zfs_destroy]
 tests = ['zfs_destroy_001_pos', 'zfs_destroy_002_pos', 'zfs_destroy_003_pos',
@@ -103,22 +120,27 @@ tests = ['zfs_destroy_001_pos', 'zfs_destroy_002_pos', 'zfs_destroy_003_pos',
     'zfs_destroy_010_pos', 'zfs_destroy_011_pos', 'zfs_destroy_012_pos',
     'zfs_destroy_013_neg', 'zfs_destroy_014_pos', 'zfs_destroy_015_pos',
     'zfs_destroy_016_pos']
+tags = ['functional', 'cli_root', 'zfs_destroy']
 
 [tests/functional/cli_root/zfs_diff]
 tests = ['zfs_diff_changes', 'zfs_diff_cliargs', 'zfs_diff_timestamp',
     'zfs_diff_types']
+tags = ['functional', 'cli_root', 'zfs_diff']
 
 [tests/functional/cli_root/zfs_get]
 tests = ['zfs_get_001_pos', 'zfs_get_002_pos', 'zfs_get_003_pos',
     'zfs_get_004_pos', 'zfs_get_005_neg', 'zfs_get_006_neg', 'zfs_get_007_neg',
     'zfs_get_008_pos', 'zfs_get_009_pos', 'zfs_get_010_neg']
+tags = ['functional', 'cli_root', 'zfs_get']
 
 [tests/functional/cli_root/zfs_inherit]
 tests = ['zfs_inherit_001_neg', 'zfs_inherit_002_neg', 'zfs_inherit_003_pos']
+tags = ['functional', 'cli_root', 'zfs_inherit']
 
 [tests/functional/cli_root/zfs_load-key]
 tests = ['zfs_load-key', 'zfs_load-key_all', 'zfs_load-key_file',
     'zfs_load-key_location', 'zfs_load-key_noop', 'zfs_load-key_recursive']
+tags = ['functional', 'cli_root', 'zfs_load-key']
 
 # zfs_mount_006_pos - https://github.com/zfsonlinux/zfs/issues/4990
 [tests/functional/cli_root/zfs_mount]
@@ -127,14 +149,17 @@ tests = ['zfs_mount_001_pos', 'zfs_mount_002_pos', 'zfs_mount_003_pos',
     'zfs_mount_008_pos', 'zfs_mount_009_neg', 'zfs_mount_010_neg',
     'zfs_mount_011_neg', 'zfs_mount_012_neg', 'zfs_mount_all_001_pos',
     'zfs_mount_encrypted', 'zfs_mount_remount']
+tags = ['functional', 'cli_root', 'zfs_mount']
 
 [tests/functional/cli_root/zfs_promote]
 tests = ['zfs_promote_001_pos', 'zfs_promote_002_pos', 'zfs_promote_003_pos',
     'zfs_promote_004_pos', 'zfs_promote_005_pos', 'zfs_promote_006_neg',
     'zfs_promote_007_neg', 'zfs_promote_008_pos', 'zfs_promote_encryptionroot']
+tags = ['functional', 'cli_root', 'zfs_promote']
 
 [tests/functional/cli_root/zfs_property]
 tests = ['zfs_written_property_001_pos']
+tags = ['functional', 'cli_root', 'zfs_property']
 
 # zfs_receive_004_neg - Fails for OpenZFS on illumos
 [tests/functional/cli_root/zfs_receive]
@@ -146,6 +171,7 @@ tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos',
     'receive-o-x_props_override', 'zfs_receive_from_encrypted',
     'zfs_receive_to_encrypted', 'zfs_receive_raw',
     'zfs_receive_raw_incremental']
+tags = ['functional', 'cli_root', 'zfs_receive']
 
 # zfs_rename_006_pos - https://github.com/zfsonlinux/zfs/issues/5647
 # zfs_rename_009_neg - https://github.com/zfsonlinux/zfs/issues/5648
@@ -156,19 +182,23 @@ tests = ['zfs_rename_001_pos', 'zfs_rename_002_pos', 'zfs_rename_003_pos',
     'zfs_rename_010_neg', 'zfs_rename_011_pos', 'zfs_rename_012_neg',
     'zfs_rename_013_pos', 'zfs_rename_encrypted_child',
     'zfs_rename_to_encrypted']
+tags = ['functional', 'cli_root', 'zfs_rename']
 
 [tests/functional/cli_root/zfs_reservation]
 tests = ['zfs_reservation_001_pos', 'zfs_reservation_002_pos']
+tags = ['functional', 'cli_root', 'zfs_reservation']
 
 [tests/functional/cli_root/zfs_rollback]
 tests = ['zfs_rollback_001_pos', 'zfs_rollback_002_pos',
     'zfs_rollback_003_neg', 'zfs_rollback_004_neg']
+tags = ['functional', 'cli_root', 'zfs_rollback']
 
 [tests/functional/cli_root/zfs_send]
 tests = ['zfs_send_001_pos', 'zfs_send_002_pos', 'zfs_send_003_pos',
     'zfs_send_004_neg', 'zfs_send_005_pos', 'zfs_send_006_pos',
     'zfs_send_007_pos', 'zfs_send_encrypted', 'zfs_send_raw',
     'zfs_send_sparse']
+tags = ['functional', 'cli_root', 'zfs_send']
 
 [tests/functional/cli_root/zfs_set]
 tests = ['cache_001_pos', 'cache_002_neg', 'canmount_001_pos',
@@ -180,39 +210,47 @@ tests = ['cache_001_pos', 'cache_002_neg', 'canmount_001_pos',
     'user_property_004_pos', 'version_001_neg', 'zfs_set_001_neg',
     'zfs_set_002_neg', 'zfs_set_003_neg', 'property_alias_001_pos',
     'mountpoint_003_pos', 'ro_props_001_pos', 'zfs_set_keylocation']
+tags = ['functional', 'cli_root', 'zfs_set']
 
 [tests/functional/cli_root/zfs_share]
 tests = ['zfs_share_001_pos', 'zfs_share_002_pos', 'zfs_share_003_pos',
     'zfs_share_004_pos', 'zfs_share_005_pos', 'zfs_share_006_pos',
     'zfs_share_007_neg', 'zfs_share_008_neg', 'zfs_share_009_neg',
     'zfs_share_010_neg', 'zfs_share_011_pos']
+tags = ['functional', 'cli_root', 'zfs_share']
 
 [tests/functional/cli_root/zfs_snapshot]
 tests = ['zfs_snapshot_001_neg', 'zfs_snapshot_002_neg',
     'zfs_snapshot_003_neg', 'zfs_snapshot_004_neg', 'zfs_snapshot_005_neg',
     'zfs_snapshot_006_pos', 'zfs_snapshot_007_neg', 'zfs_snapshot_008_neg',
     'zfs_snapshot_009_pos']
+tags = ['functional', 'cli_root', 'zfs_snapshot']
 
 [tests/functional/cli_root/zfs_unload-key]
 tests = ['zfs_unload-key', 'zfs_unload-key_all', 'zfs_unload-key_recursive']
+tags = ['functional', 'cli_root', 'zfs_unload-key']
 
 [tests/functional/cli_root/zfs_unmount]
 tests = ['zfs_unmount_001_pos', 'zfs_unmount_002_pos', 'zfs_unmount_003_pos',
     'zfs_unmount_004_pos', 'zfs_unmount_005_pos', 'zfs_unmount_006_pos',
     'zfs_unmount_007_neg', 'zfs_unmount_008_neg', 'zfs_unmount_009_pos',
     'zfs_unmount_all_001_pos']
+tags = ['functional', 'cli_root', 'zfs_unmount']
 
 [tests/functional/cli_root/zfs_unshare]
 tests = ['zfs_unshare_001_pos', 'zfs_unshare_002_pos', 'zfs_unshare_003_pos',
     'zfs_unshare_004_neg', 'zfs_unshare_005_neg', 'zfs_unshare_006_pos']
+tags = ['functional', 'cli_root', 'zfs_unshare']
 
 [tests/functional/cli_root/zfs_upgrade]
 tests = ['zfs_upgrade_001_pos', 'zfs_upgrade_002_pos', 'zfs_upgrade_003_pos',
     'zfs_upgrade_004_pos', 'zfs_upgrade_005_pos', 'zfs_upgrade_006_neg',
     'zfs_upgrade_007_neg']
+tags = ['functional', 'cli_root', 'zfs_upgrade']
 
 [tests/functional/cli_root/zpool]
 tests = ['zpool_001_neg', 'zpool_002_pos', 'zpool_003_pos']
+tags = ['functional', 'cli_root', 'zpool']
 
 [tests/functional/cli_root/zpool_add]
 tests = ['zpool_add_001_pos', 'zpool_add_002_pos', 'zpool_add_003_pos',
@@ -220,13 +258,16 @@ tests = ['zpool_add_001_pos', 'zpool_add_002_pos', 'zpool_add_003_pos',
     'zpool_add_007_neg', 'zpool_add_008_neg', 'zpool_add_009_neg',
     'zpool_add_010_pos',
     'add-o_ashift', 'add_prop_ashift']
+tags = ['functional', 'cli_root', 'zpool_add']
 
 [tests/functional/cli_root/zpool_attach]
 tests = ['zpool_attach_001_neg', 'attach-o_ashift']
+tags = ['functional', 'cli_root', 'zpool_attach']
 
 [tests/functional/cli_root/zpool_clear]
 tests = ['zpool_clear_001_pos', 'zpool_clear_002_neg', 'zpool_clear_003_neg',
     'zpool_clear_readonly']
+tags = ['functional', 'cli_root', 'zpool_clear']
 
 [tests/functional/cli_root/zpool_create]
 tests = ['zpool_create_001_pos', 'zpool_create_002_pos',
@@ -243,34 +284,42 @@ tests = ['zpool_create_001_pos', 'zpool_create_002_pos',
     'zpool_create_features_003_pos', 'zpool_create_features_004_neg',
     'zpool_create_features_005_pos',
     'create-o_ashift']
+tags = ['functional', 'cli_root', 'zpool_create']
 
 [tests/functional/cli_root/zpool_destroy]
 tests = ['zpool_destroy_001_pos', 'zpool_destroy_002_pos',
     'zpool_destroy_003_neg']
 pre =
 post =
+tags = ['functional', 'cli_root', 'zpool_destroy']
 
 [tests/functional/cli_root/zpool_detach]
 tests = ['zpool_detach_001_neg']
+tags = ['functional', 'cli_root', 'zpool_detach']
 
 [tests/functional/cli_root/zpool_events]
 tests = ['zpool_events_clear', 'zpool_events_cliargs', 'zpool_events_follow',
     'zpool_events_poolname']
+tags = ['functional', 'cli_root', 'zpool_events']
 
 [tests/functional/cli_root/zpool_expand]
 tests = ['zpool_expand_001_pos', 'zpool_expand_002_pos',
     'zpool_expand_003_neg', 'zpool_expand_004_pos']
+tags = ['functional', 'cli_root', 'zpool_expand']
 
 [tests/functional/cli_root/zpool_export]
 tests = ['zpool_export_001_pos', 'zpool_export_002_pos',
     'zpool_export_003_neg', 'zpool_export_004_pos']
+tags = ['functional', 'cli_root', 'zpool_export']
 
 [tests/functional/cli_root/zpool_get]
 tests = ['zpool_get_001_pos', 'zpool_get_002_pos', 'zpool_get_003_pos',
     'zpool_get_004_neg']
+tags = ['functional', 'cli_root', 'zpool_get']
 
 [tests/functional/cli_root/zpool_history]
 tests = ['zpool_history_001_neg', 'zpool_history_002_pos']
+tags = ['functional', 'cli_root', 'zpool_history']
 
 [tests/functional/cli_root/zpool_import]
 tests = ['zpool_import_001_pos', 'zpool_import_002_pos',
@@ -283,46 +332,57 @@ tests = ['zpool_import_001_pos', 'zpool_import_002_pos',
     'zpool_import_missing_002_pos', 'zpool_import_missing_003_pos',
     'zpool_import_rename_001_pos', 'zpool_import_all_001_pos',
     'zpool_import_encrypted', 'zpool_import_encrypted_load']
+tags = ['functional', 'cli_root', 'zpool_import']
 
 [tests/functional/cli_root/zpool_labelclear]
 tests = ['zpool_labelclear_active', 'zpool_labelclear_exported']
 pre =
 post =
+tags = ['functional', 'cli_root', 'zpool_labelclear']
 
 [tests/functional/cli_root/zpool_offline]
 tests = ['zpool_offline_001_pos', 'zpool_offline_002_neg', 'zpool_offline_003_pos']
+tags = ['functional', 'cli_root', 'zpool_offline']
 
 [tests/functional/cli_root/zpool_online]
 tests = ['zpool_online_001_pos', 'zpool_online_002_neg']
+tags = ['functional', 'cli_root', 'zpool_online']
 
 [tests/functional/cli_root/zpool_remove]
 tests = ['zpool_remove_001_neg', 'zpool_remove_002_pos',
     'zpool_remove_003_pos']
+tags = ['functional', 'cli_root', 'zpool_remove']
 
 [tests/functional/cli_root/zpool_reopen]
 tests = ['zpool_reopen_001_pos', 'zpool_reopen_002_pos',
     'zpool_reopen_003_pos', 'zpool_reopen_004_pos', 'zpool_reopen_005_pos',
     'zpool_reopen_006_neg']
+tags = ['functional', 'cli_root', 'zpool_reopen']
 
 [tests/functional/cli_root/zpool_replace]
 tests = ['zpool_replace_001_neg', 'replace-o_ashift', 'replace_prop_ashift']
+tags = ['functional', 'cli_root', 'zpool_replace']
 
 [tests/functional/cli_root/zpool_scrub]
 tests = ['zpool_scrub_001_neg', 'zpool_scrub_002_pos', 'zpool_scrub_003_pos',
     'zpool_scrub_004_pos', 'zpool_scrub_005_pos',
     'zpool_scrub_encrypted_unloaded', 'zpool_scrub_offline_device']
+tags = ['functional', 'cli_root', 'zpool_scrub']
 
 [tests/functional/cli_root/zpool_set]
 tests = ['zpool_set_001_pos', 'zpool_set_002_neg', 'zpool_set_003_neg']
+tags = ['functional', 'cli_root', 'zpool_set']
 
 [tests/functional/cli_root/zpool_status]
 tests = ['zpool_status_001_pos', 'zpool_status_002_pos','zpool_status_003_pos',
     'zpool_status_-c_disable', 'zpool_status_-c_homedir',
     'zpool_status_-c_searchpath']
 user =
+tags = ['functional', 'cli_root', 'zpool_status']
 
 [tests/functional/cli_root/zpool_sync]
 tests = ['zpool_sync_001_pos', 'zpool_sync_002_neg']
+tags = ['functional', 'cli_root', 'zpool_sync']
 
 [tests/functional/cli_root/zpool_upgrade]
 tests = ['zpool_upgrade_001_pos', 'zpool_upgrade_002_pos',
@@ -330,6 +390,7 @@ tests = ['zpool_upgrade_001_pos', 'zpool_upgrade_002_pos',
     'zpool_upgrade_005_neg', 'zpool_upgrade_006_neg',
     'zpool_upgrade_007_pos', 'zpool_upgrade_008_pos',
     'zpool_upgrade_009_neg']
+tags = ['functional', 'cli_root', 'zpool_upgrade']
 
 [tests/functional/cli_user/misc]
 tests = ['zdb_001_neg', 'zfs_001_neg', 'zfs_allow_001_neg',
@@ -348,11 +409,13 @@ tests = ['zdb_001_neg', 'zfs_001_neg', 'zfs_allow_001_neg',
     'zpool_status_001_neg', 'zpool_upgrade_001_neg', 'arcstat_001_pos',
     'arc_summary_001_pos', 'dbufstat_001_pos']
 user =
+tags = ['functional', 'cli_user', 'misc']
 
 [tests/functional/cli_user/zfs_list]
 tests = ['zfs_list_001_pos', 'zfs_list_002_pos', 'zfs_list_003_pos',
     'zfs_list_004_neg', 'zfs_list_007_pos', 'zfs_list_008_neg']
 user =
+tags = ['functional', 'cli_user', 'zfs_list']
 
 [tests/functional/cli_user/zpool_iostat]
 tests = ['zpool_iostat_001_neg', 'zpool_iostat_002_pos',
@@ -360,17 +423,21 @@ tests = ['zpool_iostat_001_neg', 'zpool_iostat_002_pos',
     'zpool_iostat_005_pos', 'zpool_iostat_-c_disable',
     'zpool_iostat_-c_homedir', 'zpool_iostat_-c_searchpath']
 user =
+tags = ['functional', 'cli_user', 'zpool_iostat']
 
 [tests/functional/cli_user/zpool_list]
 tests = ['zpool_list_001_pos', 'zpool_list_002_neg']
 user =
+tags = ['functional', 'cli_user', 'zpool_list']
 
 [tests/functional/compression]
 tests = ['compress_001_pos', 'compress_002_pos', 'compress_003_pos',
     'compress_004_pos']
+tags = ['functional', 'compression']
 
 [tests/functional/ctime]
 tests = ['ctime_001_pos' ]
+tags = ['functional', 'ctime']
 
 [tests/functional/delegate]
 tests = ['zfs_allow_001_pos', 'zfs_allow_002_pos',
@@ -380,137 +447,172 @@ tests = ['zfs_allow_001_pos', 'zfs_allow_002_pos',
     'zfs_unallow_001_pos', 'zfs_unallow_002_pos', 'zfs_unallow_003_pos',
     'zfs_unallow_004_pos', 'zfs_unallow_005_pos', 'zfs_unallow_006_pos',
     'zfs_unallow_007_neg', 'zfs_unallow_008_neg']
+tags = ['functional', 'delegate']
 
 [tests/functional/devices]
 tests = ['devices_001_pos', 'devices_002_neg', 'devices_003_pos']
+tags = ['functional', 'devices']
 
 [tests/functional/events]
 tests = ['events_001_pos', 'events_002_pos']
+tags = ['functional', 'events']
 
 [tests/functional/exec]
 tests = ['exec_001_pos', 'exec_002_neg']
+tags = ['functional', 'exec']
 
 [tests/functional/fault]
 tests = ['auto_online_001_pos', 'auto_replace_001_pos', 'auto_spare_001_pos',
     'auto_spare_002_pos.ksh']
+tags = ['functional', 'fault']
 
 [tests/functional/features/async_destroy]
 tests = ['async_destroy_001_pos']
+tags = ['functional', 'features', 'async_destroy']
 
 [tests/functional/features/large_dnode]
 tests = ['large_dnode_001_pos', 'large_dnode_002_pos', 'large_dnode_003_pos',
          'large_dnode_004_neg', 'large_dnode_005_pos', 'large_dnode_006_pos',
          'large_dnode_007_neg', 'large_dnode_008_pos', 'large_dnode_009_pos']
+tags = ['functional', 'features', 'large_dnode']
 
 [tests/functional/grow_pool]
 tests = ['grow_pool_001_pos']
+tags = ['functional', 'grow_pool']
 
 [tests/functional/grow_replicas]
 tests = ['grow_replicas_001_pos']
 pre =
 post =
+tags = ['functional', 'grow_replicas']
 
 [tests/functional/history]
 tests = ['history_001_pos', 'history_002_pos', 'history_003_pos',
     'history_004_pos', 'history_005_neg', 'history_006_neg',
     'history_007_pos', 'history_008_pos', 'history_009_pos',
     'history_010_pos']
+tags = ['functional', 'history']
 
 [tests/functional/hkdf]
 tests = ['run_hkdf_test']
+tags = ['functional', 'hkdf']
 
 [tests/functional/inheritance]
 tests = ['inherit_001_pos']
 pre =
+tags = ['functional', 'inheritance']
 
 [tests/functional/inuse]
 tests = ['inuse_001_pos', 'inuse_003_pos', 'inuse_004_pos',
     'inuse_005_pos', 'inuse_006_pos', 'inuse_007_pos', 'inuse_008_pos',
     'inuse_009_pos']
 post =
+tags = ['functional', 'inuse']
 
 [tests/functional/large_files]
 tests = ['large_files_001_pos', 'large_files_002_pos']
+tags = ['functional', 'large_files']
 
 [tests/functional/largest_pool]
 tests = ['largest_pool_001_pos']
 pre =
 post =
+tags = ['functional', 'largest_pool']
 
 [tests/functional/link_count]
 tests = ['link_count_001']
+tags = ['functional', 'link_count']
 
 [tests/functional/migration]
 tests = ['migration_001_pos', 'migration_002_pos', 'migration_003_pos',
     'migration_004_pos', 'migration_005_pos', 'migration_006_pos',
     'migration_007_pos', 'migration_008_pos', 'migration_009_pos',
     'migration_010_pos', 'migration_011_pos', 'migration_012_pos']
+tags = ['functional', 'migration']
 
 [tests/functional/mmap]
 tests = ['mmap_write_001_pos', 'mmap_read_001_pos']
+tags = ['functional', 'mmap']
 
 [tests/functional/mmp]
 tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval',
     'mmp_active_import', 'mmp_inactive_import', 'mmp_exported_import',
     'mmp_write_uberblocks', 'mmp_reset_interval']
+tags = ['functional', 'mmp']
 
 [tests/functional/mount]
 tests = ['umount_001', 'umountall_001']
+tags = ['functional', 'mount']
 
 [tests/functional/mv_files]
 tests = ['mv_files_001_pos', 'mv_files_002_pos']
+tags = ['functional', 'mv_files']
 
 [tests/functional/nestedfs]
 tests = ['nestedfs_001_pos']
+tags = ['functional', 'nestedfs']
 
 [tests/functional/no_space]
 tests = ['enospc_001_pos', 'enospc_002_pos', 'enospc_003_pos']
+tags = ['functional', 'no_space']
 
 [tests/functional/nopwrite]
 tests = ['nopwrite_copies', 'nopwrite_mtime', 'nopwrite_negative',
     'nopwrite_promoted_clone', 'nopwrite_recsize', 'nopwrite_sync',
     'nopwrite_varying_compression', 'nopwrite_volume']
+tags = ['functional', 'nopwrite']
 
 [tests/functional/online_offline]
 tests = ['online_offline_001_pos', 'online_offline_002_neg',
     'online_offline_003_neg']
+tags = ['functional', 'online_offline']
 
 [tests/functional/pool_names]
 tests = ['pool_names_001_pos', 'pool_names_002_neg']
 pre =
 post =
+tags = ['functional', 'pool_names']
 
 [tests/functional/poolversion]
 tests = ['poolversion_001_pos', 'poolversion_002_pos']
+tags = ['functional', 'poolversion']
 
 [tests/functional/privilege]
 tests = ['privilege_001_pos', 'privilege_002_pos']
+tags = ['functional', 'privilege']
 
 [tests/functional/quota]
 tests = ['quota_001_pos', 'quota_002_pos', 'quota_003_pos',
          'quota_004_pos', 'quota_005_pos', 'quota_006_neg']
+tags = ['functional', 'quota']
 
 [tests/functional/raidz]
 tests = ['raidz_001_neg', 'raidz_002_pos']
+tags = ['functional', 'raidz']
 
 [tests/functional/redundancy]
 tests = ['redundancy_001_pos', 'redundancy_002_pos', 'redundancy_003_pos',
     'redundancy_004_neg']
+tags = ['functional', 'redundancy']
 
 [tests/functional/refquota]
 tests = ['refquota_001_pos', 'refquota_002_pos', 'refquota_003_pos',
     'refquota_004_pos', 'refquota_005_pos', 'refquota_006_neg']
+tags = ['functional', 'refquota']
 
 # refreserv_004_pos - Fails for OpenZFS on illumos
 [tests/functional/refreserv]
 tests = ['refreserv_001_pos', 'refreserv_002_pos', 'refreserv_003_pos',
     'refreserv_005_pos']
+tags = ['functional', 'refreserv']
 
 [tests/functional/rename_dirs]
 tests = ['rename_dirs_001_pos']
+tags = ['functional', 'rename_dirs']
 
 [tests/functional/replacement]
 tests = ['replacement_001_pos', 'replacement_002_pos', 'replacement_003_pos']
+tags = ['functional', 'replacement']
 
 # reservation_001_pos - https://github.com/zfsonlinux/zfs/issues/4445
 # reservation_013_pos - https://github.com/zfsonlinux/zfs/issues/4444
@@ -522,9 +624,11 @@ tests = ['reservation_001_pos', 'reservation_002_pos', 'reservation_003_pos',
     'reservation_010_pos', 'reservation_011_pos', 'reservation_012_pos',
     'reservation_013_pos', 'reservation_014_pos', 'reservation_015_pos',
     'reservation_016_pos', 'reservation_017_pos']
+tags = ['functional', 'reservation']
 
 [tests/functional/rootpool]
 tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos']
+tags = ['functional', 'rootpool']
 
 # rsend_008_pos - https://github.com/zfsonlinux/zfs/issues/6066
 [tests/functional/rsend]
@@ -540,16 +644,19 @@ tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
     'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD',
     'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
     'send-c_recv_dedup', 'send_encrypted_heirarchy', 'send_freeobjects']
+tags = ['functional', 'rsend']
 
 [tests/functional/scrub_mirror]
 tests = ['scrub_mirror_001_pos', 'scrub_mirror_002_pos',
     'scrub_mirror_003_pos', 'scrub_mirror_004_pos']
+tags = ['functional', 'scrub_mirror']
 
 [tests/functional/slog]
 tests = ['slog_001_pos', 'slog_002_pos', 'slog_003_pos', 'slog_004_pos',
     'slog_005_pos', 'slog_006_pos', 'slog_007_pos', 'slog_008_neg',
     'slog_009_neg', 'slog_010_neg', 'slog_011_neg', 'slog_012_neg',
     'slog_013_pos', 'slog_014_pos', 'slog_replay_fs', 'slog_replay_volume']
+tags = ['functional', 'slog']
 
 [tests/functional/snapshot]
 tests = ['clone_001_pos', 'rollback_001_pos', 'rollback_002_pos',
@@ -559,26 +666,33 @@ tests = ['clone_001_pos', 'rollback_001_pos', 'rollback_002_pos',
     'snapshot_009_pos', 'snapshot_010_pos', 'snapshot_011_pos',
     'snapshot_012_pos', 'snapshot_013_pos', 'snapshot_014_pos',
     'snapshot_015_pos', 'snapshot_016_pos', 'snapshot_017_pos']
+tags = ['functional', 'snapshot']
 
 # snapused_004_pos - https://github.com/zfsonlinux/zfs/issues/5513
 [tests/functional/snapused]
 tests = ['snapused_001_pos', 'snapused_002_pos', 'snapused_003_pos',
     'snapused_005_pos']
+tags = ['functional', 'snapused']
 
 [tests/functional/sparse]
 tests = ['sparse_001_pos']
+tags = ['functional', 'sparse']
 
 [tests/functional/threadsappend]
 tests = ['threadsappend_001_pos']
+tags = ['functional', 'threadsappend']
 
 [tests/functional/tmpfile]
 tests = ['tmpfile_001_pos', 'tmpfile_002_pos', 'tmpfile_003_pos']
+tags = ['functional', 'tmpfile']
 
 [tests/functional/truncate]
 tests = ['truncate_001_pos', 'truncate_002_pos']
+tags = ['functional', 'truncate']
 
 [tests/functional/upgrade]
 tests = [ 'upgrade_userobj_001_pos' ]
+tags = ['functional', 'upgrade']
 
 [tests/functional/userquota]
 tests = [
@@ -589,35 +703,44 @@ tests = [
     'userquota_013_pos',
     'userspace_001_pos', 'userspace_002_pos', 'userspace_003_pos',
     'groupspace_001_pos', 'groupspace_002_pos', 'groupspace_003_pos' ]
+tags = ['functional', 'userquota']
 
 # vdev_zaps_007_pos -- fails due to a pre-existing issue with zpool split
 [tests/functional/vdev_zaps]
 tests = ['vdev_zaps_001_pos', 'vdev_zaps_002_pos', 'vdev_zaps_003_pos',
     'vdev_zaps_004_pos', 'vdev_zaps_005_pos', 'vdev_zaps_006_pos']
+tags = ['functional', 'vdev_zaps']
 
 [tests/functional/write_dirs]
 tests = ['write_dirs_001_pos', 'write_dirs_002_pos']
+tags = ['functional', 'write_dirs']
 
 [tests/functional/xattr]
 tests = ['xattr_001_pos', 'xattr_002_neg', 'xattr_003_neg', 'xattr_004_pos',
     'xattr_005_pos', 'xattr_006_pos', 'xattr_007_neg', 'xattr_008_pos',
     'xattr_009_neg', 'xattr_010_neg', 'xattr_011_pos', 'xattr_012_pos',
     'xattr_013_pos']
+tags = ['functional', 'xattr']
 
 [tests/functional/zvol/zvol_ENOSPC]
 tests = ['zvol_ENOSPC_001_pos']
+tags = ['functional', 'zvol', 'zvol_ENOSPC']
 
 [tests/functional/zvol/zvol_cli]
 tests = ['zvol_cli_001_pos', 'zvol_cli_002_pos', 'zvol_cli_003_neg']
+tags = ['functional', 'zvol', 'zvol_cli']
 
 [tests/functional/zvol/zvol_misc]
 tests = ['zvol_misc_001_neg', 'zvol_misc_002_pos', 'zvol_misc_003_neg',
     'zvol_misc_004_pos', 'zvol_misc_005_neg', 'zvol_misc_006_pos',
     'zvol_misc_snapdev', 'zvol_misc_volmode', 'zvol_misc_zil']
+tags = ['functional', 'zvol', 'zvol_misc']
 
 [tests/functional/zvol/zvol_swap]
 tests = ['zvol_swap_001_pos', 'zvol_swap_002_pos', 'zvol_swap_003_pos',
     'zvol_swap_004_pos', 'zvol_swap_005_pos', 'zvol_swap_006_pos']
+tags = ['functional', 'zvol', 'zvol_swap']
 
 [tests/functional/libzfs]
 tests = ['many_fds']
+tags = ['functional', 'libzfs']

tests/test-runner/bin/test-runner.py

@@ -119,7 +119,8 @@ class Output(object):
 class Cmd(object):
     verified_users = []
 
-    def __init__(self, pathname, outputdir=None, timeout=None, user=None):
+    def __init__(self, pathname, outputdir=None, timeout=None, user=None,
+                 tags=None):
         self.pathname = pathname
         self.outputdir = outputdir or 'BASEDIR'
         self.timeout = timeout
@@ -294,15 +295,17 @@ class Cmd(object):
 
 class Test(Cmd):
     props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
-             'post_user']
+             'post_user', 'tags']
 
     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
-                 pre=None, pre_user=None, post=None, post_user=None):
+                 pre=None, pre_user=None, post=None, post_user=None,
+                 tags=None):
         super(Test, self).__init__(pathname, outputdir, timeout, user)
         self.pre = pre or ''
         self.pre_user = pre_user or ''
         self.post = post or ''
         self.post_user = post_user or ''
+        self.tags = tags or []
 
     def __str__(self):
         post_user = pre_user = ''
@@ -311,9 +314,9 @@ class Test(Cmd):
         if len(self.post_user):
             post_user = ' (as %s)' % (self.post_user)
         return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \
-               "%s%s\nUser: %s\n" % \
+               "%s%s\nUser: %s\nTags: %s\n" % \
                (self.pathname, self.outputdir, self.timeout, self.pre,
-                pre_user, self.post, post_user, self.user)
+                pre_user, self.post, post_user, self.user, self.tags)
 
     def verify(self, logger):
         """
@@ -374,9 +377,9 @@ class TestGroup(Test):
 
     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
                  pre=None, pre_user=None, post=None, post_user=None,
-                 tests=None):
+                 tests=None, tags=None):
         super(TestGroup, self).__init__(pathname, outputdir, timeout, user,
-                                        pre, pre_user, post, post_user)
+                                        pre, pre_user, post, post_user, tags)
         self.tests = tests or []
 
     def __str__(self):
@@ -385,10 +388,10 @@ class TestGroup(Test):
             pre_user = ' (as %s)' % (self.pre_user)
         if len(self.post_user):
             post_user = ' (as %s)' % (self.post_user)
-        return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %d\n" \
+        return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %s\n" \
-               "Pre: %s%s\nPost: %s%s\nUser: %s\n" % \
+               "Pre: %s%s\nPost: %s%s\nUser: %s\nTags: %s\n" % \
                (self.pathname, self.outputdir, self.tests, self.timeout,
-                self.pre, pre_user, self.post, post_user, self.user)
+                self.pre, pre_user, self.post, post_user, self.user, self.tags)
 
     def verify(self, logger):
         """
@@ -441,6 +444,10 @@ class TestGroup(Test):
         doesn't pass, skip all the tests in this TestGroup. Run the post
         script regardless.
         """
+        # tags assigned to this test group also include the test names
+        if options.tags and not set(self.tags).intersection(set(options.tags)):
+            return
+
         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
         pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
                       user=self.pre_user)
@@ -488,7 +495,8 @@ class TestRun(object):
             ('pre', ''),
             ('pre_user', ''),
             ('post', ''),
-            ('post_user', '')
+            ('post_user', ''),
+            ('tags', [])
         ]
 
     def __str__(self):
@@ -566,7 +574,12 @@ class TestRun(object):
                 for prop in TestGroup.props:
                     for sect in ['DEFAULT', section]:
                         if config.has_option(sect, prop):
-                            setattr(testgroup, prop, config.get(sect, prop))
+                            if prop is "tags":
+                                setattr(testgroup, prop,
+                                        eval(config.get(sect, prop)))
+                            else:
+                                setattr(testgroup, prop,
+                                        config.get(sect, prop))
 
                 # Repopulate tests using eval to convert the string to a list
                 testgroup.tests = eval(config.get(section, 'tests'))
@@ -696,10 +709,13 @@ class TestRun(object):
         else:
             print 'Could not make a symlink to directory %s' % (
                 self.outputdir)
-        for test in sorted(self.tests.keys()):
+        iteration = 0
-            self.tests[test].run(self.logger, options)
+        while iteration < options.iterations:
-        for testgroup in sorted(self.testgroups.keys()):
+            for test in sorted(self.tests.keys()):
-            self.testgroups[testgroup].run(self.logger, options)
+                self.tests[test].run(self.logger, options)
+            for testgroup in sorted(self.testgroups.keys()):
+                self.testgroups[testgroup].run(self.logger, options)
+            iteration += 1
 
     def summary(self):
         if Result.total is 0:
@@ -806,6 +822,8 @@ def options_cb(option, opt_str, value, parser):
         parser.values.cmd = 'rdconfig'
     if option.dest is 'template':
         parser.values.cmd = 'wrconfig'
+    if option.dest is 'tags':
+        value = [x.strip() for x in value.split(',')]
 
     setattr(parser.values, option.dest, value)
     if option.dest in path_options:
@@ -850,6 +868,12 @@ def parse_args():
     parser.add_option('-X', action='callback', callback=options_cb, default='',
                       dest='post_user', metavar='post_user', type='string',
                       help='Specify a user to execute the post script.')
+    parser.add_option('-T', action='callback', callback=options_cb, default='',
+                      dest='tags', metavar='tags', type='string',
+                      help='Specify tags to execute specific test groups.')
+    parser.add_option('-I', action='callback', callback=options_cb, default=1,
+                      dest='iterations', metavar='iterations', type='int',
+                      help='Number of times to run the test run.')
     (options, pathnames) = parser.parse_args()
 
     if not options.runfile and not options.template: