ZTS: Use QEMU for tests on Linux and FreeBSD

This commit adds functional tests for these systems: - AlmaLinux 8, AlmaLinux 9, ArchLinux - CentOS Stream 9, Fedora 39, Fedora 40 - Debian 11, Debian 12 - FreeBSD 13, FreeBSD 14, FreeBSD 15 - Ubuntu 20.04, Ubuntu 22.04, Ubuntu 24.04 - enabled by default: - AlmaLinux 8, AlmaLinux 9 - Fedora 39, Fedora 40 - FreeBSD 13, FreeBSD 14, FreeBSD 15 Workflow for each operating system: - install qemu on the github runner - download current cloud image of operating system - start and init that image via cloud-init - install dependencies and poweroff system - start system and build openzfs and then poweroff again - clone build system and start 3 instances of it - the functional testings complete within times < 3h Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de> Signed-off-by: Tony Hutter <hutter2@llnl.gov>
2024-06-17 16:52:58 +02:00 · 2024-06-17 16:52:58 +02:00 · e50b34f940
parent bf132dbdb3
commit e50b34f940
13 changed files with 1352 additions and 1 deletions
--- a/.github/workflows/scripts/README.md
+++ b/.github/workflows/scripts/README.md
@ -0,0 +1,9 @@
+
+Workflow for each operating system:
+- install qemu on the github runner
+- download current cloud image of operating system
+- start and init that image via cloud-init
+- install dependencies and poweroff system
+- start system and build openzfs and then poweroff again
+- clone build system and start 3 instances of it
+- the functional testings complete within times < 3h
--- a/.github/workflows/scripts/merge_summary.awk
+++ b/.github/workflows/scripts/merge_summary.awk
@ -0,0 +1,109 @@
+#!/bin/awk -f
+#
+# Merge multiple ZTS tests results summaries into a single summary.  This is
+# needed when you're running different parts of ZTS on different tests
+# runners or VMs.
+#
+# Usage:
+#
+#	./merge_summary.awk summary1.txt [summary2.txt] [summary3.txt] ...
+#
+#	or:
+#
+#	cat summary*.txt | ./merge_summary.awk
+#
+BEGIN {
+	i=-1
+	pass=0
+	fail=0
+	skip=0
+	state=""
+	cl=0
+	el=0
+	upl=0
+	ul=0
+
+	# Total seconds of tests runtime
+	total=0;
+}
+
+# Skip empty lines
+/^\s*$/{next}
+
+# Skip Configuration and Test lines
+/^Test:/{state=""; next}
+/Configuration/{state="";next}
+
+# When we see "test-runner.py" stop saving config lines, and
+# save test runner lines
+/test-runner.py/{state="testrunner"; runner=runner$0"\n"; next}
+
+# We need to differentiate the PASS counts from test result lines that start
+# with PASS, like:
+#
+#   PASS mv_files/setup
+#
+# Use state="pass_count" to differentiate
+#
+/Results Summary/{state="pass_count"; next}
+/PASS/{ if (state=="pass_count") {pass += $2}}
+/FAIL/{ if (state=="pass_count") {fail += $2}}
+/SKIP/{ if (state=="pass_count") {skip += $2}}
+/Running Time/{
+	state="";
+	running[i]=$3;
+	split($3, arr, ":")
+	total += arr[1] * 60 * 60;
+	total += arr[2] * 60;
+	total += arr[3]
+	next;
+}
+
+/Tests with results other than PASS that are expected/{state="expected_lines"; next}
+/Tests with result of PASS that are unexpected/{state="unexpected_pass_lines"; next}
+/Tests with results other than PASS that are unexpected/{state="unexpected_lines"; next}
+{
+	if (state == "expected_lines") {
+		expected_lines[el] = $0
+		el++
+	}
+
+	if (state == "unexpected_pass_lines") {
+		unexpected_pass_lines[upl] = $0
+		upl++
+	}
+	if (state == "unexpected_lines") {
+		unexpected_lines[ul] = $0
+		ul++
+	}
+}
+
+# Reproduce summary
+END {
+	print runner;
+	print "\nResults Summary"
+	print "PASS\t"pass
+	print "FAIL\t"fail
+	print "SKIP\t"skip
+	print ""
+	print "Running Time:\t"strftime("%T", total, 1)
+	if (pass+fail+skip > 0) {
+		percent_passed=(pass/(pass+fail+skip) * 100)
+	}
+	printf "Percent passed:\t%3.2f%", percent_passed
+
+	print "\n\nTests with results other than PASS that are expected:"
+	asort(expected_lines, sorted)
+	for (j in sorted)
+		print sorted[j]
+
+	print "\n\nTests with result of PASS that are unexpected:"
+	asort(unexpected_pass_lines, sorted)
+	for (j in sorted)
+		print sorted[j]
+
+	print "\n\nTests with results other than PASS that are unexpected:"
+	asort(unexpected_lines, sorted)
+	for (j in sorted)
+		print sorted[j]
+}
--- a/.github/workflows/scripts/qemu-1-setup.sh
+++ b/.github/workflows/scripts/qemu-1-setup.sh
@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 1) setup qemu instance on action runner
+######################################################################
+
+set -eu
+
+# install needed packages
+sudo apt-get update
+sudo apt-get install axel cloud-image-utils daemonize guestfs-tools \
+  ksmtuned virt-manager linux-modules-extra-`uname -r`
+
+# generate ssh keys
+rm -f ~/.ssh/id_ed25519
+ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""
+
+# no need for some scheduler
+for i in /sys/block/s*/queue/scheduler; do
+  echo "none" | sudo tee $i > /dev/null
+done
+
+# this one is fast and mostly free
+sudo mount -o remount,rw,noatime,barrier=0 /mnt
+
+# we expect RAM shortage
+cat << EOF | sudo tee /etc/ksmtuned.conf > /dev/null
+KSM_MONITOR_INTERVAL=60
+
+# Millisecond sleep between ksm scans for 16Gb server.
+# Smaller servers sleep more, bigger sleep less.
+KSM_SLEEP_MSEC=10
+KSM_NPAGES_BOOST=300
+KSM_NPAGES_DECAY=-50
+KSM_NPAGES_MIN=64
+KSM_NPAGES_MAX=2048
+
+KSM_THRES_COEF=20
+KSM_THRES_CONST=2048
+
+LOGFILE=/var/log/ksmtuned.log
+DEBUG=1
+EOF
+
+sudo systemctl restart ksm
+sudo systemctl restart ksmtuned
--- a/.github/workflows/scripts/qemu-2-start.sh
+++ b/.github/workflows/scripts/qemu-2-start.sh
@ -0,0 +1,200 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 2) start qemu with some operating system, init via cloud-init
+######################################################################
+
+set -eu
+
+# short name used in zfs-qemu.yml
+OS="$1"
+
+# OS variant (virt-install --os-variant list)
+OSv=$OS
+
+# compressed with .zst extension
+FREEBSD="https://github.com/mcmilk/openzfs-freebsd-images/releases/download/v2024-08-10"
+URLzs=""
+
+# Ubuntu mirrors
+#UBMIRROR="https://cloud-images.ubuntu.com"
+#UBMIRROR="https://mirrors.cloud.tencent.com/ubuntu-cloud-images"
+UBMIRROR="https://mirror.citrahost.com/ubuntu-cloud-images"
+
+case "$OS" in
+  almalinux8)
+    OSNAME="AlmaLinux 8"
+    URL="https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
+    ;;
+  almalinux9)
+    OSNAME="AlmaLinux 9"
+    URL="https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2"
+    ;;
+  archlinux)
+    OSNAME="Archlinux"
+    URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2"
+    ;;
+  centos-stream9)
+    OSNAME="CentOS Stream 9"
+    URL="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2"
+    ;;
+  debian11)
+    OSNAME="Debian 11"
+    URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-generic-amd64.qcow2"
+    ;;
+  debian12)
+    OSNAME="Debian 12"
+    URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
+    ;;
+  fedora39)
+    OSNAME="Fedora 39"
+    OSv="fedora39"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-39-1.5.x86_64.qcow2"
+    ;;
+  fedora40)
+    OSNAME="Fedora 40"
+    OSv="fedora39"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2"
+    ;;
+  freebsd13r)
+    OSNAME="FreeBSD 13.3-RELEASE"
+    OSv="freebsd13.0"
+    URLzs="$FREEBSD/amd64-freebsd-13.3-RELEASE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd13)
+    OSNAME="FreeBSD 13.4-STABLE"
+    OSv="freebsd13.0"
+    URLzs="$FREEBSD/amd64-freebsd-13.4-STABLE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd14r)
+    OSNAME="FreeBSD 14.1-RELEASE"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-14.1-RELEASE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd14)
+    OSNAME="FreeBSD 14.1-STABLE"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-14.1-STABLE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd15)
+    OSNAME="FreeBSD 15.0-CURRENT"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-15.0-CURRENT.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  tumbleweed)
+    OSNAME="openSUSE Tumbleweed"
+    OSv="opensusetumbleweed"
+    MIRROR="http://opensuse-mirror-gce-us.susecloud.net"
+    URL="$MIRROR/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2"
+    ;;
+  ubuntu20)
+    OSNAME="Ubuntu 20.04"
+    OSv="ubuntu20.04"
+    URL="$UBMIRROR/focal/current/focal-server-cloudimg-amd64.img"
+    ;;
+  ubuntu22)
+    OSNAME="Ubuntu 22.04"
+    OSv="ubuntu22.04"
+    URL="$UBMIRROR/jammy/current/jammy-server-cloudimg-amd64.img"
+    ;;
+  ubuntu24)
+    OSNAME="Ubuntu 24.04"
+    OSv="ubuntu24.04"
+    URL="$UBMIRROR/noble/current/noble-server-cloudimg-amd64.img"
+    ;;
+  *)
+    echo "Wrong value for OS variable!"
+    exit 111
+    ;;
+esac
+
+# freebsd15 -> used in zfs-qemu.yml
+echo "$OS" > /var/tmp/os.txt
+
+# freebsd14.0 -> used for virt-install
+echo "$OSv" > /var/tmp/osvariant.txt
+
+# FreeBSD 15 (Current) -> used for summary
+echo "$OSNAME" > /var/tmp/osname.txt
+
+IMG="/mnt/tests/cloudimg.qcow2"
+DISK="/mnt/tests/openzfs.qcow2"
+sudo mkdir -p "/mnt/tests"
+sudo chown $(whoami) /mnt/tests
+
+# we are downloading via axel, curl and wget are mostly slower and
+# require more return value checking
+if [ ! -z "$URLzs" ]; then
+  echo "Loading image $URLzs ..."
+  time axel -q -o "$IMG.zst" "$URLzs"
+  zstd -q -d --rm "$IMG.zst"
+else
+  echo "Loading image $URL ..."
+  time axel -q -o "$IMG" "$URL"
+fi
+
+# 256k cluster seems the best in terms of speed for now
+qemu-img convert -q -f qcow2 -O qcow2 -c \
+  -o compression_type=zstd,cluster_size=256k $IMG $DISK
+rm -f $IMG
+
+echo "Resizing image to 60GiB ..."
+qemu-img resize -q $DISK 60G
+
+PUBKEY=`cat ~/.ssh/id_ed25519.pub`
+cat <<EOF > /tmp/user-data
+#cloud-config
+
+fqdn: $OS
+
+# user:zfs password:1
+users:
+- name: root
+  shell: $BASH
+- name: zfs
+  sudo: ALL=(ALL) NOPASSWD:ALL
+  shell: $BASH
+  lock-passwd: false
+  passwd: \$1\$EjKAQetN\$O7Tw/rZOHaeBP1AiCliUg/
+  ssh_authorized_keys:
+    - $PUBKEY
+
+growpart:
+  mode: auto
+  devices: ['/']
+  ignore_growroot_disabled: false
+EOF
+
+sudo virsh net-update default add ip-dhcp-host \
+  "<host mac='52:54:00:83:79:00' ip='192.168.122.10'/>" --live --config
+
+# 12GiB RAM for building the module, TY Github :)
+sudo virt-install \
+  --os-variant $OSv \
+  --name "openzfs" \
+  --cpu host-passthrough \
+  --virt-type=kvm --hvm \
+  --vcpus=4,sockets=1 \
+  --memory $((1024*12)) \
+  --memballoon model=virtio \
+  --graphics none \
+  --network bridge=virbr0,model=e1000,mac='52:54:00:83:79:00' \
+  --cloud-init user-data=/tmp/user-data \
+  --disk $DISK,bus=virtio,cache=none,format=qcow2,driver.discard=unmap \
+  --import --noautoconsole >/dev/null
+
+# in case the directory isn't there already
+mkdir -p $HOME/.ssh
+
+cat <<EOF >> $HOME/.ssh/config
+# no questions please
+StrictHostKeyChecking no
+
+# small timeout, used in while loops later
+ConnectTimeout 1
+EOF
--- a/.github/workflows/scripts/qemu-3-deps.sh
+++ b/.github/workflows/scripts/qemu-3-deps.sh
@ -0,0 +1,199 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 3) install dependencies for compiling and loading
+######################################################################
+
+set -eu
+
+function archlinux() {
+  echo "##[group]Running pacman -Syu"
+  sudo pacman -Syu --noconfirm
+  echo "##[endgroup]"
+
+  echo "##[group]Install Development Tools"
+  sudo pacman -Sy --noconfirm base-devel bc cpio dhclient dkms fakeroot \
+    fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils parted \
+    pax perf python-packaging python-setuptools qemu-guest-agent ksh samba \
+    sysstat rng-tools rsync wget
+  echo "##[endgroup]"
+}
+
+function debian() {
+  export DEBIAN_FRONTEND="noninteractive"
+
+  echo "##[group]Running apt-get update+upgrade"
+  sudo apt-get update -y
+  sudo apt-get upgrade -y
+  echo "##[endgroup]"
+
+  echo "##[group]Install Development Tools"
+  sudo apt-get install -y \
+    acl alien attr autoconf bc cpio curl dbench dh-python \
+    dkms fakeroot fio gdb gdebi git ksh lcov isc-dhcp-client jq \
+    libacl1-dev libaio-dev libattr1-dev libblkid-dev \
+    libcurl4-openssl-dev libdevmapper-dev libelf-dev libffi-dev \
+    libmount-dev libpam0g-dev libselinux-dev libssl-dev libtool \
+    libtool-bin libudev-dev linux-headers-$(uname -r) lsscsi \
+    nfs-kernel-server pamtester parted python3 python3-all-dev \
+    python3-cffi python3-dev python3-distlib python3-packaging \
+    python3-setuptools python3-sphinx qemu-guest-agent rng-tools \
+    rpm2cpio rsync samba sysstat uuid-dev watchdog wget xfslibs-dev \
+    zlib1g-dev
+  echo "##[endgroup]"
+}
+
+function freebsd() {
+  export ASSUME_ALWAYS_YES="YES"
+
+  echo "##[group]Install Development Tools"
+  sudo pkg install -y autoconf automake autotools base64 checkbashisms fio \
+    gdb gettext gettext-runtime git gmake gsed jq ksh93 lcov libtool lscpu \
+    pkgconf python python3 pamtester pamtester qemu-guest-agent rsync
+  sudo pkg install -xy \
+    '^samba4[[:digit:]]+$' \
+    '^py3[[:digit:]]+-cffi$' \
+    '^py3[[:digit:]]+-sysctl$' \
+    '^py3[[:digit:]]+-packaging$'
+  echo "##[endgroup]"
+}
+
+# common packages for: almalinux, centos, redhat
+function rhel() {
+  echo "##[group]Running dnf update"
+  echo "max_parallel_downloads=10" | sudo -E tee -a /etc/dnf/dnf.conf
+  sudo dnf clean all
+  sudo dnf update -y --setopt=fastestmirror=1 --refresh
+  echo "##[endgroup]"
+
+  echo "##[group]Install Development Tools"
+  sudo dnf group install -y "Development Tools"
+  sudo dnf install -y \
+    acl attr bc bzip2 curl dbench dkms elfutils-libelf-devel fio gdb git \
+    jq kernel-rpm-macros ksh libacl-devel libaio-devel libargon2-devel \
+    libattr-devel libblkid-devel libcurl-devel libffi-devel ncompress \
+    libselinux-devel libtirpc-devel libtool libudev-devel libuuid-devel \
+    lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester parted perf \
+    python3 python3-cffi python3-devel python3-packaging kernel-devel \
+    python3-setuptools qemu-guest-agent rng-tools rpcgen rpm-build rsync \
+    samba sysstat systemd watchdog wget xfsprogs-devel zlib-devel
+  echo "##[endgroup]"
+}
+
+function tumbleweed() {
+  echo "##[group]Running zypper is TODO!"
+  sleep 23456
+  echo "##[endgroup]"
+}
+
+# Install dependencies
+case "$1" in
+  almalinux8)
+    echo "##[group]Enable epel and powertools repositories"
+    sudo dnf config-manager -y --set-enabled powertools
+    sudo dnf install -y epel-release
+    echo "##[endgroup]"
+    rhel
+    echo "##[group]Install kernel-abi-whitelists"
+    sudo dnf install -y kernel-abi-whitelists
+    echo "##[endgroup]"
+    ;;
+  almalinux9|centos-stream9)
+    echo "##[group]Enable epel and crb repositories"
+    sudo dnf config-manager -y --set-enabled crb
+    sudo dnf install -y epel-release
+    echo "##[endgroup]"
+    rhel
+    echo "##[group]Install kernel-abi-stablelists"
+    sudo dnf install -y kernel-abi-stablelists
+    echo "##[endgroup]"
+    ;;
+  archlinux)
+    archlinux
+    ;;
+  debian*)
+    debian
+    echo "##[group]Install Debian specific"
+    sudo apt-get install -yq linux-perf dh-sequence-dkms
+    echo "##[endgroup]"
+    ;;
+  fedora*)
+    rhel
+    ;;
+  freebsd*)
+    freebsd
+    ;;
+  tumbleweed)
+    tumbleweed
+    ;;
+  ubuntu*)
+    debian
+    echo "##[group]Install Ubuntu specific"
+    sudo apt-get install -yq linux-tools-common libtirpc-dev \
+      linux-modules-extra-$(uname -r)
+    if [ "$1" != "ubuntu20" ]; then
+      sudo apt-get install -yq dh-sequence-dkms
+    fi
+    echo "##[endgroup]"
+    echo "##[group]Delete Ubuntu OpenZFS modules"
+    for i in `find /lib/modules -name zfs -type d`; do sudo rm -rvf $i; done
+    echo "##[endgroup]"
+    ;;
+esac
+
+# Start services
+echo "##[group]Enable services"
+case "$1" in
+  freebsd*)
+    # add virtio things
+    echo 'virtio_load="YES"' | sudo -E tee -a /boot/loader.conf
+    for i in balloon blk console random scsi; do
+      echo "virtio_${i}_load=\"YES\"" | sudo -E tee -a /boot/loader.conf
+    done
+    echo "fdescfs /dev/fd fdescfs rw 0 0" | sudo -E tee -a /etc/fstab
+    sudo -E mount /dev/fd
+    sudo -E touch /etc/zfs/exports
+    sudo -E sysrc mountd_flags="/etc/zfs/exports"
+    echo '[global]' | sudo -E tee /usr/local/etc/smb4.conf >/dev/null
+    sudo -E service nfsd enable
+    sudo -E service qemu-guest-agent enable
+    sudo -E service samba_server enable
+    ;;
+  debian*|ubuntu*)
+    sudo -E systemctl enable nfs-kernel-server
+    sudo -E systemctl enable qemu-guest-agent
+    sudo -E systemctl enable smbd
+    ;;
+  *)
+    # All other linux distros
+    sudo -E systemctl enable nfs-server
+    sudo -E systemctl enable qemu-guest-agent
+    sudo -E systemctl enable smb
+    ;;
+esac
+echo "##[endgroup]"
+
+# Enable serial console and remove 'quiet' from linux kernel cmdline
+case "$1" in
+  freebsd*)
+    # console is turned on @ FreeBSD images from here:
+    # https://github.com/mcmilk/zfs/tree/master
+    true
+    ;;
+  *)
+    # should fit for most distros, may be optimized a bit later
+    echo "##[group]Enable serial output"
+    sudo sed -i 's/GRUB_CMDLINE_LINUX_DEFAULT="/GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0,115200n8 random.trust_cpu=on/g; s/quiet //g' /etc/default/grub || true
+    for i in /boot/grub/grub.cfg /etc/grub2.cfg /etc/grub2-efi.cfg /boot/grub2/grub.cfg ; do
+      test -e $i || continue
+      echo sudo grub-mkconfig -o $i
+      sudo grub-mkconfig -o $i
+    done
+    echo "##[endgroup]"
+    ;;
+esac
+
+# reset cloud-init configuration and poweroff
+sudo cloud-init clean --logs
+sleep 2 && sudo poweroff &
+exit 0
--- a/.github/workflows/scripts/qemu-4-build.sh
+++ b/.github/workflows/scripts/qemu-4-build.sh
@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 4) configure and build openzfs modules
+######################################################################
+
+set -eu
+
+function run() {
+  LOG="/var/tmp/build-stderr.txt"
+  echo "**************************************************"
+  echo "`date` ($*)"
+  echo "**************************************************"
+  (stdbuf -eL -oL $@ || echo $? > /tmp/rv) 3>&1 1>&2 2>&3 | tee -a $LOG
+  if [ -f /tmp/rv ]; then
+    RV=`cat /tmp/rv`
+    echo "**************************************************"
+    echo "exit with value=$RV ($*)"
+    echo "**************************************************"
+    exit $RV
+  fi
+}
+
+function freebsd() {
+  export MAKE="gmake"
+  echo "##[group]Autogen.sh"
+  run ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  run ./configure \
+    --prefix=/usr/local \
+    --with-libintl-prefix=/usr/local \
+    --enable-pyzfs \
+    --enable-debug \
+    --enable-debuginfo
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  run gmake -j`sysctl -n hw.ncpu`
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  run sudo gmake install
+  echo "##[endgroup]"
+}
+
+function linux() {
+  echo "##[group]Autogen.sh"
+  run ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  run ./configure \
+    --prefix=/usr \
+    --enable-pyzfs \
+    --enable-debug \
+    --enable-debuginfo
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  run make -j$(nproc)
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  run sudo make install
+  echo "##[endgroup]"
+}
+
+function rpm_build_and_install() {
+  EXTRA_CONFIG="${1:-}"
+  echo "##[group]Autogen.sh"
+  run ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  run ./configure --enable-debug --enable-debuginfo $EXTRA_CONFIG
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  run make pkg-kmod pkg-utils
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  run sudo yum -y --skip-broken localinstall $(ls *.rpm | grep -v src.rpm)
+  echo "##[endgroup]"
+
+}
+
+function deb_build_and_install() {
+echo "##[group]Autogen.sh"
+  run ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  run ./configure \
+    --prefix=/usr \
+    --enable-pyzfs \
+    --enable-debug \
+    --enable-debuginfo
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  run make native-deb-kmod native-deb-utils
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  # Do kmod install.  Note that when you build the native debs, the
+  # packages themselves are placed in parent directory '../' rather than
+  # in the source directory like the rpms are.
+  run sudo apt-get -y install `find ../ | grep -E '\.deb$' | grep -Ev 'dkms|dracut'`
+  echo "##[endgroup]"
+}
+
+# Debug: show kernel cmdline
+if [ -e /proc/cmdline ] ; then
+  cat /proc/cmdline || true
+fi
+
+cd $HOME/zfs
+export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin"
+
+# build
+case "$1" in
+  freebsd*)
+    freebsd
+    ;;
+  alma*|centos*)
+    rpm_build_and_install "--with-spec=redhat"
+    ;;
+  fedora*)
+    rpm_build_and_install
+    ;;
+  debian*|ubuntu*)
+    deb_build_and_install
+    ;;
+  *)
+    linux
+    ;;
+esac
+
+# save some sysinfo
+uname -a > /var/tmp/uname.txt
+
+# reset cloud-init configuration and poweroff
+sudo cloud-init clean --logs
+sleep 2 && sudo poweroff &
+exit 0
--- a/.github/workflows/scripts/qemu-5-setup.sh
+++ b/.github/workflows/scripts/qemu-5-setup.sh
@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 5) start test machines and load openzfs module
+######################################################################
+
+set -eu
+
+# wait for poweroff to succeed
+PID=`pidof /usr/bin/qemu-system-x86_64`
+tail --pid=$PID -f /dev/null
+sudo virsh undefine openzfs
+
+PUBKEY=`cat ~/.ssh/id_ed25519.pub`
+OSv=`cat /var/tmp/osvariant.txt`
+OS=`cat /var/tmp/os.txt`
+
+# definition of ressources per operating system
+case "$OS" in
+  freebsd*)
+    # 2x CPU=4 RAM=6 -> FreeBSD 13 (2h 10m)
+    # 2x CPU=4 RAM=6 -> FreeBSD 14 (2h 10m)
+    VMs=2
+    CPU=4
+    RAM=6
+    ;;
+  *)
+    # 2x CPU=4 RAM=7 -> Almalinux 8 (3h 12m)
+    # 2x CPU=4 RAM=7 -> CentOS 9 (3h 2m)
+    # 2x CPU=4 RAM=7 -> Debian 11 (3h 11m)
+    # 2x CPU=4 RAM=7 -> Ubuntu 20 (2h 49m)
+    # 2x CPU=4 RAM=7 -> Ubuntu 22 (3h 26m)
+    # 2x CPU=4 RAM=7 -> Ubuntu 24 (3h 10m)
+    # 2x CPU=4 RAM=7 -> Fedora40 (3h 33m)
+    VMs=2
+    CPU=4
+    RAM=7
+    ;;
+esac
+
+# this can be different for each distro
+echo $VMs > /var/tmp/vms.txt
+
+# setup the testing vm's
+for i in `seq 1 $VMs`; do
+  echo "Generating disk for vm$i..."
+  sudo qemu-img create -q -f qcow2 -F qcow2 \
+    -o compression_type=zstd,cluster_size=256k \
+    -b /mnt/tests/openzfs.qcow2 "/mnt/tests/vm$i.qcow2"
+
+  cat <<EOF > /tmp/user-data
+#cloud-config
+
+fqdn: vm$i
+
+# user:zfs password:1
+users:
+- name: root
+  shell: $BASH
+- name: zfs
+  sudo: ALL=(ALL) NOPASSWD:ALL
+  shell: $BASH
+  lock-passwd: false
+  passwd: \$1\$EjKAQetN\$O7Tw/rZOHaeBP1AiCliUg/
+  ssh_authorized_keys:
+    - $PUBKEY
+
+growpart:
+  mode: auto
+  devices: ['/']
+  ignore_growroot_disabled: false
+EOF
+
+  sudo virsh net-update default add ip-dhcp-host \
+    "<host mac='52:54:00:83:79:0$i' ip='192.168.122.1$i'/>" --live --config
+
+  sudo virt-install \
+    --os-variant $OSv \
+    --name "vm$i" \
+    --cpu host-passthrough \
+    --virt-type=kvm --hvm \
+    --vcpus=$CPU,sockets=1 \
+    --memory $((1024*RAM)) \
+    --memballoon model=virtio \
+    --graphics none \
+    --cloud-init user-data=/tmp/user-data \
+    --network bridge=virbr0,model=e1000,mac="52:54:00:83:79:0$i" \
+    --disk /mnt/tests/vm$i.qcow2,bus=virtio,cache=none,format=qcow2,driver.discard=unmap \
+    --import --noautoconsole >/dev/null
+done
+
+# setup cron job on the host side
+case "$OS" in
+  freebsd*)
+    true
+    ;;
+  *)
+    # Linux based systems, trim the qcow2 files
+    echo "exec 1>/dev/null 2>/dev/null" > cronjob.sh
+    for i in `seq 1 $VMs`; do
+      echo "virsh domfstrim vm$i" >> cronjob.sh
+    done
+    echo "fstrim /mnt" >> cronjob.sh
+    sudo chmod +x cronjob.sh
+    sudo mv -f cronjob.sh /root/cronjob.sh
+    echo '*/20 * * * *  /root/cronjob.sh' > crontab.txt
+    sudo crontab crontab.txt
+    rm crontab.txt
+    ;;
+esac
+
+# check if the machines are okay
+echo "Waiting for vm's to come up...  (${VMs}x CPU=$CPU RAM=$RAM)"
+for i in `seq 1 $VMs`; do
+  while true; do
+    ssh 2>/dev/null zfs@192.168.122.1$i "uname -a" && break
+  done
+done
+echo "All $VMs VMs are up now."
+
+# Save the VM's serial output (ttyS0) to /var/tmp/console.txt
+# - ttyS0 on the VM corresponds to a local /dev/pty/N entry
+# - use 'virsh ttyconsole' to lookup the /dev/pty/N entry
+RESPATH="/var/tmp/test_results"
+for i in `seq 1 $VMs`; do
+  mkdir -p $RESPATH/vm$i
+  read "pty" <<< $(sudo virsh ttyconsole vm$i)
+  sudo nohup bash -c "cat $pty > $RESPATH/vm$i/console.txt" &
+done
+echo "Console logging for ${VMs}x $OS started."
--- a/.github/workflows/scripts/qemu-6-tests.sh
+++ b/.github/workflows/scripts/qemu-6-tests.sh
@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 6) load openzfs module and run the tests
+#
+# called on runner:  qemu-6-tests.sh (without extra args)
+# called on qemu-vm: qemu-6-tests.sh $OS $2/$3
+######################################################################
+
+set -o pipefail
+
+# called directly on the runner
+if [ -f /var/tmp/vms.txt ]; then
+  cd "/var/tmp"
+
+  OS=`cat os.txt`
+  VMs=`cat vms.txt`
+  SSH=`which ssh`
+
+  BASE="$HOME/work/zfs/zfs"
+  TESTS='$HOME/zfs/.github/workflows/scripts/qemu-6-tests.sh'
+  COLOR="$BASE/scripts/zfs-tests-color.sh"
+
+  # df statistics - keep an eye on disk usage
+  echo "Disk usage before:" > disk-usage.txt
+  df -h /mnt/tests >> disk-usage.txt
+
+  for i in `seq 1 $VMs`; do
+    IP="192.168.122.1$i"
+    daemonize -c /var/tmp -p vm${i}.pid -o vm${i}log.txt -- \
+      $SSH zfs@$IP $TESTS $OS $i $VMs
+    # give us the output of stdout + stderr - with prefix ;)
+    tail -fq vm${i}log.txt | $COLOR | sed -e "s/^/vm${i}: /g" &
+    echo $! > vm${i}log.pid
+  done
+
+
+  # wait for all vm's to finish
+  for i in `seq 1 $VMs`; do
+    tail --pid=`cat vm${i}.pid` -f /dev/null
+    pid=`cat vm${i}log.pid`
+    rm -f vm${i}log.pid
+    kill $pid
+  done
+
+  # df statistics part 2
+  echo "Disk usage afterwards:" >> disk-usage.txt
+  df -h /mnt/tests >> disk-usage.txt
+  echo "VM files take this space:" >> disk-usage.txt
+  du -sh /mnt/tests >> disk-usage.txt
+
+  exit 0
+fi
+
+# this part runs inside qemu vm
+export PATH="$PATH:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/sbin:/usr/local/bin"
+case "$1" in
+  freebsd*)
+    # when freebsd's zfs is loaded, unload this one
+    sudo kldstat -n zfs 2>/dev/null && sudo kldunload zfs
+    sudo -E ./zfs/scripts/zfs.sh
+    sudo dmesg -c > /var/tmp/dmesg-prerun.txt
+    TDIR="/usr/local/share/zfs"
+    ;;
+  *)
+    sudo -E modprobe zfs
+    sudo dmesg -c > /var/tmp/dmesg-prerun.txt
+    TDIR="/usr/share/zfs"
+    ;;
+esac
+
+# run functional testings
+TAGS=$2/$3
+#TAGS=casenorm,zpool_trim,trim,raidz
+
+$TDIR/zfs-tests.sh -vK -s 3G -T $TAGS
+RV=$?
+
+# we wont fail here, this will be done later
+echo $RV > /var/tmp/exitcode.txt
+exit 0
--- a/.github/workflows/scripts/qemu-7-prepare.sh
+++ b/.github/workflows/scripts/qemu-7-prepare.sh
@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 7) prepare output of the results
+######################################################################
+
+set -o pipefail
+
+cd /var/tmp
+OS=`cat os.txt`
+
+# check if building the module has failed
+RESPATH="/var/tmp/test_results"
+if [ ! -s vms.txt ]; then
+  mkdir -p $RESPATH
+  cd $RESPATH
+  # build some simple summary:
+  echo "!!! ZFS module didn't build successfully !!!" \
+    | tee summary.txt | tee clean-summary.txt
+  scp zfs@192.168.122.10:"/var/tmp/*.txt" $RESPATH || true
+  cp -f /var/tmp/*.txt $RESPATH || true
+  tar cf /tmp/qemu-$OS.tar -C $RESPATH -h . || true
+  exit 0
+fi
+
+# build was okay
+VMs=`cat vms.txt`
+
+####################################################################
+# vm${N}log.txt   -> output of ssh/tail -> for merged summary.txt
+#
+# vm${N}/build-stderr.txt  -> two copies -> moved to one file
+# vm${N}/dmesg-prerun.txt  -> dmesg output of vm start
+# vm${N}/console.txt       -> serial output of vm
+# vm${N}/uname.txt         -> output of uname -a on test vm
+#
+# vm${N}/current/log       -> if not there, kernel panic loading
+# vm${N}/current/results   -> if not there, kernel panic testings
+####################################################################
+
+BASE="$HOME/work/zfs/zfs"
+MERGE="$BASE/.github/workflows/scripts/merge_summary.awk"
+
+# catch result files of testings
+for i in `seq 1 $VMs`; do
+  rsync -arL zfs@192.168.122.1$i:$RESPATH/current $RESPATH/vm$i || true
+  scp zfs@192.168.122.1$i:"/var/tmp/*.txt" $RESPATH/vm$i || true
+done
+cp -f /var/tmp/*.txt $RESPATH || true
+cd $RESPATH
+
+# Save a list of all failed test logs for easy access
+awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }; \
+  /\[SKIP\]|\[PASS\]/{ show=0; } show' \
+  vm*/current/log >> summary-failure-logs.txt
+
+$MERGE vm*log.txt > summary-clean.txt
+$MERGE vm*log.txt | $BASE/scripts/zfs-tests-color.sh > summary.txt
+
+# we should have "vm count" identical build-stderr.txt files, need only one
+for i in `seq 1 $VMs`; do
+  file="vm$i/build-stderr.txt"
+  test -s "$file" && mv -f $file build-stderr.txt
+done
+
+for i in `seq 1 $VMs`; do
+  file="vm${i}log.txt"
+  test -s $file && cat $file | $BASE/scripts/zfs-tests-color.sh > $file.color
+done
+
+# artifact ready now
+tar cf /tmp/qemu-$OS.tar -C $RESPATH -h . || true
--- a/.github/workflows/scripts/qemu-7-summary.sh
+++ b/.github/workflows/scripts/qemu-7-summary.sh
@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 7) output the prepared results
+######################################################################
+
+# max size in KiB of debug output
+DEBUG_MAX="100"
+
+set -o pipefail
+
+cd /var/tmp/test_results
+OS=`cat os.txt`
+RV=0
+
+# helper function for showing some content with headline
+function showfile() {
+  content=`dd if=$1 bs=1024 count=$DEBUG_MAX 2>/dev/null`
+  hl="$2"
+  SIZE=`stat --printf="%s" "$file"`
+  SIZE=$((SIZE/1024))
+  kb=" ($SIZE KiB)"
+cat <<EOF > tmp$$
+##[group]$hl${kb}
+$content
+##[endgroup]
+EOF
+  cat tmp$$
+  rm -f tmp$$
+}
+
+# overview
+cat summary.txt
+echo ""
+
+echo "Full logs for download: $1"
+echo ""
+
+echo "File listing:"
+ls -l
+
+echo ""
+file="build-stderr.txt"
+test -s "$file" && showfile "$file" "Stderr of module build"
+
+# all other logs are only generated with the testings
+test -s vms.txt || exit 1
+
+# build was okay
+VMs=`cat vms.txt`
+
+# Did we have a test failure?
+if grep -vq 0 vm*/exitcode.txt; then
+  echo ""
+  file="summary-failure-logs.txt"
+
+  showfile "$file" "One or more tests failed, debug file"
+  echo ""
+
+  cat summary.txt
+  echo ""
+
+  echo "Full logs for download: $1"
+  RV=1
+fi
+
+for i in `seq 1 $VMs`; do
+  file="vm$i/dmesg-prerun.txt"
+  test -s "$file" && showfile "$file" "vm$i: dmesg kernel"
+
+  file="vm$i/console.txt"
+  test -s "$file" && showfile "$file" "vm$i: serial console"
+
+  file="vm${i}log.txt.color"
+  test -s "$file" && showfile "$file" "vm$i: test results"
+done
+
+exit $RV
--- a/.github/workflows/scripts/qemu-8-summary.sh
+++ b/.github/workflows/scripts/qemu-8-summary.sh
@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 8) generate github summary page of all the testings
+######################################################################
+
+function output() {
+  echo -e $* >> "out-$logfile.md"
+}
+
+function outfile() {
+  CUR=`stat --printf="%s" "out-$logfile.md"`
+  ADD=`stat --printf="%s" "$1"`
+  X=$((CUR+ADD))
+  if [ $X -gt $((1024*1023)) ]; then
+    logfile=$((logfile+1))
+  fi
+  cat "$1" >> "out-$logfile.md"
+}
+
+function send2github() {
+  test -f "$1" || exit 0
+  dd if="$1" bs=1023k count=1 >> $GITHUB_STEP_SUMMARY
+}
+
+# generate summary of one test
+function generate() {
+  osname=`cat osname.txt`
+  VMs=`cat vms.txt`
+
+  logfile=$((logfile+1))
+  output "\n## Functional Tests: $osname\n"
+  for i in `seq 1 $VMs`; do
+    for f in uname.txt; do
+      test -s vm$i/$f && cat vm$i/$f >> $f
+      touch $f
+    done
+  done
+
+  output "<pre>"
+  outfile uname.txt
+  output "</pre>"
+
+  if [ -s "summary-clean.txt" ]; then
+    output "<pre>"
+    outfile "summary-clean.txt"
+    output "</pre>"
+  fi
+}
+
+# https://docs.github.com/en/enterprise-server@3.6/actions/using-workflows/workflow-commands-for-github-actions#step-isolation-and-limits
+# Job summaries are isolated between steps and each step is restricted to a maximum size of 1MiB.
+# [ ] can not show all error findings here
+# [x] split files into smaller ones and create additional steps
+
+# first call, generate all summaries
+if [ ! -f out-1.md ]; then
+  # create ./zts-report.py for generate()
+  TEMPLATE="tests/test-runner/bin/zts-report.py.in"
+  cat $TEMPLATE| sed -e 's|@PYTHON_SHEBANG@|python3|' > ./zts-report.py
+  chmod +x ./zts-report.py
+
+  logfile="0"
+  for tarfile in Logs-functional-*/qemu-*.tar; do
+    if [ ! -s "$tarfile" ]; then
+      output "\n# Functional Tests: unknown\n"
+      output ":exclamation: Tarfile $tarfile is empty :exclamation:"
+      continue
+    fi
+    rm -rf vm* *.txt
+    tar xf "$tarfile"
+    generate
+  done
+  send2github out-1.md
+else
+  send2github out-$1.md
+fi
--- a/.github/workflows/zfs-qemu.yml
+++ b/.github/workflows/zfs-qemu.yml
@ -0,0 +1,139 @@
+name: zfs-qemu
+
+on:
+  push:
+  pull_request:
+
+jobs:
+
+  qemu-vm:
+    name: qemu-vm
+    strategy:
+      fail-fast: false
+      matrix:
+        # all:
+        # os: [almalinux8, almalinux9, archlinux, centos-stream9, fedora39, fedora40, debian11, debian12, freebsd13, freebsd13r, freebsd14, freebsd14r, freebsd15, ubuntu20, ubuntu22, ubuntu24]
+        # openzfs:
+        os: [almalinux8, almalinux9, centos-stream9, fedora39, fedora40, freebsd13, freebsd13r, freebsd14, freebsd14r, freebsd15, ubuntu20, ubuntu22, ubuntu24]
+        # freebsd:
+        # os: [freebsd13, freebsd13r, freebsd14, freebsd14r]
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: Setup QEMU
+      timeout-minutes: 10
+      run: .github/workflows/scripts/qemu-1-setup.sh
+
+    - name: Start build machine
+      timeout-minutes: 10
+      run: .github/workflows/scripts/qemu-2-start.sh ${{ matrix.os }}
+
+    - name: Install dependencies
+      timeout-minutes: 20
+      run: |
+        echo "Install dependencies in QEMU machine"
+        IP=192.168.122.10
+        while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do
+          ssh 2>/dev/null zfs@$IP "uname -a" && break
+        done
+        scp .github/workflows/scripts/qemu-3-deps.sh zfs@$IP:qemu-3-deps.sh
+        PID=`pidof /usr/bin/qemu-system-x86_64`
+        ssh zfs@$IP '$HOME/qemu-3-deps.sh' ${{ matrix.os }}
+        # wait for poweroff to succeed
+        tail --pid=$PID -f /dev/null
+        sleep 5 # avoid this: "error: Domain is already active"
+        rm -f $HOME/.ssh/known_hosts
+
+    - name: Build modules
+      timeout-minutes: 30
+      run: |
+        echo "Build modules in QEMU machine"
+        sudo virsh start openzfs
+        IP=192.168.122.10
+        while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do
+          ssh 2>/dev/null zfs@$IP "uname -a" && break
+        done
+        rsync -ar $HOME/work/zfs/zfs zfs@$IP:./
+        ssh zfs@$IP '$HOME/zfs/.github/workflows/scripts/qemu-4-build.sh' ${{ matrix.os }}
+
+    - name: Setup testing machines
+      timeout-minutes: 5
+      run: .github/workflows/scripts/qemu-5-setup.sh
+
+    - name: Run tests
+      timeout-minutes: 270
+      run: .github/workflows/scripts/qemu-6-tests.sh ${{ matrix.os }}
+
+    - name: Prepare artifacts
+      if: always()
+      timeout-minutes: 10
+      run: .github/workflows/scripts/qemu-7-prepare.sh
+
+    - uses: actions/upload-artifact@v4
+      id: artifact-upload
+      if: always()
+      with:
+        name: Logs-functional-${{ matrix.os }}
+        path: /tmp/qemu-${{ matrix.os }}.tar
+        if-no-files-found: ignore
+
+    - name: Test Summary
+      if: success() || failure()
+      run: .github/workflows/scripts/qemu-7-summary.sh '${{ steps.artifact-upload.outputs.artifact-url }}'
+
+  cleanup:
+    if: always()
+    name: Cleanup
+    runs-on: ubuntu-latest
+    needs: [ qemu-vm ]
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - uses: actions/download-artifact@v4
+    - name: Generating summary
+      run: .github/workflows/scripts/qemu-8-summary.sh
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 2
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 3
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 4
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 5
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 6
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 7
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 8
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 9
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 10
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 11
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 12
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 13
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 14
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 15
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 16
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 17
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 18
+    - name: Generating summary...
+      run: .github/workflows/scripts/qemu-8-summary.sh 19
+    - uses: actions/upload-artifact@v4
+      with:
+        name: Summary Files
+        path: out-*
--- a/scripts/zfs-tests.sh
+++ b/scripts/zfs-tests.sh
@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 # shellcheck disable=SC2154
 #
 # CDDL HEADER START
@ -208,6 +208,46 @@ find_runfile() {
 	fi
 }

+# Given a TAGS with a format like "1/3" or "2/3" then divide up the test list
+# into portions and print that portion.  So "1/3" for "the first third of the
+# test tags".
+#
+#
+split_tags() {
+	# Get numerator and denominator
+	NUM=$(echo $TAGS | cut -d/ -f1)
+	DEN=$(echo $TAGS | cut -d/ -f2)
+	# At the point this is called, RUNFILES will contain a comma separated
+	# list of full paths to the runfiles, like:
+	#
+	# "/home/hutter/qemu/tests/runfiles/common.run,/home/hutter/qemu/tests/runfiles/linux.run"
+	#
+	# So to get tags for our selected tests we do:
+	#
+	# 1. Remove unneeded chars: [],\
+	# 2. Print out the last field of each tag line.  This will be the tag
+	#    for the test (like 'zpool_add').
+	# 3. Remove duplicates between the runfiles.  If the same tag is defined
+	#    in multiple runfiles, then when you do '-T <tag>' ZTS is smart
+	#    enough to know to run the tag in each runfile.  So '-T zpool_add'
+	#    will run the zpool_add from common.run and linux.run.
+	# 4. Ignore the 'functional' tag since we only want individual tests
+	# 5. Print out the tests in our faction of all tests.  This uses modulus
+	#    so "1/3" will run tests 1,3,6,9 etc.  That way the tests are
+	#    interleaved so, say, "3/4" isn't running all the zpool_* tests that
+	#    appear alphabetically at the end.
+	# 6. Remove trailing comma from list
+	#
+	# TAGS will then look like:
+	#
+	# "append,atime,bootfs,cachefile,checksum,cp_files,deadman,dos_attributes, ..."
+
+	cat ${RUNFILES/,/ } | tr -d [],\' | awk '/tags = /{print $NF}' | sort | \
+		uniq | grep -v functional | \
+		awk -v num=$NUM -v den=$DEN '{ if(NR % den == (num - 1)) {printf "%s,",$0}}' | \
+		sed -E 's/,$//'
+}
+
 #
 # Symlink file if it appears under any of the given paths.
 #
@ -331,6 +371,10 @@ OPTIONS:
 	-t PATH|NAME  Run single test at PATH relative to test suite,
 	                or search for test by NAME
 	-T TAGS     Comma separated list of tags (default: 'functional')
+	            Alternately, specify a fraction like "1/3" or "2/3" to
+		     run the first third of tests or 2nd third of the tests.  This
+		     is useful for splitting up the test amongst different
+		     runners.
 	-u USER     Run single test as USER (default: root)

 EXAMPLES:
@ -489,6 +533,8 @@ fi
 #
 TAGS=${TAGS:='functional'}

+
+
 #
 # Attempt to locate the runfiles describing the test workload.
 #
@ -509,6 +555,23 @@ done
 unset IFS
 RUNFILES=${R#,}

+# The tag can be a fraction to indicate which portion of ZTS to run, Like
+#
+# 	"1/3": Run first one third of all tests in runfiles
+#	"2/3": Run second one third of all test in runfiles
+#	"6/10": Run 6th tenth of all tests in runfiles
+#
+# This is useful for splitting up the test across multiple runners.
+#
+# After this code block, TAGS will be transformed from something like
+# "1/3" to a comma separate taglist, like:
+#
+# "append,atime,bootfs,cachefile,checksum,cp_files,deadman,dos_attributes, ..."
+#
+if echo $TAGS | grep -Eq '^[0-9]+/[0-9]+$' ; then
+	TAGS=$(split_tags)
+fi
+
 #
 # This script should not be run as root.  Instead the test user, which may
 # be a normal user account, needs to be configured such that it can