Tag zfs-2.0.7

META file and changelog updated.

Signed-off-by: Tony Hutter <hutter2@llnl.gov>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -25,14 +25,16 @@ Type | Version/Name
  --- | ---
 Distribution Name	|
 Distribution Version	|
-Linux Kernel	|
+Kernel Version	|
 Architecture	|
-ZFS Version	|
-SPL Version	|
+OpenZFS Version	|
 <!--
-Commands to find ZFS/SPL versions:
-modinfo zfs | grep -iw version
-modinfo spl | grep -iw version
+Command to find OpenZFS version:
+zfs version
+
+Commands to find kernel version:
+uname -r		# Linux
+freebsd-version -r	# FreeBSD
 -->
 
 ### Describe the problem you're observing

.github/ISSUE_TEMPLATE/config.yml

@@ -7,5 +7,5 @@ contact_links:
     url: https://lists.freebsd.org/mailman/listinfo/freebsd-fs
     about: Get community support for OpenZFS on FreeBSD
   - name: OpenZFS on IRC
-    url: https://webchat.freenode.net/#openzfs
+    url: https://web.libera.chat/#openzfs
     about: Use IRC to get community support for OpenZFS

.github/workflows/checkstyle.yaml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   checkstyle:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
       with:
@@ -18,12 +18,13 @@ jobs:
         sudo apt-get install --yes -qq zlib1g-dev uuid-dev libattr1-dev libblkid-dev libselinux-dev libudev-dev libssl-dev python-dev python-setuptools python-cffi python3 python3-dev python3-setuptools python3-cffi
         # packages for tests
         sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fio
-        sudo apt-get install --yes -qq mandoc cppcheck pax-utils abigail-tools # devscripts - enable then bashisms fixed
+        sudo apt-get install --yes -qq mandoc cppcheck pax-utils devscripts
         sudo -E pip --quiet install flake8
     - name: Prepare
       run: |
         sh ./autogen.sh
         ./configure
+        make -j$(nproc)
     - name: Checkstyle
       run: |
         make checkstyle
@@ -31,6 +32,19 @@ jobs:
       run: |
         make lint
     - name: CheckABI
+      id: CheckABI
       run: |
-        make -j$(nproc)
-        make checkabi
+        sudo docker run -v $(pwd):/source ghcr.io/openzfs/libabigail make checkabi
+    - name: StoreABI
+      if: failure() && steps.CheckABI.outcome == 'failure'
+      run: |
+        sudo docker run -v $(pwd):/source ghcr.io/openzfs/libabigail make storeabi
+    - name: Prepare artifacts
+      if: failure() && steps.CheckABI.outcome == 'failure'
+      run: |
+        find -name *.abi | tar -cf abi_files.tar -T -
+    - uses: actions/upload-artifact@v2
+      if: failure() && steps.CheckABI.outcome == 'failure'
+      with:
+        name: New ABI files (use only if you're sure about interface changes)
+        path: abi_files.tar

.github/workflows/zfs-tests-functional.yml

@@ -26,7 +26,7 @@ jobs:
           xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
           libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
           libpam0g-dev pamtester python-dev python-setuptools python-cffi \
-          python3 python3-dev python3-setuptools python3-cffi
+          python3 python3-dev python3-setuptools python3-cffi python3-packaging
     - name: Autogen.sh
       run: |
         sh autogen.sh
@@ -44,6 +44,17 @@ jobs:
         sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
         sudo depmod
         sudo modprobe zfs
+        # Workaround for cloud-init bug
+        # see https://github.com/openzfs/zfs/issues/12644
+        FILE=/lib/udev/rules.d/10-cloud-init-hook-hotplug.rules
+        if [ -r "${FILE}" ]; then
+          HASH=$(md5sum "${FILE}" | awk '{ print $1 }')
+          if [ "${HASH}" = "121ff0ef1936cd2ef65aec0458a35772" ]; then
+            # Just shove a zd* exclusion right above the hotplug hook...
+            sudo sed -i -e s/'LABEL="cloudinit_hook"'/'KERNEL=="zd*", GOTO="cloudinit_end"\n&'/ "${FILE}"
+            sudo udevadm control --reload-rules
+          fi
+        fi
         # Workaround to provide additional free space for testing.
         #   https://github.com/actions/virtual-environments/issues/2840
         sudo rm -rf /usr/share/dotnet
@@ -52,7 +63,7 @@ jobs:
         sudo rm -rf "$AGENT_TOOLSDIRECTORY"
     - name: Tests
       run: |
-        /usr/share/zfs/zfs-tests.sh -v -s 3G
+        /usr/share/zfs/zfs-tests.sh -vR -s 3G
     - name: Prepare artifacts
       if: failure()
       run: |
@@ -61,7 +72,7 @@ jobs:
         sudo cp /var/log/syslog $RESULTS_PATH/
         sudo chmod +r $RESULTS_PATH/*
         # Replace ':' in dir names, actions/upload-artifact doesn't support it
-        for f in $(find $RESULTS_PATH -name '*:*'); do mv "$f" "${f//:/__}"; done
+        for f in $(find /var/tmp/test_results -name '*:*'); do mv "$f" "${f//:/__}"; done
     - uses: actions/upload-artifact@v2
       if: failure()
       with:

.github/workflows/zfs-tests-sanity.yml

@@ -22,7 +22,7 @@ jobs:
           xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
           libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
           libpam0g-dev pamtester python-dev python-setuptools python-cffi \
-          python3 python3-dev python3-setuptools python3-cffi
+          python3 python3-dev python3-setuptools python3-cffi python3-packaging
     - name: Autogen.sh
       run: |
         sh autogen.sh
@@ -40,6 +40,17 @@ jobs:
         sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
         sudo depmod
         sudo modprobe zfs
+        # Workaround for cloud-init bug
+        # see https://github.com/openzfs/zfs/issues/12644
+        FILE=/lib/udev/rules.d/10-cloud-init-hook-hotplug.rules
+        if [ -r "${FILE}" ]; then
+          HASH=$(md5sum "${FILE}" | awk '{ print $1 }')
+          if [ "${HASH}" = "121ff0ef1936cd2ef65aec0458a35772" ]; then
+            # Just shove a zd* exclusion right above the hotplug hook...
+            sudo sed -i -e s/'LABEL="cloudinit_hook"'/'KERNEL=="zd*", GOTO="cloudinit_end"\n&'/ "${FILE}"
+            sudo udevadm control --reload-rules
+          fi
+        fi
         # Workaround to provide additional free space for testing.
         #   https://github.com/actions/virtual-environments/issues/2840
         sudo rm -rf /usr/share/dotnet
@@ -48,7 +59,7 @@ jobs:
         sudo rm -rf "$AGENT_TOOLSDIRECTORY"
     - name: Tests
       run: |
-        /usr/share/zfs/zfs-tests.sh -v -s 3G -r sanity
+        /usr/share/zfs/zfs-tests.sh -vR -s 3G -r sanity
     - name: Prepare artifacts
       if: failure()
       run: |
@@ -57,7 +68,7 @@ jobs:
         sudo cp /var/log/syslog $RESULTS_PATH/
         sudo chmod +r $RESULTS_PATH/*
         # Replace ':' in dir names, actions/upload-artifact doesn't support it
-        for f in $(find $RESULTS_PATH -name '*:*'); do mv "$f" "${f//:/__}"; done
+        for f in $(find /var/tmp/test_results -name '*:*'); do mv "$f" "${f//:/__}"; done
     - uses: actions/upload-artifact@v2
       if: failure()
       with:

.github/workflows/zloop.yml

@@ -22,8 +22,8 @@ jobs:
           xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
           libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
           libpam0g-dev \
-          python-dev python-setuptools python-cffi \
-          python3 python3-dev python3-setuptools python3-cffi
+          python-dev python-setuptools python-cffi python-packaging \
+          python3 python3-dev python3-setuptools python3-cffi python3-packaging
     - name: Autogen.sh
       run: |
         sh autogen.sh

META

@@ -1,10 +1,10 @@
 Meta:          1
 Name:          zfs
 Branch:        1.0
-Version:       2.0.5
+Version:       2.0.7
 Release:       1
 Release-Tags:  relext
 License:       CDDL
 Author:        OpenZFS
-Linux-Maximum: 5.12
+Linux-Maximum: 5.15
 Linux-Minimum: 3.10

cmd/arc_summary/arc_summary3

@@ -58,7 +58,6 @@ SECTION_PATHS = {'arc': 'arcstats',
                  'dmu': 'dmu_tx',
                  'l2arc': 'arcstats',  # L2ARC stuff lives in arcstats
                  'vdev': 'vdev_cache_stats',
-                 'xuio': 'xuio_stats',
                  'zfetch': 'zfetchstats',
                  'zil': 'zil'}
 

cmd/mount_zfs/mount_zfs.c

@@ -367,7 +367,7 @@ main(int argc, char **argv)
 				    "mount the filesystem again.\n"), dataset);
 				return (MOUNT_SYSERR);
 			}
-			/* fallthru */
+			fallthrough;
 #endif
 		default:
 			(void) fprintf(stderr, gettext("filesystem "

cmd/vdev_id/vdev_id

@@ -79,6 +79,34 @@
 # channel 86:00.0 1         A
 # channel 86:00.0 0         B
 
+# #
+# # Example vdev_id.conf - multipath / multijbod-daisychaining
+# #
+#
+# multipath yes
+# multijbod yes
+#
+# #       PCI_ID  HBA PORT  CHANNEL NAME
+# channel 85:00.0 1         A
+# channel 85:00.0 0         B
+# channel 86:00.0 1         A
+# channel 86:00.0 0         B
+
+# #
+# # Example vdev_id.conf - multipath / mixed
+# #
+#
+# multipath yes
+# slot mix
+#
+# #       PCI_ID  HBA PORT  CHANNEL NAME
+# channel 85:00.0 3         A
+# channel 85:00.0 2         B
+# channel 86:00.0 3         A
+# channel 86:00.0 2         B
+# channel af:00.0 0         C
+# channel af:00.0 1         C
+
 # #
 # # Example vdev_id.conf - alias
 # #
@@ -92,9 +120,10 @@ PATH=/bin:/sbin:/usr/bin:/usr/sbin
 CONFIG=/etc/zfs/vdev_id.conf
 PHYS_PER_PORT=
 DEV=
-MULTIPATH=
 TOPOLOGY=
 BAY=
+ENCL_ID=""
+UNIQ_ENCL_ID=""
 
 usage() {
 	cat << EOF
@@ -107,22 +136,25 @@ Usage: vdev_id [-h]
   -e    Create enclose device symlinks only (/dev/by-enclosure)
   -g    Storage network topology [default="$TOPOLOGY"]
   -m    Run in multipath mode
+  -j    Run in multijbod mode
   -p    number of phy's per switch port [default=$PHYS_PER_PORT]
   -h    show this summary
 EOF
-	exit 0
+	exit 1
+	# exit with error to avoid processing usage message by a udev rule
 }
 
 map_slot() {
 	LINUX_SLOT=$1
 	CHANNEL=$2
 
-	MAPPED_SLOT=`awk "\\$1 == \"slot\" && \\$2 == ${LINUX_SLOT} && \
-			\\$4 ~ /^${CHANNEL}$|^$/ { print \\$3; exit }" $CONFIG`
+	MAPPED_SLOT=$(awk -v linux_slot="$LINUX_SLOT" -v channel="$CHANNEL" \
+			'$1 == "slot" && $2 == linux_slot && \
+			($4 ~ "^"channel"$" || $4 ~ /^$/) { print $3; exit}' $CONFIG)
 	if [ -z "$MAPPED_SLOT" ] ; then
 		MAPPED_SLOT=$LINUX_SLOT
 	fi
-	printf "%d" ${MAPPED_SLOT}
+	printf "%d" "${MAPPED_SLOT}"
 }
 
 map_channel() {
@@ -132,40 +164,120 @@ map_channel() {
 
 	case $TOPOLOGY in
 		"sas_switch")
-		MAPPED_CHAN=`awk "\\$1 == \"channel\" && \\$2 == ${PORT} \
-			{ print \\$3; exit }" $CONFIG`
+		MAPPED_CHAN=$(awk -v port="$PORT" \
+			'$1 == "channel" && $2 == port \
+			{ print $3; exit }' $CONFIG)
 		;;
 		"sas_direct"|"scsi")
-		MAPPED_CHAN=`awk "\\$1 == \"channel\" && \
-			\\$2 == \"${PCI_ID}\" && \\$3 == ${PORT} \
-			{ print \\$4; exit }" $CONFIG`
+		MAPPED_CHAN=$(awk -v pciID="$PCI_ID" -v port="$PORT" \
+			'$1 == "channel" && $2 == pciID && $3 == port \
+			{print $4}' $CONFIG)
 		;;
 	esac
-	printf "%s" ${MAPPED_CHAN}
+	printf "%s" "${MAPPED_CHAN}"
+}
+
+get_encl_id() {
+	set -- $(echo $1)
+	count=$#
+
+	i=1
+	while [ $i -le $count ] ; do
+		d=$(eval echo '$'{$i})
+		id=$(cat "/sys/class/enclosure/${d}/id")
+		ENCL_ID="${ENCL_ID} $id"
+		i=$((i + 1))
+	done
+}
+
+get_uniq_encl_id() {
+	for uuid in ${ENCL_ID}; do
+		found=0
+
+		for count in ${UNIQ_ENCL_ID}; do
+			if [ $count = $uuid ]; then
+				found=1
+				break
+			fi
+		done
+
+		if [ $found -eq 0 ]; then
+			UNIQ_ENCL_ID="${UNIQ_ENCL_ID} $uuid"
+		fi
+	done
+}
+
+# map_jbod explainer: The bsg driver knows the difference between a SAS
+# expander and fanout expander. Use hostX instance along with top-level
+# (whole enclosure) expander instances in /sys/class/enclosure and
+# matching a field in an array of expanders, using the index of the
+# matched array field as the enclosure instance, thereby making jbod IDs
+# dynamic. Avoids reliance on high overhead userspace commands like
+# multipath and lsscsi and instead uses existing sysfs data.  $HOSTCHAN
+# variable derived from devpath gymnastics in sas_handler() function.
+map_jbod() {
+	DEVEXP=$(ls -l "/sys/block/$DEV/device/" | grep enclos | awk -F/ '{print $(NF-1) }')
+	DEV=$1
+
+	# Use "set --" to create index values (Arrays)
+	set -- $(ls -l /sys/class/enclosure | grep -v "^total" | awk '{print $9}')
+	# Get count of total elements
+	JBOD_COUNT=$#
+	JBOD_ITEM=$*
+
+	# Build JBODs (enclosure)  id from sys/class/enclosure/<dev>/id
+	get_encl_id "$JBOD_ITEM"
+	# Different expander instances for each paths.
+	# Filter out and keep only unique id.
+	get_uniq_encl_id
+
+	# Identify final 'mapped jbod'
+	j=0
+	for count in ${UNIQ_ENCL_ID}; do
+		i=1
+		j=$((j + 1))
+		while [ $i -le $JBOD_COUNT ] ; do
+			d=$(eval echo '$'{$i})
+			id=$(cat "/sys/class/enclosure/${d}/id")
+			if [ "$d" = "$DEVEXP" ] && [ $id = $count ] ; then
+				MAPPED_JBOD=$j
+				break
+			fi
+			i=$((i + 1))
+		done
+	done
+
+	printf "%d" "${MAPPED_JBOD}"
 }
 
 sas_handler() {
 	if [ -z "$PHYS_PER_PORT" ] ; then
-		PHYS_PER_PORT=`awk "\\$1 == \"phys_per_port\" \
-			{print \\$2; exit}" $CONFIG`
+		PHYS_PER_PORT=$(awk '$1 == "phys_per_port" \
+			{print $2; exit}' $CONFIG)
 	fi
 	PHYS_PER_PORT=${PHYS_PER_PORT:-4}
-	if ! echo $PHYS_PER_PORT | grep -q -E '^[0-9]+$' ; then
+
+	if ! echo "$PHYS_PER_PORT" | grep -q -E '^[0-9]+$' ; then
 		echo "Error: phys_per_port value $PHYS_PER_PORT is non-numeric"
 		exit 1
 	fi
 
 	if [ -z "$MULTIPATH_MODE" ] ; then
-		MULTIPATH_MODE=`awk "\\$1 == \"multipath\" \
-			{print \\$2; exit}" $CONFIG`
+		MULTIPATH_MODE=$(awk '$1 == "multipath" \
+			{print $2; exit}' $CONFIG)
+	fi
+
+	if [ -z "$MULTIJBOD_MODE" ] ; then
+		MULTIJBOD_MODE=$(awk '$1 == "multijbod" \
+			{print $2; exit}' $CONFIG)
 	fi
 
 	# Use first running component device if we're handling a dm-mpath device
 	if [ "$MULTIPATH_MODE" = "yes" ] ; then
 		# If udev didn't tell us the UUID via DM_NAME, check /dev/mapper
 		if [ -z "$DM_NAME" ] ; then
-			DM_NAME=`ls -l --full-time /dev/mapper |
-				awk "/\/$DEV$/{print \\$9}"`
+			DM_NAME=$(ls -l --full-time /dev/mapper |
+				grep "$DEV"$ | awk '{print $9}')
 		fi
 
 		# For raw disks udev exports DEVTYPE=partition when
@@ -175,28 +287,50 @@ sas_handler() {
 		# we have to append the -part suffix directly in the
 		# helper.
 		if [ "$DEVTYPE" != "partition" ] ; then
-			PART=`echo $DM_NAME | awk -Fp '/p/{print "-part"$2}'`
+			# Match p[number], remove the 'p' and prepend "-part"
+			PART=$(echo "$DM_NAME" |
+				awk 'match($0,/p[0-9]+$/) {print "-part"substr($0,RSTART+1,RLENGTH-1)}')
 		fi
 
 		# Strip off partition information.
-		DM_NAME=`echo $DM_NAME | sed 's/p[0-9][0-9]*$//'`
+		DM_NAME=$(echo "$DM_NAME" | sed 's/p[0-9][0-9]*$//')
 		if [ -z "$DM_NAME" ] ; then
 			return
 		fi
 
-		# Get the raw scsi device name from multipath -ll. Strip off
-		# leading pipe symbols to make field numbering consistent.
-		DEV=`multipath -ll $DM_NAME |
-			awk '/running/{gsub("^[|]"," "); print $3 ; exit}'`
+		# Utilize DM device name to gather subordinate block devices
+		# using sysfs to avoid userspace utilities
+
+		# If our DEVNAME is something like /dev/dm-177, then we may be
+		# able to get our DMDEV from it.
+		DMDEV=$(echo $DEVNAME | sed 's;/dev/;;g')
+		if [ ! -e /sys/block/$DMDEV/slaves/* ] ; then
+			# It's not there, try looking in /dev/mapper
+			DMDEV=$(ls -l --full-time /dev/mapper | grep $DM_NAME |
+			awk '{gsub("../", " "); print $NF}')
+		fi
+
+		# Use sysfs pointers in /sys/block/dm-X/slaves because using
+		# userspace tools creates lots of overhead and should be avoided
+		# whenever possible. Use awk to isolate lowest instance of
+		# sd device member in dm device group regardless of string
+		# length.
+		DEV=$(ls "/sys/block/$DMDEV/slaves" | awk '
+			{ len=sprintf ("%20s",length($0)); gsub(/ /,0,str); a[NR]=len "_" $0; }
+			END {
+				asort(a)
+				print substr(a[1],22)
+			}')
+
 		if [ -z "$DEV" ] ; then
 			return
 		fi
 	fi
 
-	if echo $DEV | grep -q ^/devices/ ; then
+	if echo "$DEV" | grep -q ^/devices/ ; then
 		sys_path=$DEV
 	else
-		sys_path=`udevadm info -q path -p /sys/block/$DEV 2>/dev/null`
+		sys_path=$(udevadm info -q path -p "/sys/block/$DEV" 2>/dev/null)
 	fi
 
 	# Use positional parameters as an ad-hoc array
@@ -206,84 +340,104 @@ sas_handler() {
 
 	# Get path up to /sys/.../hostX
 	i=1
-	while [ $i -le $num_dirs ] ; do
-		d=$(eval echo \${$i})
+
+	while [ $i -le "$num_dirs" ] ; do
+		d=$(eval echo '$'{$i})
 		scsi_host_dir="$scsi_host_dir/$d"
-		echo $d | grep -q -E '^host[0-9]+$' && break
-		i=$(($i + 1))
+		echo "$d" | grep -q -E '^host[0-9]+$' && break
+		i=$((i + 1))
 	done
 
-	if [ $i = $num_dirs ] ; then
+	# Lets grab the SAS host channel number and save it for JBOD sorting later
+	HOSTCHAN=$(echo "$d" | awk -F/ '{ gsub("host","",$NF); print $NF}')
+
+	if [ $i = "$num_dirs" ] ; then
 		return
 	fi
 
-	PCI_ID=$(eval echo \${$(($i -1))} | awk -F: '{print $2":"$3}')
+	PCI_ID=$(eval echo '$'{$((i -1))} | awk -F: '{print $2":"$3}')
 
 	# In sas_switch mode, the directory four levels beneath
 	# /sys/.../hostX contains symlinks to phy devices that reveal
 	# the switch port number.  In sas_direct mode, the phy links one
 	# directory down reveal the HBA port.
 	port_dir=$scsi_host_dir
+
 	case $TOPOLOGY in
-		"sas_switch") j=$(($i + 4)) ;;
-		"sas_direct") j=$(($i + 1)) ;;
+		"sas_switch") j=$((i + 4)) ;;
+		"sas_direct") j=$((i + 1)) ;;
 	esac
 
-	i=$(($i + 1))
+	i=$((i + 1))
+
 	while [ $i -le $j ] ; do
-		port_dir="$port_dir/$(eval echo \${$i})"
-		i=$(($i + 1))
+		port_dir="$port_dir/$(eval echo '$'{$i})"
+		i=$((i + 1))
 	done
 
-	PHY=`ls -d $port_dir/phy* 2>/dev/null | head -1 | awk -F: '{print $NF}'`
+	PHY=$(ls -vd "$port_dir"/phy* 2>/dev/null | head -1 | awk -F: '{print $NF}')
 	if [ -z "$PHY" ] ; then
 		PHY=0
 	fi
-	PORT=$(( $PHY / $PHYS_PER_PORT ))
+	PORT=$((PHY / PHYS_PER_PORT))
 
 	# Look in /sys/.../sas_device/end_device-X for the bay_identifier
 	# attribute.
 	end_device_dir=$port_dir
-	while [ $i -lt $num_dirs ] ; do
-		d=$(eval echo \${$i})
+
+	while [ $i -lt "$num_dirs" ] ; do
+		d=$(eval echo '$'{$i})
 		end_device_dir="$end_device_dir/$d"
-		if echo $d | grep -q '^end_device' ; then
+		if echo "$d" | grep -q '^end_device' ; then
 			end_device_dir="$end_device_dir/sas_device/$d"
 			break
 		fi
-		i=$(($i + 1))
+		i=$((i + 1))
 	done
 
+	# Add 'mix' slot type for environments where dm-multipath devices
+	# include end-devices connected via SAS expanders or direct connection
+	# to SAS HBA. A mixed connectivity environment such as pool devices
+	# contained in a SAS JBOD and spare drives or log devices directly
+	# connected in a server backplane without expanders in the I/O path.
 	SLOT=
+
 	case $BAY in
 	"bay")
-		SLOT=`cat $end_device_dir/bay_identifier 2>/dev/null`
+		SLOT=$(cat "$end_device_dir/bay_identifier" 2>/dev/null)
+		;;
+	"mix")
+		if [ $(cat "$end_device_dir/bay_identifier" 2>/dev/null) ] ; then
+			SLOT=$(cat "$end_device_dir/bay_identifier" 2>/dev/null)
+		else
+			SLOT=$(cat "$end_device_dir/phy_identifier" 2>/dev/null)
+		fi
 		;;
 	"phy")
-		SLOT=`cat $end_device_dir/phy_identifier 2>/dev/null`
+		SLOT=$(cat "$end_device_dir/phy_identifier" 2>/dev/null)
 		;;
 	"port")
-		d=$(eval echo \${$i})
-		SLOT=`echo $d | sed -e 's/^.*://'`
+		d=$(eval echo '$'{$i})
+		SLOT=$(echo "$d" | sed -e 's/^.*://')
 		;;
 	"id")
-		i=$(($i + 1))
-		d=$(eval echo \${$i})
-		SLOT=`echo $d | sed -e 's/^.*://'`
+		i=$((i + 1))
+		d=$(eval echo '$'{$i})
+		SLOT=$(echo "$d" | sed -e 's/^.*://')
 		;;
 	"lun")
-		i=$(($i + 2))
-		d=$(eval echo \${$i})
-		SLOT=`echo $d | sed -e 's/^.*://'`
+		i=$((i + 2))
+		d=$(eval echo '$'{$i})
+		SLOT=$(echo "$d" | sed -e 's/^.*://')
 		;;
 	"ses")
 		# look for this SAS path in all SCSI Enclosure Services
 		# (SES) enclosures
-		sas_address=`cat $end_device_dir/sas_address 2>/dev/null`
-		enclosures=`lsscsi -g | \
-			sed -n -e '/enclosu/s/^.* \([^ ][^ ]*\) *$/\1/p'`
+		sas_address=$(cat "$end_device_dir/sas_address" 2>/dev/null)
+		enclosures=$(lsscsi -g | \
+			sed -n -e '/enclosu/s/^.* \([^ ][^ ]*\) *$/\1/p')
 		for enclosure in $enclosures; do
-			set -- $(sg_ses -p aes $enclosure | \
+			set -- $(sg_ses -p aes "$enclosure" | \
 				awk "/device slot number:/{slot=\$12} \
 					/SAS address: $sas_address/\
 					{print slot}")
@@ -298,42 +452,55 @@ sas_handler() {
 		return
 	fi
 
-	CHAN=`map_channel $PCI_ID $PORT`
-	SLOT=`map_slot $SLOT $CHAN`
-	if [ -z "$CHAN" ] ; then
-		return
+	if [ "$MULTIJBOD_MODE" = "yes" ] ; then
+		CHAN=$(map_channel "$PCI_ID" "$PORT")
+		SLOT=$(map_slot "$SLOT" "$CHAN")
+		JBOD=$(map_jbod "$DEV")
+
+		if [ -z "$CHAN" ] ; then
+			return
+		fi
+		echo "${CHAN}"-"${JBOD}"-"${SLOT}${PART}"
+	else
+		CHAN=$(map_channel "$PCI_ID" "$PORT")
+		SLOT=$(map_slot "$SLOT" "$CHAN")
+
+		if [ -z "$CHAN" ] ; then
+			return
+		fi
+		echo "${CHAN}${SLOT}${PART}"
 	fi
-	echo ${CHAN}${SLOT}${PART}
 }
 
 scsi_handler() {
 	if [ -z "$FIRST_BAY_NUMBER" ] ; then
-		FIRST_BAY_NUMBER=`awk "\\$1 == \"first_bay_number\" \
-			{print \\$2; exit}" $CONFIG`
+		FIRST_BAY_NUMBER=$(awk '$1 == "first_bay_number" \
+			{print $2; exit}' $CONFIG)
 	fi
 	FIRST_BAY_NUMBER=${FIRST_BAY_NUMBER:-0}
 
 	if [ -z "$PHYS_PER_PORT" ] ; then
-		PHYS_PER_PORT=`awk "\\$1 == \"phys_per_port\" \
-			{print \\$2; exit}" $CONFIG`
+		PHYS_PER_PORT=$(awk '$1 == "phys_per_port" \
+			{print $2; exit}' $CONFIG)
 	fi
 	PHYS_PER_PORT=${PHYS_PER_PORT:-4}
-	if ! echo $PHYS_PER_PORT | grep -q -E '^[0-9]+$' ; then
+
+	if ! echo "$PHYS_PER_PORT" | grep -q -E '^[0-9]+$' ; then
 		echo "Error: phys_per_port value $PHYS_PER_PORT is non-numeric"
 		exit 1
 	fi
 
 	if [ -z "$MULTIPATH_MODE" ] ; then
-		MULTIPATH_MODE=`awk "\\$1 == \"multipath\" \
-			{print \\$2; exit}" $CONFIG`
+		MULTIPATH_MODE=$(awk '$1 == "multipath" \
+			{print $2; exit}' $CONFIG)
 	fi
 
 	# Use first running component device if we're handling a dm-mpath device
 	if [ "$MULTIPATH_MODE" = "yes" ] ; then
 		# If udev didn't tell us the UUID via DM_NAME, check /dev/mapper
 		if [ -z "$DM_NAME" ] ; then
-			DM_NAME=`ls -l --full-time /dev/mapper |
-				awk "/\/$DEV$/{print \\$9}"`
+			DM_NAME=$(ls -l --full-time /dev/mapper |
+				grep "$DEV"$ | awk '{print $9}')
 		fi
 
 		# For raw disks udev exports DEVTYPE=partition when
@@ -343,28 +510,30 @@ scsi_handler() {
 		# we have to append the -part suffix directly in the
 		# helper.
 		if [ "$DEVTYPE" != "partition" ] ; then
-			PART=`echo $DM_NAME | awk -Fp '/p/{print "-part"$2}'`
+			# Match p[number], remove the 'p' and prepend "-part"
+			PART=$(echo "$DM_NAME" |
+			    awk 'match($0,/p[0-9]+$/) {print "-part"substr($0,RSTART+1,RLENGTH-1)}')
 		fi
 
 		# Strip off partition information.
-		DM_NAME=`echo $DM_NAME | sed 's/p[0-9][0-9]*$//'`
+		DM_NAME=$(echo "$DM_NAME" | sed 's/p[0-9][0-9]*$//')
 		if [ -z "$DM_NAME" ] ; then
 			return
 		fi
 
 		# Get the raw scsi device name from multipath -ll. Strip off
 		# leading pipe symbols to make field numbering consistent.
-		DEV=`multipath -ll $DM_NAME |
-			awk '/running/{gsub("^[|]"," "); print $3 ; exit}'`
+		DEV=$(multipath -ll "$DM_NAME" |
+			awk '/running/{gsub("^[|]"," "); print $3 ; exit}')
 		if [ -z "$DEV" ] ; then
 			return
 		fi
 	fi
 
-	if echo $DEV | grep -q ^/devices/ ; then
+	if echo "$DEV" | grep -q ^/devices/ ; then
 		sys_path=$DEV
 	else
-		sys_path=`udevadm info -q path -p /sys/block/$DEV 2>/dev/null`
+		sys_path=$(udevadm info -q path -p "/sys/block/$DEV" 2>/dev/null)
 	fi
 
 	# expect sys_path like this, for example:
@@ -377,44 +546,47 @@ scsi_handler() {
 
 	# Get path up to /sys/.../hostX
 	i=1
-	while [ $i -le $num_dirs ] ; do
-		d=$(eval echo \${$i})
+
+	while [ $i -le "$num_dirs" ] ; do
+		d=$(eval echo '$'{$i})
 		scsi_host_dir="$scsi_host_dir/$d"
-		echo $d | grep -q -E '^host[0-9]+$' && break
-		i=$(($i + 1))
+
+		echo "$d" | grep -q -E '^host[0-9]+$' && break
+		i=$((i + 1))
 	done
 
-	if [ $i = $num_dirs ] ; then
+	if [ $i = "$num_dirs" ] ; then
 		return
 	fi
 
-	PCI_ID=$(eval echo \${$(($i -1))} | awk -F: '{print $2":"$3}')
+	PCI_ID=$(eval echo '$'{$((i -1))} | awk -F: '{print $2":"$3}')
 
 	# In scsi mode, the directory two levels beneath
 	# /sys/.../hostX reveals the port and slot.
 	port_dir=$scsi_host_dir
-	j=$(($i + 2))
+	j=$((i + 2))
 
-	i=$(($i + 1))
+	i=$((i + 1))
 	while [ $i -le $j ] ; do
-		port_dir="$port_dir/$(eval echo \${$i})"
-		i=$(($i + 1))
+		port_dir="$port_dir/$(eval echo '$'{$i})"
+		i=$((i + 1))
 	done
 
-	set -- $(echo $port_dir | sed -e 's/^.*:\([^:]*\):\([^:]*\)$/\1 \2/')
+	set -- $(echo "$port_dir" | sed -e 's/^.*:\([^:]*\):\([^:]*\)$/\1 \2/')
 	PORT=$1
-	SLOT=$(($2 + $FIRST_BAY_NUMBER))
+	SLOT=$(($2 + FIRST_BAY_NUMBER))
 
 	if [ -z "$SLOT" ] ; then
 		return
 	fi
 
-	CHAN=`map_channel $PCI_ID $PORT`
-	SLOT=`map_slot $SLOT $CHAN`
+	CHAN=$(map_channel "$PCI_ID" "$PORT")
+	SLOT=$(map_slot "$SLOT" "$CHAN")
+
 	if [ -z "$CHAN" ] ; then
 		return
 	fi
-	echo ${CHAN}${SLOT}${PART}
+	echo "${CHAN}${SLOT}${PART}"
 }
 
 # Figure out the name for the enclosure symlink
@@ -425,7 +597,7 @@ enclosure_handler () {
 
 	# Get the enclosure ID ("0:0:0:0")
 	ENC=$(basename $(readlink -m "/sys/$DEVPATH/../.."))
-	if [ ! -d /sys/class/enclosure/$ENC ] ; then
+	if [ ! -d "/sys/class/enclosure/$ENC" ] ; then
 		# Not an enclosure, bail out
 		return
 	fi
@@ -433,14 +605,14 @@ enclosure_handler () {
 	# Get the long sysfs device path to our enclosure. Looks like:
 	# /devices/pci0000:00/0000:00:03.0/0000:05:00.0/host0/port-0:0/ ... /enclosure/0:0:0:0
 
-	ENC_DEVICE=$(readlink /sys/class/enclosure/$ENC)
+	ENC_DEVICE=$(readlink "/sys/class/enclosure/$ENC")
 
 	# Grab the full path to the hosts port dir:
 	# /devices/pci0000:00/0000:00:03.0/0000:05:00.0/host0/port-0:0
-	PORT_DIR=$(echo $ENC_DEVICE | grep -Eo '.+host[0-9]+/port-[0-9]+:[0-9]+')
+	PORT_DIR=$(echo "$ENC_DEVICE" | grep -Eo '.+host[0-9]+/port-[0-9]+:[0-9]+')
 
 	# Get the port number
-	PORT_ID=$(echo $PORT_DIR | grep -Eo "[0-9]+$")
+	PORT_ID=$(echo "$PORT_DIR" | grep -Eo "[0-9]+$")
 
 	# The PCI directory is two directories up from the port directory
 	# /sys/devices/pci0000:00/0000:00:03.0/0000:05:00.0
@@ -451,7 +623,7 @@ enclosure_handler () {
 
 	# Name our device according to vdev_id.conf (like "L0" or "U1").
 	NAME=$(awk "/channel/{if (\$1 == \"channel\" && \$2 == \"$PCI_ID\" && \
-		\$3 == \"$PORT_ID\") {print \$4int(count[\$4])}; count[\$4]++}" $CONFIG)
+		\$3 == \"$PORT_ID\") {print \$4\$3}}" $CONFIG)
 
 	echo "${NAME}"
 }
@@ -487,9 +659,11 @@ alias_handler () {
 	#          ambiguity seems unavoidable, so devices using this facility
 	#          must not use such names.
 	DM_PART=
-	if echo $DM_NAME | grep -q -E 'p[0-9][0-9]*$' ; then
+	if echo "$DM_NAME" | grep -q -E 'p[0-9][0-9]*$' ; then
 		if [ "$DEVTYPE" != "partition" ] ; then
-			DM_PART=`echo $DM_NAME | awk -Fp '/p/{print "-part"$2}'`
+			# Match p[number], remove the 'p' and prepend "-part"
+			DM_PART=$(echo "$DM_NAME" |
+			    awk 'match($0,/p[0-9]+$/) {print "-part"substr($0,RSTART+1,RLENGTH-1)}')
 		fi
 	fi
 
@@ -497,21 +671,25 @@ alias_handler () {
 	for link in $DEVLINKS ; do
 		# Remove partition information to match key of top-level device.
 		if [ -n "$DM_PART" ] ; then
-			link=`echo $link | sed 's/p[0-9][0-9]*$//'`
+			link=$(echo "$link" | sed 's/p[0-9][0-9]*$//')
 		fi
 		# Check both the fully qualified and the base name of link.
-		for l in $link `basename $link` ; do
-			alias=`awk "\\$1 == \"alias\" && \\$3 == \"${l}\" \
-					{ print \\$2; exit }" $CONFIG`
-			if [ -n "$alias" ] ; then
-				echo ${alias}${DM_PART}
-				return
+		for l in $link $(basename "$link") ; do
+			if [ ! -z "$l" ]; then
+				alias=$(awk -v var="$l" '($1 == "alias") && \
+					($3 == var) \
+					{ print $2; exit }' $CONFIG)
+				if [ -n "$alias" ] ; then
+					echo "${alias}${DM_PART}"
+					return
+				fi
 			fi
 		done
 	done
 }
 
-while getopts 'c:d:eg:mp:h' OPTION; do
+# main
+while getopts 'c:d:eg:jmp:h' OPTION; do
 	case ${OPTION} in
 	c)
 		CONFIG=${OPTARG}
@@ -524,7 +702,9 @@ while getopts 'c:d:eg:mp:h' OPTION; do
 	# create the enclosure device symlinks only.  We also need
 	# "enclosure_symlinks yes" set in vdev_id.config to actually create the
 	# symlink.
-	ENCLOSURE_MODE=$(awk '{if ($1 == "enclosure_symlinks") print $2}' $CONFIG)
+	ENCLOSURE_MODE=$(awk '{if ($1 == "enclosure_symlinks") \
+		print $2}' "$CONFIG")
+
 	if [ "$ENCLOSURE_MODE" != "yes" ] ; then
 		exit 0
 	fi
@@ -535,6 +715,9 @@ while getopts 'c:d:eg:mp:h' OPTION; do
 	p)
 		PHYS_PER_PORT=${OPTARG}
 		;;
+	j)
+		MULTIJBOD_MODE=yes
+		;;
 	m)
 		MULTIPATH_MODE=yes
 		;;
@@ -544,9 +727,9 @@ while getopts 'c:d:eg:mp:h' OPTION; do
 	esac
 done
 
-if [ ! -r $CONFIG ] ; then
+if [ ! -r "$CONFIG" ] ; then
 	echo "Error: Config file \"$CONFIG\" not found"
-	exit 0
+	exit 1
 fi
 
 if [ -z "$DEV" ] && [ -z "$ENCLOSURE_MODE" ] ; then
@@ -555,11 +738,11 @@ if [ -z "$DEV" ] && [ -z "$ENCLOSURE_MODE" ] ; then
 fi
 
 if [ -z "$TOPOLOGY" ] ; then
-	TOPOLOGY=`awk "\\$1 == \"topology\" {print \\$2; exit}" $CONFIG`
+	TOPOLOGY=$(awk '($1 == "topology") {print $2; exit}' "$CONFIG")
 fi
 
 if [ -z "$BAY" ] ; then
-	BAY=`awk "\\$1 == \"slot\" {print \\$2; exit}" $CONFIG`
+	BAY=$(awk '($1 == "slot") {print $2; exit}' "$CONFIG")
 fi
 
 TOPOLOGY=${TOPOLOGY:-sas_direct}
@@ -572,7 +755,7 @@ if [ "$ENCLOSURE_MODE" = "yes" ] && [ "$TOPOLOGY" = "sas_direct" ] ; then
 	fi
 
 	# Just create the symlinks to the enclosure devices and then exit.
-	ENCLOSURE_PREFIX=$(awk '/enclosure_symlinks_prefix/{print $2}' $CONFIG)
+	ENCLOSURE_PREFIX=$(awk '/enclosure_symlinks_prefix/{print $2}' "$CONFIG")
 	if [ -z "$ENCLOSURE_PREFIX" ] ; then
 		ENCLOSURE_PREFIX="enc"
 	fi
@@ -582,16 +765,16 @@ if [ "$ENCLOSURE_MODE" = "yes" ] && [ "$TOPOLOGY" = "sas_direct" ] ; then
 fi
 
 # First check if an alias was defined for this device.
-ID_VDEV=`alias_handler`
+ID_VDEV=$(alias_handler)
 
 if [ -z "$ID_VDEV" ] ; then
 	BAY=${BAY:-bay}
 	case $TOPOLOGY in
 		sas_direct|sas_switch)
-			ID_VDEV=`sas_handler`
+			ID_VDEV=$(sas_handler)
 			;;
 		scsi)
-			ID_VDEV=`scsi_handler`
+			ID_VDEV=$(scsi_handler)
 			;;
 		*)
 			echo "Error: unknown topology $TOPOLOGY"

cmd/zdb/zdb.c

@@ -161,12 +161,6 @@ static int dump_bpobj_cb(void *arg, const blkptr_t *bp, boolean_t free,
     dmu_tx_t *tx);
 
 typedef struct sublivelist_verify {
-	/* all ALLOC'd blkptr_t in one sub-livelist */
-	zfs_btree_t sv_all_allocs;
-
-	/* all FREE'd blkptr_t in one sub-livelist */
-	zfs_btree_t sv_all_frees;
-
 	/* FREE's that haven't yet matched to an ALLOC, in one sub-livelist */
 	zfs_btree_t sv_pair;
 
@@ -225,29 +219,68 @@ typedef struct sublivelist_verify_block {
 
 static void zdb_print_blkptr(const blkptr_t *bp, int flags);
 
+typedef struct sublivelist_verify_block_refcnt {
+	/* block pointer entry in livelist being verified */
+	blkptr_t svbr_blk;
+
+	/*
+	 * Refcount gets incremented to 1 when we encounter the first
+	 * FREE entry for the svfbr block pointer and a node for it
+	 * is created in our ZDB verification/tracking metadata.
+	 *
+	 * As we encounter more FREE entries we increment this counter
+	 * and similarly decrement it whenever we find the respective
+	 * ALLOC entries for this block.
+	 *
+	 * When the refcount gets to 0 it means that all the FREE and
+	 * ALLOC entries of this block have paired up and we no longer
+	 * need to track it in our verification logic (e.g. the node
+	 * containing this struct in our verification data structure
+	 * should be freed).
+	 *
+	 * [refer to sublivelist_verify_blkptr() for the actual code]
+	 */
+	uint32_t svbr_refcnt;
+} sublivelist_verify_block_refcnt_t;
+
+static int
+sublivelist_block_refcnt_compare(const void *larg, const void *rarg)
+{
+	const sublivelist_verify_block_refcnt_t *l = larg;
+	const sublivelist_verify_block_refcnt_t *r = rarg;
+	return (livelist_compare(&l->svbr_blk, &r->svbr_blk));
+}
+
 static int
 sublivelist_verify_blkptr(void *arg, const blkptr_t *bp, boolean_t free,
     dmu_tx_t *tx)
 {
 	ASSERT3P(tx, ==, NULL);
 	struct sublivelist_verify *sv = arg;
-	char blkbuf[BP_SPRINTF_LEN];
+	sublivelist_verify_block_refcnt_t current = {
+			.svbr_blk = *bp,
+
+			/*
+			 * Start with 1 in case this is the first free entry.
+			 * This field is not used for our B-Tree comparisons
+			 * anyway.
+			 */
+			.svbr_refcnt = 1,
+	};
+
 	zfs_btree_index_t where;
+	sublivelist_verify_block_refcnt_t *pair =
+	    zfs_btree_find(&sv->sv_pair, &current, &where);
 	if (free) {
-		zfs_btree_add(&sv->sv_pair, bp);
-		/* Check if the FREE is a duplicate */
-		if (zfs_btree_find(&sv->sv_all_frees, bp, &where) != NULL) {
-			snprintf_blkptr_compact(blkbuf, sizeof (blkbuf), bp,
-			    free);
-			(void) printf("\tERROR: Duplicate FREE: %s\n", blkbuf);
+		if (pair == NULL) {
+			/* first free entry for this block pointer */
+			zfs_btree_add(&sv->sv_pair, &current);
 		} else {
-			zfs_btree_add_idx(&sv->sv_all_frees, bp, &where);
+			pair->svbr_refcnt++;
 		}
 	} else {
-		/* Check if the ALLOC has been freed */
-		if (zfs_btree_find(&sv->sv_pair, bp, &where) != NULL) {
-			zfs_btree_remove_idx(&sv->sv_pair, &where);
-		} else {
+		if (pair == NULL) {
+			/* block that is currently marked as allocated */
 			for (int i = 0; i < SPA_DVAS_PER_BP; i++) {
 				if (DVA_IS_EMPTY(&bp->blk_dva[i]))
 					break;
@@ -262,16 +295,16 @@ sublivelist_verify_blkptr(void *arg, const blkptr_t *bp, boolean_t free,
 					    &svb, &where);
 				}
 			}
-		}
-		/* Check if the ALLOC is a duplicate */
-		if (zfs_btree_find(&sv->sv_all_allocs, bp, &where) != NULL) {
-			snprintf_blkptr_compact(blkbuf, sizeof (blkbuf), bp,
-			    free);
-			(void) printf("\tERROR: Duplicate ALLOC: %s\n", blkbuf);
 		} else {
-			zfs_btree_add_idx(&sv->sv_all_allocs, bp, &where);
+			/* alloc matches a free entry */
+			pair->svbr_refcnt--;
+			if (pair->svbr_refcnt == 0) {
+				/* all allocs and frees have been matched */
+				zfs_btree_remove_idx(&sv->sv_pair, &where);
+			}
 		}
 	}
+
 	return (0);
 }
 
@@ -279,32 +312,22 @@ static int
 sublivelist_verify_func(void *args, dsl_deadlist_entry_t *dle)
 {
 	int err;
-	char blkbuf[BP_SPRINTF_LEN];
 	struct sublivelist_verify *sv = args;
 
-	zfs_btree_create(&sv->sv_all_allocs, livelist_compare,
-	    sizeof (blkptr_t));
-
-	zfs_btree_create(&sv->sv_all_frees, livelist_compare,
-	    sizeof (blkptr_t));
-
-	zfs_btree_create(&sv->sv_pair, livelist_compare,
-	    sizeof (blkptr_t));
+	zfs_btree_create(&sv->sv_pair, sublivelist_block_refcnt_compare,
+	    sizeof (sublivelist_verify_block_refcnt_t));
 
 	err = bpobj_iterate_nofree(&dle->dle_bpobj, sublivelist_verify_blkptr,
 	    sv, NULL);
 
-	zfs_btree_clear(&sv->sv_all_allocs);
-	zfs_btree_destroy(&sv->sv_all_allocs);
-
-	zfs_btree_clear(&sv->sv_all_frees);
-	zfs_btree_destroy(&sv->sv_all_frees);
-
-	blkptr_t *e;
+	sublivelist_verify_block_refcnt_t *e;
 	zfs_btree_index_t *cookie = NULL;
 	while ((e = zfs_btree_destroy_nodes(&sv->sv_pair, &cookie)) != NULL) {
-		snprintf_blkptr_compact(blkbuf, sizeof (blkbuf), e, B_TRUE);
-		(void) printf("\tERROR: Unmatched FREE: %s\n", blkbuf);
+		char blkbuf[BP_SPRINTF_LEN];
+		snprintf_blkptr_compact(blkbuf, sizeof (blkbuf),
+		    &e->svbr_blk, B_TRUE);
+		(void) printf("\tERROR: %d unmatched FREE(s): %s\n",
+		    e->svbr_refcnt, blkbuf);
 	}
 	zfs_btree_destroy(&sv->sv_pair);
 
@@ -613,10 +636,14 @@ mv_populate_livelist_allocs(metaslab_verify_t *mv, sublivelist_verify_t *sv)
 /*
  * [Livelist Check]
  * Iterate through all the sublivelists and:
- * - report leftover frees
- * - report double ALLOCs/FREEs
+ * - report leftover frees (**)
  * - record leftover ALLOCs together with their TXG [see Cross Check]
  *
+ * (**) Note: Double ALLOCs are valid in datasets that have dedup
+ *      enabled. Similarly double FREEs are allowed as well but
+ *      only if they pair up with a corresponding ALLOC entry once
+ *      we our done with our sublivelist iteration.
+ *
  * [Spacemap Check]
  * for each metaslab:
  * - iterate over spacemap and then the metaslab's entries in the
@@ -2184,7 +2211,8 @@ snprintf_zstd_header(spa_t *spa, char *blkbuf, size_t buflen,
 		(void) snprintf(blkbuf + strlen(blkbuf),
 		    buflen - strlen(blkbuf),
 		    " ZSTD:size=%u:version=%u:level=%u:EMBEDDED",
-		    zstd_hdr.c_len, zstd_hdr.version, zstd_hdr.level);
+		    zstd_hdr.c_len, zfs_get_hdrversion(&zstd_hdr),
+		    zfs_get_hdrlevel(&zstd_hdr));
 		return;
 	}
 
@@ -2208,7 +2236,8 @@ snprintf_zstd_header(spa_t *spa, char *blkbuf, size_t buflen,
 	(void) snprintf(blkbuf + strlen(blkbuf),
 	    buflen - strlen(blkbuf),
 	    " ZSTD:size=%u:version=%u:level=%u:NORMAL",
-	    zstd_hdr.c_len, zstd_hdr.version, zstd_hdr.level);
+	    zstd_hdr.c_len, zfs_get_hdrversion(&zstd_hdr),
+	    zfs_get_hdrlevel(&zstd_hdr));
 
 	abd_return_buf_copy(pabd, buf, BP_GET_LSIZE(bp));
 }
@@ -4060,7 +4089,7 @@ cksum_record_compare(const void *x1, const void *x2)
 	const cksum_record_t *l = (cksum_record_t *)x1;
 	const cksum_record_t *r = (cksum_record_t *)x2;
 	int arraysize = ARRAY_SIZE(l->cksum.zc_word);
-	int difference;
+	int difference = 0;
 
 	for (int i = 0; i < arraysize; i++) {
 		difference = TREE_CMP(l->cksum.zc_word[i], r->cksum.zc_word[i]);
@@ -4535,7 +4564,7 @@ dump_path_impl(objset_t *os, uint64_t obj, char *name)
 	case DMU_OT_DIRECTORY_CONTENTS:
 		if (s != NULL && *(s + 1) != '\0')
 			return (dump_path_impl(os, child_obj, s + 1));
-		/*FALLTHROUGH*/
+		fallthrough;
 	case DMU_OT_PLAIN_FILE_CONTENTS:
 		dump_object(os, child_obj, dump_opt['v'], &header, NULL, 0);
 		return (0);

cmd/zed/zed.d/statechange-led.sh

@@ -1,21 +1,21 @@
 #!/bin/sh
 #
-# Turn off/on the VDEV's enclosure fault LEDs when the pool's state changes.
+# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes.
 #
-# Turn the VDEV's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL.
-# Turn the LED off when it's back ONLINE again.
+# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL.
+# Turn its LED off when it's back ONLINE again.
 #
 # This script run in two basic modes:
 #
 # 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then
-# only set the LED for that particular VDEV. This is the case for statechange
+# only set the LED for that particular vdev. This is the case for statechange
 # events and some vdev_* events.
 #
-# 2. If those vars are not set, then check the state of all VDEVs in the pool
+# 2. If those vars are not set, then check the state of all vdevs in the pool
 # and set the LEDs accordingly.  This is the case for pool_import events.
 #
 # Note that this script requires that your enclosure be supported by the
-# Linux SCSI enclosure services (ses) driver.  The script will do nothing
+# Linux SCSI Enclosure services (SES) driver.  The script will do nothing
 # if you have no enclosure, or if your enclosure isn't supported.
 #
 # Exit codes:
@@ -29,7 +29,8 @@
 [ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
 . "${ZED_ZEDLET_DIR}/zed-functions.sh"
 
-if [ ! -d /sys/class/enclosure ] ; then
+if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then
+	# No JBOD enclosure or NVMe slots
 	exit 1
 fi
 
@@ -59,6 +60,10 @@ check_and_set_led()
 	file="$1"
 	val="$2"
 
+	if [ -z "$val" ]; then
+		return 0
+	fi
+
 	if [ ! -e "$file" ] ; then
 		return 3
 	fi
@@ -66,11 +71,11 @@ check_and_set_led()
 	# If another process is accessing the LED when we attempt to update it,
 	# the update will be lost so retry until the LED actually changes or we
 	# timeout.
-	for _ in $(seq 1 5); do
+	for _ in 1 2 3 4 5; do
 		# We want to check the current state first, since writing to the
 		# 'fault' entry always causes a SES command, even if the
 		# current state is already what you want.
-		current=$(cat "${file}")
+		read -r current < "${file}"
 
 		# On some enclosures if you write 1 to fault, and read it back,
 		# it will return 2.  Treat all non-zero values as 1 for
@@ -85,27 +90,84 @@ check_and_set_led()
 		else
 			break
 		fi
-        done
+	done
+}
+
+# Fault LEDs for JBODs and NVMe drives are handled a little differently.
+#
+# On JBODs the fault LED is called 'fault' and on a path like this:
+#
+#   /sys/class/enclosure/0:0:1:0/SLOT 10/fault
+#
+# On NVMe it's called 'attention' and on a path like this:
+#
+#   /sys/bus/pci/slot/0/attention
+#
+# This function returns the full path to the fault LED file for a given
+# enclosure/slot directory.
+#
+path_to_led()
+{
+	dir=$1
+	if [ -f "$dir/fault" ] ; then
+		echo "$dir/fault"
+	elif [ -f "$dir/attention" ] ; then
+		echo "$dir/attention"
+	fi
 }
 
 state_to_val()
 {
 	state="$1"
-	if [ "$state" = "FAULTED" ] || [ "$state" = "DEGRADED" ] || \
-	   [ "$state" = "UNAVAIL" ] ; then
-		echo 1
-	elif [ "$state" = "ONLINE" ] ; then
-		echo 0
-	fi
+	case "$state" in
+		FAULTED|DEGRADED|UNAVAIL)
+			echo 1
+			;;
+		ONLINE)
+			echo 0
+			;;
+	esac
 }
 
-# process_pool ([pool])
 #
-# Iterate through a pool (or pools) and set the VDEV's enclosure slot LEDs to
-# the VDEV's state.
+# Given a nvme name like 'nvme0n1', pass back its slot directory
+# like "/sys/bus/pci/slots/0"
+#
+nvme_dev_to_slot()
+{
+	dev="$1"
+
+	# Get the address "0000:01:00.0"
+	address=$(cat "/sys/class/block/$dev/device/address")
+
+	# For each /sys/bus/pci/slots subdir that is an actual number
+	# (rather than weird directories like "1-3/").
+	# shellcheck disable=SC2010
+	for i in $(ls /sys/bus/pci/slots/ | grep -E "^[0-9]+$") ; do
+		this_address=$(cat "/sys/bus/pci/slots/$i/address")
+
+		# The format of address is a little different between
+		# /sys/class/block/$dev/device/address and
+		# /sys/bus/pci/slots/
+		#
+		# address=           "0000:01:00.0"
+		# this_address =     "0000:01:00"
+		#
+		if echo "$address" | grep -Eq ^"$this_address" ; then
+			echo "/sys/bus/pci/slots/$i"
+			break
+		fi
+	done
+}
+
+
+# process_pool (pool)
+#
+# Iterate through a pool and set the vdevs' enclosure slot LEDs to
+# those vdevs' state.
 #
 # Arguments
-#   pool:	Optional pool name.  If not specified, iterate though all pools.
+#   pool:	Pool name.
 #
 # Return
 #  0 on success, 3 on missing sysfs path
@@ -113,19 +175,27 @@ state_to_val()
 process_pool()
 {
 	pool="$1"
+
+	# The output will be the vdevs only (from "grep '/dev/'"):
+	#
+	#    U45     ONLINE       0     0     0   /dev/sdk          0
+	#    U46     ONLINE       0     0     0   /dev/sdm          0
+	#    U47     ONLINE       0     0     0   /dev/sdn          0
+	#    U50     ONLINE       0     0     0  /dev/sdbn          0
+	#
+	ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | (
 	rc=0
-
-	# Lookup all the current LED values and paths in parallel
-	#shellcheck disable=SC2016
-	cmd='echo led_token=$(cat "$VDEV_ENC_SYSFS_PATH/fault"),"$VDEV_ENC_SYSFS_PATH",'
-	out=$($ZPOOL status -vc "$cmd" "$pool" | grep 'led_token=')
-
-	#shellcheck disable=SC2034
-	echo "$out" | while read -r vdev state read write chksum therest; do
+	while read -r vdev state _ _ _ therest; do
 		# Read out current LED value and path
-		tmp=$(echo "$therest" | sed 's/^.*led_token=//g')
-		vdev_enc_sysfs_path=$(echo "$tmp" | awk -F ',' '{print $2}')
-		current_val=$(echo "$tmp" | awk -F ',' '{print $1}')
+		# Get dev name (like 'sda')
+		dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')")
+		vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*)
+		if [ ! -d "$vdev_enc_sysfs_path" ] ; then
+			# This is not a JBOD disk, but it could be a PCI NVMe drive
+			vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev")
+		fi
+
+		current_val=$(echo "$therest" | awk '{print $NF}')
 
 		if [ "$current_val" != "0" ] ; then
 			current_val=1
@@ -136,40 +206,33 @@ process_pool()
 			continue
 		fi
 
-		if [ ! -e "$vdev_enc_sysfs_path/fault" ] ; then
-			#shellcheck disable=SC2030
-			rc=1
-			zed_log_msg "vdev $vdev '$file/fault' doesn't exist"
-			continue;
+		led_path=$(path_to_led "$vdev_enc_sysfs_path")
+		if [ ! -e "$led_path" ] ; then
+			rc=3
+			zed_log_msg "vdev $vdev '$led_path' doesn't exist"
+			continue
 		fi
 
 		val=$(state_to_val "$state")
 
 		if [ "$current_val" = "$val" ] ; then
 			# LED is already set correctly
-			continue;
+			continue
 		fi
 
-		if ! check_and_set_led "$vdev_enc_sysfs_path/fault" "$val"; then
-			rc=1
+		if ! check_and_set_led "$led_path" "$val"; then
+			rc=3
 		fi
-
 	done
-
-	#shellcheck disable=SC2031
-	if [ "$rc" = "0" ] ; then
-		return 0
-	else
-		# We didn't see a sysfs entry that we wanted to set
-		return 3
-	fi
+	exit "$rc"; )
 }
 
 if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then
-	# Got a statechange for an individual VDEV
+	# Got a statechange for an individual vdev
 	val=$(state_to_val "$ZEVENT_VDEV_STATE_STR")
 	vdev=$(basename "$ZEVENT_VDEV_PATH")
-	check_and_set_led "$ZEVENT_VDEV_ENC_SYSFS_PATH/fault" "$val"
+	ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH")
+	check_and_set_led "$ledpath" "$val"
 else
 	# Process the entire pool
 	poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID")

cmd/zed/zed.d/zed.rc

@@ -89,8 +89,8 @@
 
 ##
 # Turn on/off enclosure LEDs when drives get DEGRADED/FAULTED.  This works for
-# device mapper and multipath devices as well.  Your enclosure must be
-# supported by the Linux SES driver for this to work.
+# device mapper and multipath devices as well.  This works with JBOD enclosures
+# and NVMe PCI drives (assuming they're supported by Linux in sysfs).
 #
 ZED_USE_ENCLOSURE_LEDS=1
 

cmd/zed/zed_file.c

@@ -12,11 +12,11 @@
  * You may not use this file except in compliance with the license.
  */
 
+#include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <limits.h>
 #include <string.h>
-#include <sys/resource.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
@@ -160,6 +160,13 @@ zed_file_is_locked(int fd)
 	return (lock.l_pid);
 }
 
+
+#if __APPLE__
+#define	PROC_SELF_FD "/dev/fd"
+#else /* Linux-compatible layout */
+#define	PROC_SELF_FD "/proc/self/fd"
+#endif
+
 /*
  * Close all open file descriptors greater than or equal to [lowfd].
  * Any errors encountered while closing file descriptors are ignored.
@@ -167,20 +174,21 @@ zed_file_is_locked(int fd)
 void
 zed_file_close_from(int lowfd)
 {
-	const int maxfd_def = 256;
-	int errno_bak;
-	struct rlimit rl;
-	int maxfd;
+	int errno_bak = errno;
+	int maxfd = 0;
 	int fd;
+	DIR *fddir;
+	struct dirent *fdent;
 
-	errno_bak = errno;
-
-	if (getrlimit(RLIMIT_NOFILE, &rl) < 0) {
-		maxfd = maxfd_def;
-	} else if (rl.rlim_max == RLIM_INFINITY) {
-		maxfd = maxfd_def;
+	if ((fddir = opendir(PROC_SELF_FD)) != NULL) {
+		while ((fdent = readdir(fddir)) != NULL) {
+			fd = atoi(fdent->d_name);
+			if (fd > maxfd && fd != dirfd(fddir))
+				maxfd = fd;
+		}
+		(void) closedir(fddir);
 	} else {
-		maxfd = rl.rlim_max;
+		maxfd = sysconf(_SC_OPEN_MAX);
 	}
 	for (fd = lowfd; fd < maxfd; fd++)
 		(void) close(fd);

cmd/zfs/zfs_main.c

@@ -730,6 +730,32 @@ finish_progress(char *done)
 	pt_header = NULL;
 }
 
+/* This function checks if the passed fd refers to /dev/null or /dev/zero */
+#ifdef __linux__
+static boolean_t
+is_dev_nullzero(int fd)
+{
+	struct stat st;
+	fstat(fd, &st);
+	return (major(st.st_rdev) == 1 && (minor(st.st_rdev) == 3 /* null */ ||
+	    minor(st.st_rdev) == 5 /* zero */));
+}
+#endif
+
+static void
+note_dev_error(int err, int fd)
+{
+#ifdef __linux__
+	if (err == EINVAL && is_dev_nullzero(fd)) {
+		(void) fprintf(stderr,
+		    gettext("Error: Writing directly to /dev/{null,zero} files"
+		    " on certain kernels is not currently implemented.\n"
+		    "(As a workaround, "
+		    "try \"zfs send [...] | cat > /dev/null\")\n"));
+	}
+#endif
+}
+
 static int
 zfs_mount_and_share(libzfs_handle_t *hdl, const char *dataset, zfs_type_t type)
 {
@@ -4448,11 +4474,16 @@ zfs_do_send(int argc, char **argv)
 
 		err = zfs_send_saved(zhp, &flags, STDOUT_FILENO,
 		    resume_token);
+		if (err != 0)
+			note_dev_error(errno, STDOUT_FILENO);
 		zfs_close(zhp);
 		return (err != 0);
 	} else if (resume_token != NULL) {
-		return (zfs_send_resume(g_zfs, &flags, STDOUT_FILENO,
-		    resume_token));
+		err = zfs_send_resume(g_zfs, &flags, STDOUT_FILENO,
+		    resume_token);
+		if (err != 0)
+			note_dev_error(errno, STDOUT_FILENO);
+		return (err);
 	}
 
 	/*
@@ -4496,6 +4527,8 @@ zfs_do_send(int argc, char **argv)
 		err = zfs_send_one(zhp, fromname, STDOUT_FILENO, &flags,
 		    redactbook);
 		zfs_close(zhp);
+		if (err != 0)
+			note_dev_error(errno, STDOUT_FILENO);
 		return (err != 0);
 	}
 
@@ -4572,6 +4605,7 @@ zfs_do_send(int argc, char **argv)
 		nvlist_free(dbgnv);
 	}
 	zfs_close(zhp);
+	note_dev_error(errno, STDOUT_FILENO);
 
 	return (err != 0);
 }
@@ -7310,6 +7344,7 @@ unshare_unmount(int op, int argc, char **argv)
 				if (zfs_prop_get_int(zhp, ZFS_PROP_CANMOUNT) ==
 				    ZFS_CANMOUNT_NOAUTO)
 					continue;
+				break;
 			default:
 				break;
 			}

cmd/zpool/Makefile.am

@@ -25,7 +25,8 @@ zpool_LDADD = \
 	$(abs_top_builddir)/lib/libzfs/libzfs.la \
 	$(abs_top_builddir)/lib/libzfs_core/libzfs_core.la \
 	$(abs_top_builddir)/lib/libnvpair/libnvpair.la \
-	$(abs_top_builddir)/lib/libuutil/libuutil.la
+	$(abs_top_builddir)/lib/libuutil/libuutil.la \
+	$(abs_top_builddir)/lib/libzutil/libzutil.la
 
 zpool_LDADD += $(LTLIBINTL)
 

cmd/zpool/zpool.d/ses

@@ -41,7 +41,13 @@ for i in $scripts ; do
 		val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null)
 		;;
 	fault_led)
-		val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null)
+		# JBODs fault LED is called 'fault', NVMe fault LED is called
+		# 'attention'.
+		if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then
+			val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null)
+		elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then
+			val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null)
+		fi
 		;;
 	locate_led)
 		val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null)

cmd/zpool/zpool_iter.c

@@ -264,51 +264,6 @@ for_each_pool(int argc, char **argv, boolean_t unavail,
 	return (ret);
 }
 
-static int
-for_each_vdev_cb(zpool_handle_t *zhp, nvlist_t *nv, pool_vdev_iter_f func,
-    void *data)
-{
-	nvlist_t **child;
-	uint_t c, children;
-	int ret = 0;
-	int i;
-	char *type;
-
-	const char *list[] = {
-	    ZPOOL_CONFIG_SPARES,
-	    ZPOOL_CONFIG_L2CACHE,
-	    ZPOOL_CONFIG_CHILDREN
-	};
-
-	for (i = 0; i < ARRAY_SIZE(list); i++) {
-		if (nvlist_lookup_nvlist_array(nv, list[i], &child,
-		    &children) == 0) {
-			for (c = 0; c < children; c++) {
-				uint64_t ishole = 0;
-
-				(void) nvlist_lookup_uint64(child[c],
-				    ZPOOL_CONFIG_IS_HOLE, &ishole);
-
-				if (ishole)
-					continue;
-
-				ret |= for_each_vdev_cb(zhp, child[c], func,
-				    data);
-			}
-		}
-	}
-
-	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) != 0)
-		return (ret);
-
-	/* Don't run our function on root vdevs */
-	if (strcmp(type, VDEV_TYPE_ROOT) != 0) {
-		ret |= func(zhp, nv, data);
-	}
-
-	return (ret);
-}
-
 /*
  * This is the equivalent of for_each_pool() for vdevs.  It iterates thorough
  * all vdevs in the pool, ignoring root vdevs and holes, calling func() on
@@ -327,7 +282,7 @@ for_each_vdev(zpool_handle_t *zhp, pool_vdev_iter_f func, void *data)
 		verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
 		    &nvroot) == 0);
 	}
-	return (for_each_vdev_cb(zhp, nvroot, func, data));
+	return (for_each_vdev_cb((void *) zhp, nvroot, func, data));
 }
 
 /*
@@ -598,7 +553,7 @@ vdev_run_cmd_thread(void *cb_cmd_data)
 
 /* For each vdev in the pool run a command */
 static int
-for_each_vdev_run_cb(zpool_handle_t *zhp, nvlist_t *nv, void *cb_vcdl)
+for_each_vdev_run_cb(void *zhp_data, nvlist_t *nv, void *cb_vcdl)
 {
 	vdev_cmd_data_list_t *vcdl = cb_vcdl;
 	vdev_cmd_data_t *data;
@@ -606,6 +561,7 @@ for_each_vdev_run_cb(zpool_handle_t *zhp, nvlist_t *nv, void *cb_vcdl)
 	char *vname = NULL;
 	char *vdev_enc_sysfs_path = NULL;
 	int i, match = 0;
+	zpool_handle_t *zhp = zhp_data;
 
 	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) != 0)
 		return (1);

cmd/zpool/zpool_main.c

@@ -5049,11 +5049,12 @@ get_stat_flags(zpool_list_t *list)
  * Return 1 if cb_data->cb_vdev_names[0] is this vdev's name, 0 otherwise.
  */
 static int
-is_vdev_cb(zpool_handle_t *zhp, nvlist_t *nv, void *cb_data)
+is_vdev_cb(void *zhp_data, nvlist_t *nv, void *cb_data)
 {
 	iostat_cbdata_t *cb = cb_data;
 	char *name = NULL;
 	int ret = 0;
+	zpool_handle_t *zhp = zhp_data;
 
 	name = zpool_vdev_name(g_zfs, zhp, nv, cb->cb_name_flags);
 

cmd/zpool/zpool_util.h

@@ -27,6 +27,7 @@
 
 #include <libnvpair.h>
 #include <libzfs.h>
+#include <libzutil.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -67,7 +68,6 @@ int for_each_pool(int, char **, boolean_t unavail, zprop_list_t **,
     boolean_t, zpool_iter_f, void *);
 
 /* Vdev list functions */
-typedef int (*pool_vdev_iter_f)(zpool_handle_t *, nvlist_t *, void *);
 int for_each_vdev(zpool_handle_t *zhp, pool_vdev_iter_f func, void *data);
 
 typedef struct zpool_list zpool_list_t;

config/Rules.am

@@ -26,6 +26,7 @@ AM_LIBTOOLFLAGS = --silent
 AM_CFLAGS  = -std=gnu99 -Wall -Wstrict-prototypes -Wmissing-prototypes
 AM_CFLAGS += -fno-strict-aliasing
 AM_CFLAGS += $(NO_OMIT_FRAME_POINTER)
+AM_CFLAGS += $(IMPLICIT_FALLTHROUGH)
 AM_CFLAGS += $(DEBUG_CFLAGS)
 AM_CFLAGS += $(ASAN_CFLAGS)
 AM_CFLAGS += $(CODE_COVERAGE_CFLAGS) $(NO_FORMAT_ZERO_LENGTH)

config/always-compiler-options.m4

@@ -161,6 +161,29 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_UNUSED_BUT_SET_VARIABLE], [
 	AC_SUBST([NO_UNUSED_BUT_SET_VARIABLE])
 ])
 
+dnl #
+dnl # Check if gcc supports -Wimplicit-fallthrough option.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_IMPLICIT_FALLTHROUGH], [
+	AC_MSG_CHECKING([whether $CC supports -Wimplicit-fallthrough])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Werror -Wimplicit-fallthrough"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		IMPLICIT_FALLTHROUGH=-Wimplicit-fallthrough
+		AC_DEFINE([HAVE_IMPLICIT_FALLTHROUGH], 1,
+			[Define if compiler supports -Wimplicit-fallthrough])
+		AC_MSG_RESULT([yes])
+	], [
+		IMPLICIT_FALLTHROUGH=
+		AC_MSG_RESULT([no])
+	])
+
+	CFLAGS="$saved_flags"
+	AC_SUBST([IMPLICIT_FALLTHROUGH])
+])
+
 dnl #
 dnl # Check if gcc supports -fno-omit-frame-pointer option.
 dnl #

config/always-pyzfs.m4

@@ -46,6 +46,21 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
 	])
 	AC_SUBST(DEFINE_PYZFS)
 
+	dnl #
+	dnl # Python "packaging" (or, failing that, "distlib") module is required to build and install pyzfs
+	dnl #
+	AS_IF([test "x$enable_pyzfs" = xcheck -o "x$enable_pyzfs" = xyes], [
+		ZFS_AC_PYTHON_MODULE([packaging], [], [
+			ZFS_AC_PYTHON_MODULE([distlib], [], [
+				AS_IF([test "x$enable_pyzfs" = xyes], [
+					AC_MSG_ERROR("Python $PYTHON_VERSION packaging and distlib modules are not installed")
+				], [test "x$enable_pyzfs" != xno], [
+					enable_pyzfs=no
+				])
+			])
+		])
+	])
+
 	dnl #
 	dnl # Require python-devel libraries
 	dnl #

config/ax_python_devel.m4

@@ -97,9 +97,18 @@ AC_DEFUN([AX_PYTHON_DEVEL],[
 	# Check for a version of Python >= 2.1.0
 	#
 	AC_MSG_CHECKING([for a version of Python >= '2.1.0'])
-	ac_supports_python_ver=`$PYTHON -c "import sys; \
-		ver = sys.version.split ()[[0]]; \
-		print (ver >= '2.1.0')"`
+	ac_supports_python_ver=`cat<<EOD | $PYTHON -
+from __future__ import print_function;
+import sys;
+try:
+	from packaging import version;
+except ImportError:
+	from distlib import version;
+ver = sys.version.split ()[[0]];
+(tst_cmp, tst_ver) = ">= '2.1.0'".split ();
+tst_ver = tst_ver.strip ("'");
+eval ("print (version.LegacyVersion (ver)"+ tst_cmp +"version.LegacyVersion (tst_ver))")
+EOD`
 	if test "$ac_supports_python_ver" != "True"; then
 		if test -z "$PYTHON_NOVERSIONCHECK"; then
 			AC_MSG_RESULT([no])
@@ -126,9 +135,21 @@ to something else than an empty string.
 	#
 	if test -n "$1"; then
 		AC_MSG_CHECKING([for a version of Python $1])
-		ac_supports_python_ver=`$PYTHON -c "import sys; \
-			ver = sys.version.split ()[[0]]; \
-			print (ver $1)"`
+		# Why the strip ()?  Because if we don't, version.parse
+		# will, for example, report 3.10.0 >= '3.11.0'
+		ac_supports_python_ver=`cat<<EOD | $PYTHON -
+
+from __future__ import print_function;
+import sys;
+try:
+	from packaging import version;
+except ImportError:
+	from distlib import version;
+ver = sys.version.split ()[[0]];
+(tst_cmp, tst_ver) = "$1".split ();
+tst_ver = tst_ver.strip ("'");
+eval ("print (version.LegacyVersion (ver)"+ tst_cmp +"version.LegacyVersion (tst_ver))")
+EOD`
 		if test "$ac_supports_python_ver" = "True"; then
 		   AC_MSG_RESULT([yes])
 		else

config/kernel-acl.m4

@@ -162,6 +162,9 @@ dnl #
 dnl # 3.1 API change,
 dnl # Check if inode_operations contains the function get_acl
 dnl #
+dnl # 5.15 API change,
+dnl # Added the bool rcu argument to get_acl for rcu path walk.
+dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_GET_ACL], [
 	ZFS_LINUX_TEST_SRC([inode_operations_get_acl], [
 		#include <linux/fs.h>
@@ -174,14 +177,32 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_GET_ACL], [
 			.get_acl = get_acl_fn,
 		};
 	],[])
+
+	ZFS_LINUX_TEST_SRC([inode_operations_get_acl_rcu], [
+		#include <linux/fs.h>
+
+		struct posix_acl *get_acl_fn(struct inode *inode, int type,
+		    bool rcu) { return NULL; }
+
+		static const struct inode_operations
+		    iops __attribute__ ((unused)) = {
+			.get_acl = get_acl_fn,
+		};
+	],[])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_INODE_OPERATIONS_GET_ACL], [
 	AC_MSG_CHECKING([whether iops->get_acl() exists])
 	ZFS_LINUX_TEST_RESULT([inode_operations_get_acl], [
 		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_GET_ACL, 1, [iops->get_acl() exists])
 	],[
-		ZFS_LINUX_TEST_ERROR([iops->get_acl()])
+		ZFS_LINUX_TEST_RESULT([inode_operations_get_acl_rcu], [
+			AC_MSG_RESULT(yes)
+			AC_DEFINE(HAVE_GET_ACL_RCU, 1, [iops->get_acl() takes rcu])
+		],[
+			ZFS_LINUX_TEST_ERROR([iops->get_acl()])
+		])
 	])
 ])
 

config/kernel-bio.m4

@@ -191,6 +191,24 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BIO_SET_DEV], [
 	], [], [ZFS_META_LICENSE])
 ])
 
+dnl #
+dnl # Linux 5.16 API
+dnl #
+dnl # bio_set_dev is no longer a helper macro and is now an inline function,
+dnl # meaning that the function it calls internally can no longer be overridden
+dnl # by our code
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_BIO_SET_DEV_MACRO], [
+	ZFS_LINUX_TEST_SRC([bio_set_dev_macro], [
+		#include <linux/bio.h>
+		#include <linux/fs.h>
+	],[
+		#ifndef bio_set_dev
+		#error Not a macro
+		#endif
+	], [], [ZFS_META_LICENSE])
+])
+
 AC_DEFUN([ZFS_AC_KERNEL_BIO_SET_DEV], [
 	AC_MSG_CHECKING([whether bio_set_dev() is available])
 	ZFS_LINUX_TEST_RESULT([bio_set_dev], [
@@ -205,6 +223,15 @@ AC_DEFUN([ZFS_AC_KERNEL_BIO_SET_DEV], [
 			AC_DEFINE(HAVE_BIO_SET_DEV_GPL_ONLY, 1,
 			    [bio_set_dev() GPL-only])
 		])
+
+		AC_MSG_CHECKING([whether bio_set_dev() is a macro])
+		ZFS_LINUX_TEST_RESULT([bio_set_dev_macro], [
+			AC_MSG_RESULT(yes)
+			AC_DEFINE(HAVE_BIO_SET_DEV_MACRO, 1,
+			    [bio_set_dev() is a macro])
+		],[
+			AC_MSG_RESULT(no)
+		])
 	],[
 		AC_MSG_RESULT(no)
 	])
@@ -294,9 +321,8 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BIO_SUBMIT_BIO], [
 	ZFS_LINUX_TEST_SRC([submit_bio], [
 		#include <linux/bio.h>
 	],[
-		blk_qc_t blk_qc;
 		struct bio *bio = NULL;
-		blk_qc = submit_bio(bio);
+		(void) submit_bio(bio);
 	])
 ])
 
@@ -396,6 +422,58 @@ AC_DEFUN([ZFS_AC_KERNEL_BIO_BDEV_DISK], [
 	])
 ])
 
+dnl #
+dnl # Linux 5.16 API
+dnl #
+dnl # The Linux 5.16 API for submit_bio changed the return type to be
+dnl # void instead of int
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_BDEV_SUBMIT_BIO_RETURNS_VOID], [
+	ZFS_LINUX_TEST_SRC([bio_bdev_submit_bio_void], [
+		#include <linux/blkdev.h>
+	],[
+		struct block_device_operations *bdev = NULL;
+		__attribute__((unused)) void(*f)(struct bio *) = bdev->submit_bio;
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_BDEV_SUBMIT_BIO_RETURNS_VOID], [
+	AC_MSG_CHECKING(
+		[whether block_device_operations->submit_bio() returns void])
+	ZFS_LINUX_TEST_RESULT([bio_bdev_submit_bio_void], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_BDEV_SUBMIT_BIO_RETURNS_VOID, 1,
+			[block_device_operations->submit_bio() returns void])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
+
+dnl #
+dnl # Linux 5.16 API
+dnl #
+dnl # The Linux 5.16 API moved struct blkcg_gq into linux/blk-cgroup.h, which
+dnl # has been around since 2015. This test looks for the presence of that
+dnl # header, so that it can be conditionally included where it exists, but
+dnl # still be backward compatible with kernels that pre-date its introduction.
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_CGROUP_HEADER], [
+	ZFS_LINUX_TEST_SRC([blk_cgroup_header], [
+		#include <linux/blk-cgroup.h>
+	], [])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_BLK_CGROUP_HEADER], [
+	AC_MSG_CHECKING([for existence of linux/blk-cgroup.h])
+	ZFS_LINUX_TEST_RESULT([blk_cgroup_header],[
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_LINUX_BLK_CGROUP_HEADER, 1,
+			[linux/blk-cgroup.h exists])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
+
 AC_DEFUN([ZFS_AC_KERNEL_SRC_BIO], [
 	ZFS_AC_KERNEL_SRC_REQ
 	ZFS_AC_KERNEL_SRC_BIO_OPS
@@ -407,6 +485,9 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BIO], [
 	ZFS_AC_KERNEL_SRC_BIO_CURRENT_BIO_LIST
 	ZFS_AC_KERNEL_SRC_BLKG_TRYGET
 	ZFS_AC_KERNEL_SRC_BIO_BDEV_DISK
+	ZFS_AC_KERNEL_SRC_BDEV_SUBMIT_BIO_RETURNS_VOID
+	ZFS_AC_KERNEL_SRC_BIO_SET_DEV_MACRO
+	ZFS_AC_KERNEL_SRC_BLK_CGROUP_HEADER
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_BIO], [
@@ -429,4 +510,6 @@ AC_DEFUN([ZFS_AC_KERNEL_BIO], [
 	ZFS_AC_KERNEL_BIO_CURRENT_BIO_LIST
 	ZFS_AC_KERNEL_BLKG_TRYGET
 	ZFS_AC_KERNEL_BIO_BDEV_DISK
+	ZFS_AC_KERNEL_BDEV_SUBMIT_BIO_RETURNS_VOID
+	ZFS_AC_KERNEL_BLK_CGROUP_HEADER
 ])

config/kernel-blk-queue.m4

@@ -48,7 +48,45 @@ AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_BDI], [
 ])
 
 dnl #
-dnl # 2.6.32 - 4.x API,
+dnl # 5.9: added blk_queue_update_readahead(),
+dnl # 5.15: renamed to disk_update_readahead()
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_UPDATE_READAHEAD], [
+	ZFS_LINUX_TEST_SRC([blk_queue_update_readahead], [
+		#include <linux/blkdev.h>
+	],[
+		struct request_queue q;
+		blk_queue_update_readahead(&q);
+	])
+
+	ZFS_LINUX_TEST_SRC([disk_update_readahead], [
+		#include <linux/blkdev.h>
+	],[
+		struct gendisk disk;
+		disk_update_readahead(&disk);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_UPDATE_READAHEAD], [
+	AC_MSG_CHECKING([whether blk_queue_update_readahead() exists])
+	ZFS_LINUX_TEST_RESULT([blk_queue_update_readahead], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_BLK_QUEUE_UPDATE_READAHEAD, 1,
+		    [blk_queue_update_readahead() exists])
+	],[
+		AC_MSG_CHECKING([whether disk_update_readahead() exists])
+		ZFS_LINUX_TEST_RESULT([disk_update_readahead], [
+			AC_MSG_RESULT(yes)
+			AC_DEFINE(HAVE_DISK_UPDATE_READAHEAD, 1,
+			    [disk_update_readahead() exists])
+		],[
+			AC_MSG_RESULT(no)
+		])
+	])
+])
+
+dnl #
+dnl # 2.6.32 API,
 dnl #   blk_queue_discard()
 dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_DISCARD], [
@@ -280,6 +318,7 @@ AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_MAX_SEGMENTS], [
 AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE], [
 	ZFS_AC_KERNEL_SRC_BLK_QUEUE_PLUG
 	ZFS_AC_KERNEL_SRC_BLK_QUEUE_BDI
+	ZFS_AC_KERNEL_SRC_BLK_QUEUE_UPDATE_READAHEAD
 	ZFS_AC_KERNEL_SRC_BLK_QUEUE_DISCARD
 	ZFS_AC_KERNEL_SRC_BLK_QUEUE_SECURE_ERASE
 	ZFS_AC_KERNEL_SRC_BLK_QUEUE_FLAG_SET
@@ -292,6 +331,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE], [
 AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE], [
 	ZFS_AC_KERNEL_BLK_QUEUE_PLUG
 	ZFS_AC_KERNEL_BLK_QUEUE_BDI
+	ZFS_AC_KERNEL_BLK_QUEUE_UPDATE_READAHEAD
 	ZFS_AC_KERNEL_BLK_QUEUE_DISCARD
 	ZFS_AC_KERNEL_BLK_QUEUE_SECURE_ERASE
 	ZFS_AC_KERNEL_BLK_QUEUE_FLAG_SET

config/kernel-blkdev.m4

@@ -294,6 +294,27 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BDEV_WHOLE], [
 	])
 ])
 
+dnl #
+dnl # 5.13 API change
+dnl # blkdev_get_by_path() no longer handles ERESTARTSYS
+dnl #
+dnl # Unfortunately we're forced to rely solely on the kernel version
+dnl # number in order to determine the expected behavior.  This was an
+dnl # internal change to blkdev_get_by_dev(), see commit a8ed1a0607.
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_GET_ERESTARTSYS], [
+	AC_MSG_CHECKING([whether blkdev_get_by_path() handles ERESTARTSYS])
+	AS_VERSION_COMPARE([$LINUX_VERSION], [5.13.0], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_BLKDEV_GET_ERESTARTSYS, 1,
+			[blkdev_get_by_path() handles ERESTARTSYS])
+	],[
+		AC_MSG_RESULT(no)
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
+
 AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV], [
 	ZFS_AC_KERNEL_SRC_BLKDEV_GET_BY_PATH
 	ZFS_AC_KERNEL_SRC_BLKDEV_PUT
@@ -318,4 +339,5 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV], [
 	ZFS_AC_KERNEL_BLKDEV_CHECK_DISK_CHANGE
 	ZFS_AC_KERNEL_BLKDEV_BDEV_CHECK_MEDIA_CHANGE
 	ZFS_AC_KERNEL_BLKDEV_BDEV_WHOLE
+	ZFS_AC_KERNEL_BLKDEV_GET_ERESTARTSYS
 ])

config/kernel-fpu.m4

@@ -2,6 +2,9 @@ dnl #
 dnl # Handle differences in kernel FPU code.
 dnl #
 dnl # Kernel
+dnl # 5.16:	XCR code put into asm/fpu/xcr.h
+dnl # 		HAVE_KERNEL_FPU_XCR_HEADER
+dnl #
 dnl # 5.0:	Wrappers have been introduced to save/restore the FPU state.
 dnl #		This change was made to the 4.19.38 and 4.14.120 LTS kernels.
 dnl #		HAVE_KERNEL_FPU_INTERNAL
@@ -25,6 +28,18 @@ AC_DEFUN([ZFS_AC_KERNEL_FPU_HEADER], [
 		AC_DEFINE(HAVE_KERNEL_FPU_API_HEADER, 1,
 		    [kernel has asm/fpu/api.h])
 		AC_MSG_RESULT(asm/fpu/api.h)
+		AC_MSG_CHECKING([whether fpu/xcr header is available])
+		ZFS_LINUX_TRY_COMPILE([
+			#include <linux/module.h>
+			#include <asm/fpu/xcr.h>
+		],[
+		],[
+			AC_DEFINE(HAVE_KERNEL_FPU_XCR_HEADER, 1,
+				[kernel has asm/fpu/xcr.h])
+			AC_MSG_RESULT(asm/fpu/xcr.h)
+		],[
+			AC_MSG_RESULT(no asm/fpu/xcr.h)
+		])
 	],[
 		AC_MSG_RESULT(i387.h & xcr.h)
 	])

config/kernel-make-request-fn.m4

@@ -42,6 +42,13 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MAKE_REQUEST_FN], [
 		struct block_device_operations o;
 		o.submit_bio = NULL;
 	])
+
+	ZFS_LINUX_TEST_SRC([blk_alloc_disk], [
+		#include <linux/blkdev.h>
+	],[
+		struct gendisk *disk  __attribute__ ((unused));
+		disk = blk_alloc_disk(NUMA_NO_NODE);
+	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_MAKE_REQUEST_FN], [
@@ -56,6 +63,19 @@ AC_DEFUN([ZFS_AC_KERNEL_MAKE_REQUEST_FN], [
 
 		AC_DEFINE(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS, 1,
 		    [submit_bio is member of struct block_device_operations])
+
+		dnl #
+		dnl # Linux 5.14 API Change:
+		dnl # blk_alloc_queue() + alloc_disk() combo replaced by
+		dnl # a single call to blk_alloc_disk().
+		dnl #
+		AC_MSG_CHECKING([whether blk_alloc_disk() exists])
+		ZFS_LINUX_TEST_RESULT([blk_alloc_disk], [
+			AC_MSG_RESULT(yes)
+			AC_DEFINE([HAVE_BLK_ALLOC_DISK], 1, [blk_alloc_disk() exists])
+		], [
+			AC_MSG_RESULT(no)
+		])
 	],[
 		AC_MSG_RESULT(no)
 

config/kernel-pagemap-folio_wait_bit.m4

@@ -0,0 +1,26 @@
+dnl #
+dnl # Linux 5.16 no longer allows directly calling wait_on_page_bit, and
+dnl # instead requires you to call folio-specific functions. In this case,
+dnl # wait_on_page_bit(pg, PG_writeback) becomes
+dnl # folio_wait_bit(pg, PG_writeback)
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_PAGEMAP_FOLIO_WAIT_BIT], [
+	ZFS_LINUX_TEST_SRC([pagemap_has_folio_wait_bit], [
+		#include <linux/pagemap.h>
+	],[
+		static struct folio *f = NULL;
+
+		folio_wait_bit(f, PG_writeback);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_PAGEMAP_FOLIO_WAIT_BIT], [
+	AC_MSG_CHECKING([folio_wait_bit() exists])
+	ZFS_LINUX_TEST_RESULT([pagemap_has_folio_wait_bit], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE(HAVE_PAGEMAP_FOLIO_WAIT_BIT, 1,
+			[folio_wait_bit() exists])
+	],[
+		AC_MSG_RESULT([no])
+	])
+])

config/kernel-siginfo.m4

@@ -0,0 +1,21 @@
+dnl #
+dnl # 4.20 API change
+dnl # Added kernel_siginfo_t
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_SIGINFO], [
+	ZFS_LINUX_TEST_SRC([siginfo], [
+		#include <linux/signal_types.h>
+	],[
+		kernel_siginfo_t info __attribute__ ((unused));
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SIGINFO], [
+	AC_MSG_CHECKING([whether kernel_siginfo_t tyepedef exists])
+	ZFS_LINUX_TEST_RESULT([siginfo], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_SIGINFO, 1, [kernel_siginfo_t exists])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])

config/kernel-signal-stop.m4

@@ -0,0 +1,21 @@
+dnl #
+dnl # 4.4 API change
+dnl # Added kernel_signal_stop
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_SIGNAL_STOP], [
+	ZFS_LINUX_TEST_SRC([signal_stop], [
+		#include <linux/sched/signal.h>
+	],[
+		kernel_signal_stop();
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SIGNAL_STOP], [
+	AC_MSG_CHECKING([whether signal_stop() exists])
+	ZFS_LINUX_TEST_RESULT([signal_stop], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_SIGNAL_STOP, 1, [signal_stop() exists])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])

config/kernel-special-state.m4

@@ -0,0 +1,21 @@
+dnl #
+dnl # 4.17 API change
+dnl # Added set_special_state() function
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_SET_SPECIAL_STATE], [
+	ZFS_LINUX_TEST_SRC([set_special_state], [
+		#include <linux/sched.h>
+	],[
+		set_special_state(TASK_STOPPED);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SET_SPECIAL_STATE], [
+	AC_MSG_CHECKING([whether set_special_state() exists])
+	ZFS_LINUX_TEST_RESULT([set_special_state], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_SET_SPECIAL_STATE, 1, [set_special_state() exists])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])

config/kernel-stdarg.m4

@@ -0,0 +1,32 @@
+dnl #
+dnl # Linux 5.15 gets rid of -isystem and external <stdarg.h> inclusion
+dnl # and ships its own <linux/stdarg.h>. Check if this header file does
+dnl # exist and provide all necessary definitions for variable argument
+dnl # functions. Adjust the inclusion of <stdarg.h> according to the
+dnl # results.
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_STANDALONE_LINUX_STDARG], [
+	ZFS_LINUX_TEST_SRC([has_standalone_linux_stdarg], [
+		#include <linux/stdarg.h>
+
+		#if !defined(va_start) || !defined(va_end) || \
+		    !defined(va_arg) || !defined(va_copy)
+		#error "<linux/stdarg.h> is invalid"
+		#endif
+	],[])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_STANDALONE_LINUX_STDARG], [
+	dnl #
+	dnl # Linux 5.15 ships its own stdarg.h and doesn't allow to
+	dnl # include compiler headers.
+	dnl #
+	AC_MSG_CHECKING([whether standalone <linux/stdarg.h> exists])
+	ZFS_LINUX_TEST_RESULT([has_standalone_linux_stdarg], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE(HAVE_STANDALONE_LINUX_STDARG, 1,
+			[standalone <linux/stdarg.h> exists])
+	],[
+		AC_MSG_RESULT([no])
+	])
+])

config/kernel-vfs-iov_iter.m4

@@ -99,6 +99,14 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_IOV_ITER], [
 
 		bytes = copy_from_iter((void *)&buf, size, &iter);
 	])
+
+	ZFS_LINUX_TEST_SRC([iov_iter_type], [
+		#include <linux/fs.h>
+		#include <linux/uio.h>
+	],[
+		struct iov_iter iter = { 0 };
+		__attribute__((unused)) enum iter_type i = iov_iter_type(&iter);
+	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_VFS_IOV_ITER], [
@@ -193,6 +201,20 @@ AC_DEFUN([ZFS_AC_KERNEL_VFS_IOV_ITER], [
 		enable_vfs_iov_iter="no"
 	])
 
+	dnl #
+	dnl # This checks for iov_iter_type() in linux/uio.h. It is not
+	dnl # required, however, and the module will compiled without it
+	dnl # using direct access of the member attribute
+	dnl #
+	AC_MSG_CHECKING([whether iov_iter_type() is available])
+	ZFS_LINUX_TEST_RESULT([iov_iter_type], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_IOV_ITER_TYPE, 1,
+		    [iov_iter_type() is available])
+	],[
+		AC_MSG_RESULT(no)
+	])
+
 	dnl #
 	dnl # As of the 4.9 kernel support is provided for iovecs, kvecs,
 	dnl # bvecs and pipes in the iov_iter structure.  As long as the

config/kernel-vfs-set_page_dirty.m4

@@ -0,0 +1,34 @@
+dnl #
+dnl # Linux 5.14 adds a change to require set_page_dirty to be manually
+dnl # wired up in struct address_space_operations. Determine if this needs
+dnl # to be done. This patch set also introduced __set_page_dirty_nobuffers
+dnl # declaration in linux/pagemap.h, so these tests look for the presence
+dnl # of that function to tell the compiler to assign set_page_dirty in
+dnl # module/os/linux/zfs/zpl_file.c
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_SET_PAGE_DIRTY_NOBUFFERS], [
+	ZFS_LINUX_TEST_SRC([vfs_has_set_page_dirty_nobuffers], [
+		#include <linux/pagemap.h>
+		#include <linux/fs.h>
+
+		static const struct address_space_operations
+		    aops __attribute__ ((unused)) = {
+			.set_page_dirty = __set_page_dirty_nobuffers,
+		};
+	],[])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_VFS_SET_PAGE_DIRTY_NOBUFFERS], [
+	dnl #
+	dnl # Linux 5.14 change requires set_page_dirty() to be assigned
+	dnl # in address_space_operations()
+	dnl #
+	AC_MSG_CHECKING([__set_page_dirty_nobuffers exists])
+	ZFS_LINUX_TEST_RESULT([vfs_has_set_page_dirty_nobuffers], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE(HAVE_VFS_SET_PAGE_DIRTY_NOBUFFERS, 1,
+			[__set_page_dirty_nobuffers exists])
+	],[
+		AC_MSG_RESULT([no])
+	])
+])

config/kernel.m4

@@ -128,6 +128,12 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [
 	ZFS_AC_KERNEL_SRC_MKNOD
 	ZFS_AC_KERNEL_SRC_SYMLINK
 	ZFS_AC_KERNEL_SRC_BIO_MAX_SEGS
+	ZFS_AC_KERNEL_SRC_SIGNAL_STOP
+	ZFS_AC_KERNEL_SRC_SIGINFO
+	ZFS_AC_KERNEL_SRC_SET_SPECIAL_STATE
+	ZFS_AC_KERNEL_SRC_VFS_SET_PAGE_DIRTY_NOBUFFERS
+	ZFS_AC_KERNEL_SRC_STANDALONE_LINUX_STDARG
+	ZFS_AC_KERNEL_SRC_PAGEMAP_FOLIO_WAIT_BIT
 
 	AC_MSG_CHECKING([for available kernel interfaces])
 	ZFS_LINUX_TEST_COMPILE_ALL([kabi])
@@ -229,6 +235,12 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [
 	ZFS_AC_KERNEL_MKNOD
 	ZFS_AC_KERNEL_SYMLINK
 	ZFS_AC_KERNEL_BIO_MAX_SEGS
+	ZFS_AC_KERNEL_SIGNAL_STOP
+	ZFS_AC_KERNEL_SIGINFO
+	ZFS_AC_KERNEL_SET_SPECIAL_STATE
+	ZFS_AC_KERNEL_VFS_SET_PAGE_DIRTY_NOBUFFERS
+	ZFS_AC_KERNEL_STANDALONE_LINUX_STDARG
+	ZFS_AC_KERNEL_PAGEMAP_FOLIO_WAIT_BIT
 ])
 
 dnl #

config/zfs-build.m4

@@ -158,6 +158,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [
 
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_UNUSED_BUT_SET_VARIABLE
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_BOOL_COMPARE
+	ZFS_AC_CONFIG_ALWAYS_CC_IMPLICIT_FALLTHROUGH
 	ZFS_AC_CONFIG_ALWAYS_CC_FRAME_LARGER_THAN
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_TRUNCATION
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_ZERO_LENGTH

configure.ac

@@ -221,6 +221,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/cmd/mktree/Makefile
 	tests/zfs-tests/cmd/mmap_exec/Makefile
 	tests/zfs-tests/cmd/mmap_libaio/Makefile
+	tests/zfs-tests/cmd/mmap_seek/Makefile
 	tests/zfs-tests/cmd/mmapwrite/Makefile
 	tests/zfs-tests/cmd/nvlist_to_lua/Makefile
 	tests/zfs-tests/cmd/randfree_file/Makefile

contrib/pam_zfs_key/pam_zfs_key.c

@@ -82,7 +82,11 @@ alloc_pw_size(size_t len)
 		return (NULL);
 	}
 	pw->len = len;
-	pw->value = malloc(len);
+	/*
+	 * The use of malloc() triggers a spurious gcc 11 -Wmaybe-uninitialized
+	 * warning in the mlock() function call below, so use calloc().
+	 */
+	pw->value = calloc(len, 1);
 	if (!pw->value) {
 		free(pw);
 		return (NULL);
@@ -99,7 +103,11 @@ alloc_pw_string(const char *source)
 		return (NULL);
 	}
 	pw->len = strlen(source) + 1;
-	pw->value = malloc(pw->len);
+	/*
+	 * The use of malloc() triggers a spurious gcc 11 -Wmaybe-uninitialized
+	 * warning in the mlock() function call below, so use calloc().
+	 */
+	pw->value = calloc(pw->len, 1);
 	if (!pw->value) {
 		free(pw);
 		return (NULL);

include/libzutil.h

@@ -159,6 +159,16 @@ void color_start(char *color);
 void color_end(void);
 int printf_color(char *color, char *format, ...);
 
+/*
+ * These functions are used by the ZFS libraries and cmd/zpool code, but are
+ * not exported in the ABI.
+ */
+typedef int (*pool_vdev_iter_f)(void *, nvlist_t *, void *);
+int for_each_vdev_cb(void *zhp, nvlist_t *nv, pool_vdev_iter_f func,
+    void *data);
+int for_each_vdev_in_nvlist(nvlist_t *nvroot, pool_vdev_iter_f func,
+    void *data);
+void update_vdevs_config_dev_sysfs_path(nvlist_t *config);
 #ifdef	__cplusplus
 }
 #endif

include/os/freebsd/linux/compiler.h

@@ -67,6 +67,7 @@
 #define	__always_inline			inline
 #define	noinline			__noinline
 #define	____cacheline_aligned		__aligned(CACHE_LINE_SIZE)
+#define	fallthrough			__attribute__((__fallthrough__))
 
 #if !defined(_KERNEL) && !defined(_STANDALONE)
 #define	likely(x)			__builtin_expect(!!(x), 1)

include/os/freebsd/spl/sys/uio.h

@@ -35,81 +35,47 @@
 #include <sys/_uio.h>
 #include <sys/debug.h>
 
-
-
-#define	uio_loffset	uio_offset
-
-typedef	struct uio	uio_t;
 typedef	struct iovec	iovec_t;
-typedef	enum uio_seg	uio_seg_t;
+typedef	enum uio_seg	zfs_uio_seg_t;
+typedef	enum uio_rw	zfs_uio_rw_t;
 
-typedef enum xuio_type {
-	UIOTYPE_ASYNCIO,
-	UIOTYPE_ZEROCOPY
-} xuio_type_t;
+typedef struct zfs_uio {
+	struct uio	*uio;
+} zfs_uio_t;
 
-typedef struct xuio {
-	uio_t	xu_uio;
-
-	/* Extended uio fields */
-	enum xuio_type xu_type; /* What kind of uio structure? */
-	union {
-		struct {
-			int xu_zc_rw;
-			void *xu_zc_priv;
-		} xu_zc;
-	} xu_ext;
-} xuio_t;
-
-#define	XUIO_XUZC_PRIV(xuio)	xuio->xu_ext.xu_zc.xu_zc_priv
-#define	XUIO_XUZC_RW(xuio)	xuio->xu_ext.xu_zc.xu_zc_rw
-
-static __inline int
-zfs_uiomove(void *cp, size_t n, enum uio_rw dir, uio_t *uio)
-{
-
-	ASSERT(uio->uio_rw == dir);
-	return (uiomove(cp, (int)n, uio));
-}
-#define	uiomove(cp, n, dir, uio)	zfs_uiomove((cp), (n), (dir), (uio))
-
-int uiocopy(void *p, size_t n, enum uio_rw rw, struct uio *uio, size_t *cbytes);
-void uioskip(uio_t *uiop, size_t n);
-
-#define	uio_segflg(uio)			(uio)->uio_segflg
-#define	uio_offset(uio)			(uio)->uio_loffset
-#define	uio_resid(uio)			(uio)->uio_resid
-#define	uio_iovcnt(uio)			(uio)->uio_iovcnt
-#define	uio_iovlen(uio, idx)		(uio)->uio_iov[(idx)].iov_len
-#define	uio_iovbase(uio, idx)		(uio)->uio_iov[(idx)].iov_base
-#define	uio_fault_disable(uio, set)
+#define	GET_UIO_STRUCT(u)	(u)->uio
+#define	zfs_uio_segflg(u)	GET_UIO_STRUCT(u)->uio_segflg
+#define	zfs_uio_offset(u)	GET_UIO_STRUCT(u)->uio_offset
+#define	zfs_uio_resid(u)	GET_UIO_STRUCT(u)->uio_resid
+#define	zfs_uio_iovcnt(u)	GET_UIO_STRUCT(u)->uio_iovcnt
+#define	zfs_uio_iovlen(u, idx)	GET_UIO_STRUCT(u)->uio_iov[(idx)].iov_len
+#define	zfs_uio_iovbase(u, idx)	GET_UIO_STRUCT(u)->uio_iov[(idx)].iov_base
+#define	zfs_uio_td(u)		GET_UIO_STRUCT(u)->uio_td
+#define	zfs_uio_rw(u)		GET_UIO_STRUCT(u)->uio_rw
+#define	zfs_uio_fault_disable(u, set)
+#define	zfs_uio_prefaultpages(size, u)	(0)
 
 static inline void
-uio_iov_at_index(uio_t *uio, uint_t idx, void **base, uint64_t *len)
+zfs_uio_setoffset(zfs_uio_t *uio, offset_t off)
 {
-	*base = uio_iovbase(uio, idx);
-	*len = uio_iovlen(uio, idx);
+	zfs_uio_offset(uio) = off;
 }
 
 static inline void
-uio_advance(uio_t *uio, size_t size)
+zfs_uio_advance(zfs_uio_t *uio, size_t size)
 {
-	uio->uio_resid -= size;
-	uio->uio_loffset += size;
+	zfs_uio_resid(uio) -= size;
+	zfs_uio_offset(uio) += size;
 }
 
-static inline offset_t
-uio_index_at_offset(uio_t *uio, offset_t off, uint_t *vec_idx)
+static __inline void
+zfs_uio_init(zfs_uio_t *uio, struct uio *uio_s)
 {
-	*vec_idx = 0;
-	while (*vec_idx < uio_iovcnt(uio) && off >= uio_iovlen(uio, *vec_idx)) {
-		off -= uio_iovlen(uio, *vec_idx);
-		(*vec_idx)++;
-	}
-
-	return (off);
+	GET_UIO_STRUCT(uio) = uio_s;
 }
 
+int zfs_uio_fault_move(void *p, size_t n, zfs_uio_rw_t dir, zfs_uio_t *uio);
+
 #endif /* !_STANDALONE */
 
 #endif	/* !_OPENSOLARIS_SYS_UIO_H_ */

include/os/freebsd/spl/sys/vnode.h

@@ -59,6 +59,8 @@ enum symfollow { NO_FOLLOW = NOFOLLOW };
 #include <sys/file.h>
 #include <sys/filedesc.h>
 #include <sys/syscallsubr.h>
+#include <sys/vm.h>
+#include <vm/vm_object.h>
 
 typedef	struct vop_vector	vnodeops_t;
 #define	VOP_FID		VOP_VPTOFH
@@ -88,6 +90,22 @@ vn_is_readonly(vnode_t *vp)
 #define	vn_has_cached_data(vp)	\
 	((vp)->v_object != NULL && \
 	(vp)->v_object->resident_page_count > 0)
+
+static __inline void
+vn_flush_cached_data(vnode_t *vp, boolean_t sync)
+{
+#if __FreeBSD_version > 1300054
+	if (vm_object_mightbedirty(vp->v_object)) {
+#else
+	if (vp->v_object->flags & OBJ_MIGHTBEDIRTY) {
+#endif
+		int flags = sync ? OBJPC_SYNC : 0;
+		zfs_vmobject_wlock(vp->v_object);
+		vm_object_page_clean(vp->v_object, 0, 0, flags);
+		zfs_vmobject_wunlock(vp->v_object);
+	}
+}
+
 #define	vn_exists(vp)		do { } while (0)
 #define	vn_invalid(vp)		do { } while (0)
 #define	vn_renamepath(tdvp, svp, tnm, lentnm)	do { } while (0)

include/os/freebsd/zfs/sys/freebsd_crypto.h

@@ -92,7 +92,7 @@ int freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
 void freebsd_crypt_freesession(freebsd_crypt_session_t *sessp);
 
 int freebsd_crypt_uio(boolean_t, freebsd_crypt_session_t *,
-	struct zio_crypt_info *, uio_t *, crypto_key_t *, uint8_t *,
+	struct zio_crypt_info *, zfs_uio_t *, crypto_key_t *, uint8_t *,
 	size_t, size_t);
 
 #endif /* _ZFS_FREEBSD_CRYPTO_H */

include/os/freebsd/zfs/sys/zfs_context_os.h

@@ -42,7 +42,6 @@
 #include <linux/types.h>
 
 #define	cond_resched()		kern_yield(PRI_USER)
-#define	uio_prefaultpages(size, uio) (0)
 
 #define	taskq_create_sysdc(a, b, d, e, p, dc, f) \
 	    (taskq_create(a, b, maxclsyspri, d, e, f))

include/os/freebsd/zfs/sys/zfs_znode_impl.h

@@ -40,6 +40,7 @@
 #include <sys/zil.h>
 #include <sys/zfs_project.h>
 #include <vm/vm_object.h>
+#include <sys/uio.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -116,8 +117,10 @@ extern minor_t zfsdev_minor_alloc(void);
 #define	Z_ISLNK(type) ((type) == VLNK)
 #define	Z_ISDIR(type) ((type) == VDIR)
 
-#define	zn_has_cached_data(zp)	vn_has_cached_data(ZTOV(zp))
-#define	zn_rlimit_fsize(zp, uio, td)	vn_rlimit_fsize(ZTOV(zp), (uio), (td))
+#define	zn_has_cached_data(zp)		vn_has_cached_data(ZTOV(zp))
+#define	zn_flush_cached_data(zp, sync)	vn_flush_cached_data(ZTOV(zp), sync)
+#define	zn_rlimit_fsize(zp, uio) \
+    vn_rlimit_fsize(ZTOV(zp), GET_UIO_STRUCT(uio), zfs_uio_td(uio))
 
 /* Called on entry to each ZFS vnode and vfs operation  */
 #define	ZFS_ENTER(zfsvfs) \

include/os/linux/kernel/linux/blkdev_compat.h

@@ -30,9 +30,9 @@
 #define	_ZFS_BLKDEV_H
 
 #include <linux/blkdev.h>
-#include <linux/elevator.h>
 #include <linux/backing-dev.h>
 #include <linux/hdreg.h>
+#include <linux/major.h>
 #include <linux/msdos_fs.h>	/* for SECTOR_* */
 
 #ifndef HAVE_BLK_QUEUE_FLAG_SET
@@ -92,11 +92,14 @@ blk_queue_set_write_cache(struct request_queue *q, bool wc, bool fua)
 static inline void
 blk_queue_set_read_ahead(struct request_queue *q, unsigned long ra_pages)
 {
+#if !defined(HAVE_BLK_QUEUE_UPDATE_READAHEAD) && \
+	!defined(HAVE_DISK_UPDATE_READAHEAD)
 #ifdef HAVE_BLK_QUEUE_BDI_DYNAMIC
 	q->backing_dev_info->ra_pages = ra_pages;
 #else
 	q->backing_dev_info.ra_pages = ra_pages;
 #endif
+#endif
 }
 
 #ifdef HAVE_BIO_BVEC_ITER

include/os/linux/kernel/linux/compiler_compat.h

@@ -28,6 +28,14 @@
 
 #include <linux/compiler.h>
 
+#if !defined(fallthrough)
+#if defined(HAVE_IMPLICIT_FALLTHROUGH)
+#define	fallthrough		__attribute__((__fallthrough__))
+#else
+#define	fallthrough		((void)0)
+#endif
+#endif
+
 #if !defined(READ_ONCE)
 #define	READ_ONCE(x)		ACCESS_ONCE(x)
 #endif

include/os/linux/kernel/linux/simd_x86.h

@@ -88,6 +88,9 @@
 #if defined(HAVE_KERNEL_FPU_API_HEADER)
 #include <asm/fpu/api.h>
 #include <asm/fpu/internal.h>
+#if defined(HAVE_KERNEL_FPU_XCR_HEADER)
+#include <asm/fpu/xcr.h>
+#endif
 #else
 #include <asm/i387.h>
 #include <asm/xcr.h>

include/os/linux/spl/sys/cmn_err.h

@@ -24,7 +24,11 @@
 #ifndef _SPL_CMN_ERR_H
 #define	_SPL_CMN_ERR_H
 
+#if defined(_KERNEL) && defined(HAVE_STANDALONE_LINUX_STDARG)
+#include <linux/stdarg.h>
+#else
 #include <stdarg.h>
+#endif
 
 #define	CE_CONT		0 /* continuation */
 #define	CE_NOTE		1 /* notice */

include/os/linux/spl/sys/signal.h

@@ -33,22 +33,6 @@
 #define	FORREAL		0	/* Usual side-effects */
 #define	JUSTLOOKING	1	/* Don't stop the process */
 
-/*
- * The "why" argument indicates the allowable side-effects of the call:
- *
- * FORREAL:  Extract the next pending signal from p_sig into p_cursig;
- * stop the process if a stop has been requested or if a traced signal
- * is pending.
- *
- * JUSTLOOKING:  Don't stop the process, just indicate whether or not
- * a signal might be pending (FORREAL is needed to tell for sure).
- */
-static __inline__ int
-issig(int why)
-{
-	ASSERT(why == FORREAL || why == JUSTLOOKING);
-
-	return (signal_pending(current));
-}
+extern int issig(int why);
 
 #endif /* SPL_SIGNAL_H */

include/os/linux/spl/sys/thread.h

@@ -70,4 +70,17 @@ extern struct task_struct *spl_kthread_create(int (*func)(void *),
 
 extern proc_t p0;
 
+#ifdef HAVE_SIGINFO
+typedef kernel_siginfo_t spl_kernel_siginfo_t;
+#else
+typedef siginfo_t spl_kernel_siginfo_t;
+#endif
+
+#ifdef HAVE_SET_SPECIAL_STATE
+#define	spl_set_special_state(x) set_special_state((x))
+#else
+#define	spl_set_special_state(x) __set_current_state((x))
+#endif
+
+
 #endif  /* _SPL_THREAD_H */

include/os/linux/spl/sys/uio.h

@@ -36,21 +36,21 @@
 
 typedef struct iovec iovec_t;
 
-typedef enum uio_rw {
+typedef enum zfs_uio_rw {
 	UIO_READ =		0,
 	UIO_WRITE =		1,
-} uio_rw_t;
+} zfs_uio_rw_t;
 
-typedef enum uio_seg {
+typedef enum zfs_uio_seg {
 	UIO_USERSPACE =		0,
 	UIO_SYSSPACE =		1,
 	UIO_BVEC =		2,
 #if defined(HAVE_VFS_IOV_ITER)
 	UIO_ITER =		3,
 #endif
-} uio_seg_t;
+} zfs_uio_seg_t;
 
-typedef struct uio {
+typedef struct zfs_uio {
 	union {
 		const struct iovec	*uio_iov;
 		const struct bio_vec	*uio_bvec;
@@ -60,91 +60,39 @@ typedef struct uio {
 	};
 	int		uio_iovcnt;
 	offset_t	uio_loffset;
-	uio_seg_t	uio_segflg;
+	zfs_uio_seg_t	uio_segflg;
 	boolean_t	uio_fault_disable;
 	uint16_t	uio_fmode;
 	uint16_t	uio_extflg;
 	ssize_t		uio_resid;
 	size_t		uio_skip;
-} uio_t;
+} zfs_uio_t;
 
-typedef struct aio_req {
-	uio_t		*aio_uio;
-	void		*aio_private;
-} aio_req_t;
+#define	zfs_uio_segflg(u)		(u)->uio_segflg
+#define	zfs_uio_offset(u)		(u)->uio_loffset
+#define	zfs_uio_resid(u)		(u)->uio_resid
+#define	zfs_uio_iovcnt(u)		(u)->uio_iovcnt
+#define	zfs_uio_iovlen(u, idx)		(u)->uio_iov[(idx)].iov_len
+#define	zfs_uio_iovbase(u, idx)		(u)->uio_iov[(idx)].iov_base
+#define	zfs_uio_fault_disable(u, set)	(u)->uio_fault_disable = set
+#define	zfs_uio_rlimit_fsize(z, u)	(0)
+#define	zfs_uio_fault_move(p, n, rw, u)	zfs_uiomove((p), (n), (rw), (u))
 
-typedef enum xuio_type {
-	UIOTYPE_ASYNCIO,
-	UIOTYPE_ZEROCOPY,
-} xuio_type_t;
-
-
-#define	UIOA_IOV_MAX    16
-
-typedef struct uioa_page_s {
-	int	uioa_pfncnt;
-	void	**uioa_ppp;
-	caddr_t	uioa_base;
-	size_t	uioa_len;
-} uioa_page_t;
-
-typedef struct xuio {
-	uio_t xu_uio;
-	enum xuio_type xu_type;
-	union {
-		struct {
-			uint32_t xu_a_state;
-			ssize_t xu_a_mbytes;
-			uioa_page_t *xu_a_lcur;
-			void **xu_a_lppp;
-			void *xu_a_hwst[4];
-			uioa_page_t xu_a_locked[UIOA_IOV_MAX];
-		} xu_aio;
-
-		struct {
-			int xu_zc_rw;
-			void *xu_zc_priv;
-		} xu_zc;
-	} xu_ext;
-} xuio_t;
-
-#define	XUIO_XUZC_PRIV(xuio)	xuio->xu_ext.xu_zc.xu_zc_priv
-#define	XUIO_XUZC_RW(xuio)	xuio->xu_ext.xu_zc.xu_zc_rw
-
-#define	uio_segflg(uio)			(uio)->uio_segflg
-#define	uio_offset(uio)			(uio)->uio_loffset
-#define	uio_resid(uio)			(uio)->uio_resid
-#define	uio_iovcnt(uio)			(uio)->uio_iovcnt
-#define	uio_iovlen(uio, idx)		(uio)->uio_iov[(idx)].iov_len
-#define	uio_iovbase(uio, idx)		(uio)->uio_iov[(idx)].iov_base
-#define	uio_fault_disable(uio, set)	(uio)->uio_fault_disable = set
+extern int zfs_uio_prefaultpages(ssize_t, zfs_uio_t *);
 
 static inline void
-uio_iov_at_index(uio_t *uio, uint_t idx, void **base, uint64_t *len)
+zfs_uio_setoffset(zfs_uio_t *uio, offset_t off)
 {
-	*base = uio_iovbase(uio, idx);
-	*len = uio_iovlen(uio, idx);
+	uio->uio_loffset = off;
 }
 
 static inline void
-uio_advance(uio_t *uio, size_t size)
+zfs_uio_advance(zfs_uio_t *uio, size_t size)
 {
 	uio->uio_resid -= size;
 	uio->uio_loffset += size;
 }
 
-static inline offset_t
-uio_index_at_offset(uio_t *uio, offset_t off, uint_t *vec_idx)
-{
-	*vec_idx = 0;
-	while (*vec_idx < uio_iovcnt(uio) && off >= uio_iovlen(uio, *vec_idx)) {
-		off -= uio_iovlen(uio, *vec_idx);
-		(*vec_idx)++;
-	}
-
-	return (off);
-}
-
 static inline void
 iov_iter_init_compat(struct iov_iter *iter, unsigned int dir,
     const struct iovec *iov, unsigned long nr_segs, size_t count)
@@ -159,8 +107,9 @@ iov_iter_init_compat(struct iov_iter *iter, unsigned int dir,
 }
 
 static inline void
-uio_iovec_init(uio_t *uio, const struct iovec *iov, unsigned long nr_segs,
-    offset_t offset, uio_seg_t seg, ssize_t resid, size_t skip)
+zfs_uio_iovec_init(zfs_uio_t *uio, const struct iovec *iov,
+    unsigned long nr_segs, offset_t offset, zfs_uio_seg_t seg, ssize_t resid,
+    size_t skip)
 {
 	ASSERT(seg == UIO_USERSPACE || seg == UIO_SYSSPACE);
 
@@ -176,7 +125,7 @@ uio_iovec_init(uio_t *uio, const struct iovec *iov, unsigned long nr_segs,
 }
 
 static inline void
-uio_bvec_init(uio_t *uio, struct bio *bio)
+zfs_uio_bvec_init(zfs_uio_t *uio, struct bio *bio)
 {
 	uio->uio_bvec = &bio->bi_io_vec[BIO_BI_IDX(bio)];
 	uio->uio_iovcnt = bio->bi_vcnt - BIO_BI_IDX(bio);
@@ -191,7 +140,7 @@ uio_bvec_init(uio_t *uio, struct bio *bio)
 
 #if defined(HAVE_VFS_IOV_ITER)
 static inline void
-uio_iov_iter_init(uio_t *uio, struct iov_iter *iter, offset_t offset,
+zfs_uio_iov_iter_init(zfs_uio_t *uio, struct iov_iter *iter, offset_t offset,
     ssize_t resid, size_t skip)
 {
 	uio->uio_iter = iter;

include/os/linux/zfs/sys/zfs_context_os.h

@@ -23,8 +23,8 @@
 #ifndef ZFS_CONTEXT_OS_H
 #define	ZFS_CONTEXT_OS_H
 
-#include <sys/uio_impl.h>
 #include <linux/dcache_compat.h>
 #include <linux/utsname_compat.h>
+#include <linux/compiler_compat.h>
 
 #endif

include/os/linux/zfs/sys/zfs_vnops_os.h

@@ -61,7 +61,7 @@ extern int zfs_rename(znode_t *sdzp, char *snm, znode_t *tdzp,
     char *tnm, cred_t *cr, int flags);
 extern int zfs_symlink(znode_t *dzp, char *name, vattr_t *vap,
     char *link, znode_t **zpp, cred_t *cr, int flags);
-extern int zfs_readlink(struct inode *ip, uio_t *uio, cred_t *cr);
+extern int zfs_readlink(struct inode *ip, zfs_uio_t *uio, cred_t *cr);
 extern int zfs_link(znode_t *tdzp, znode_t *szp,
     char *name, cred_t *cr, int flags);
 extern void zfs_inactive(struct inode *ip);

include/os/linux/zfs/sys/zfs_znode_impl.h

@@ -70,8 +70,9 @@ extern "C" {
 #define	Z_ISDEV(type)	(S_ISCHR(type) || S_ISBLK(type) || S_ISFIFO(type))
 #define	Z_ISDIR(type)	S_ISDIR(type)
 
-#define	zn_has_cached_data(zp)	((zp)->z_is_mapped)
-#define	zn_rlimit_fsize(zp, uio, td)	(0)
+#define	zn_has_cached_data(zp)		((zp)->z_is_mapped)
+#define	zn_flush_cached_data(zp, sync)	write_inode_now(ZTOI(zp), sync)
+#define	zn_rlimit_fsize(zp, uio)	(0)
 
 /*
  * zhold() wraps igrab() on Linux, and igrab() may fail when the

include/os/linux/zfs/sys/zpl.h

@@ -70,7 +70,11 @@ extern int zpl_set_acl(struct user_namespace *userns, struct inode *ip,
 extern int zpl_set_acl(struct inode *ip, struct posix_acl *acl, int type);
 #endif /* HAVE_SET_ACL_USERNS */
 #endif /* HAVE_SET_ACL */
+#if defined(HAVE_GET_ACL_RCU)
+extern struct posix_acl *zpl_get_acl(struct inode *ip, int type, bool rcu);
+#elif defined(HAVE_GET_ACL)
 extern struct posix_acl *zpl_get_acl(struct inode *ip, int type);
+#endif
 extern int zpl_init_acl(struct inode *ip, struct inode *dir);
 extern int zpl_chmod_acl(struct inode *ip);
 #else

include/sys/crypto/api.h

@@ -58,7 +58,7 @@ typedef struct {
  */
 
 #define	CRYPTO_MECH_INVALID	((uint64_t)-1)
-extern crypto_mech_type_t crypto_mech2id(crypto_mech_name_t name);
+extern crypto_mech_type_t crypto_mech2id(char *name);
 
 /*
  * Create and destroy context templates.

include/sys/crypto/common.h

@@ -244,7 +244,7 @@ typedef struct crypto_data {
 		iovec_t cdu_raw;		/* Pointer and length	    */
 
 		/* uio scatter-gather format */
-		uio_t	*cdu_uio;
+		zfs_uio_t	*cdu_uio;
 
 	} cdu;	/* Crypto Data Union */
 } crypto_data_t;

include/sys/dmu.h

@@ -847,14 +847,14 @@ void dmu_write_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size,
 void dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
 	dmu_tx_t *tx);
 #ifdef _KERNEL
-int dmu_read_uio(objset_t *os, uint64_t object, struct uio *uio, uint64_t size);
-int dmu_read_uio_dbuf(dmu_buf_t *zdb, struct uio *uio, uint64_t size);
-int dmu_read_uio_dnode(dnode_t *dn, struct uio *uio, uint64_t size);
-int dmu_write_uio(objset_t *os, uint64_t object, struct uio *uio, uint64_t size,
+int dmu_read_uio(objset_t *os, uint64_t object, zfs_uio_t *uio, uint64_t size);
+int dmu_read_uio_dbuf(dmu_buf_t *zdb, zfs_uio_t *uio, uint64_t size);
+int dmu_read_uio_dnode(dnode_t *dn, zfs_uio_t *uio, uint64_t size);
+int dmu_write_uio(objset_t *os, uint64_t object, zfs_uio_t *uio, uint64_t size,
 	dmu_tx_t *tx);
-int dmu_write_uio_dbuf(dmu_buf_t *zdb, struct uio *uio, uint64_t size,
+int dmu_write_uio_dbuf(dmu_buf_t *zdb, zfs_uio_t *uio, uint64_t size,
 	dmu_tx_t *tx);
-int dmu_write_uio_dnode(dnode_t *dn, struct uio *uio, uint64_t size,
+int dmu_write_uio_dnode(dnode_t *dn, zfs_uio_t *uio, uint64_t size,
 	dmu_tx_t *tx);
 #endif
 struct arc_buf *dmu_request_arcbuf(dmu_buf_t *handle, int size);
@@ -864,18 +864,6 @@ int dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset,
 int dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset,
     struct arc_buf *buf, dmu_tx_t *tx);
 #define	dmu_assign_arcbuf	dmu_assign_arcbuf_by_dbuf
-#ifdef HAVE_UIO_ZEROCOPY
-int dmu_xuio_init(struct xuio *uio, int niov);
-void dmu_xuio_fini(struct xuio *uio);
-int dmu_xuio_add(struct xuio *uio, struct arc_buf *abuf, offset_t off,
-    size_t n);
-int dmu_xuio_cnt(struct xuio *uio);
-struct arc_buf *dmu_xuio_arcbuf(struct xuio *uio, int i);
-void dmu_xuio_clear(struct xuio *uio, int i);
-#endif /* HAVE_UIO_ZEROCOPY */
-void xuio_stat_wbuf_copied(void);
-void xuio_stat_wbuf_nocopy(void);
-
 extern int zfs_prefetch_disable;
 extern int zfs_max_recordsize;
 

include/sys/dmu_impl.h

@@ -237,13 +237,6 @@ extern "C" {
 struct objset;
 struct dmu_pool;
 
-typedef struct dmu_xuio {
-	int next;
-	int cnt;
-	struct arc_buf **bufs;
-	iovec_t *iovp;
-} dmu_xuio_t;
-
 typedef struct dmu_sendstatus {
 	list_node_t dss_link;
 	int dss_outfd;

include/sys/dnode.h

@@ -171,7 +171,7 @@ enum dnode_dirtycontext {
  * example, reading 32 dnodes from a 16k dnode block and all of the spill
  * blocks could issue 33 separate reads. Now suppose those dnodes have size
  * 1024 and therefore don't need spill blocks. Then the worst case number
- * of blocks read is reduced to from 33 to two--one per dnode block.
+ * of blocks read is reduced from 33 to two--one per dnode block.
  *
  * ZFS-on-Linux systems that make heavy use of extended attributes benefit
  * from this feature. In particular, ZFS-on-Linux supports the xattr=sa
@@ -232,8 +232,8 @@ typedef struct dnode_phys {
 	 * Both dn_pad2 and dn_pad3 are protected by the block's MAC. This
 	 * allows us to protect any fields that might be added here in the
 	 * future. In either case, developers will want to check
-	 * zio_crypt_init_uios_dnode() to ensure the new field is being
-	 * protected properly.
+	 * zio_crypt_init_uios_dnode() and zio_crypt_do_dnode_hmac_updates()
+	 * to ensure the new field is being protected and updated properly.
 	 */
 	uint64_t dn_pad3[4];
 
@@ -425,6 +425,7 @@ boolean_t dnode_add_ref(dnode_t *dn, void *ref);
 void dnode_rele(dnode_t *dn, void *ref);
 void dnode_rele_and_unlock(dnode_t *dn, void *tag, boolean_t evicting);
 int dnode_try_claim(objset_t *os, uint64_t object, int slots);
+boolean_t dnode_is_dirty(dnode_t *dn);
 void dnode_setdirty(dnode_t *dn, dmu_tx_t *tx);
 void dnode_set_dirtyctx(dnode_t *dn, dmu_tx_t *tx, void *tag);
 void dnode_sync(dnode_t *dn, dmu_tx_t *tx);

include/sys/dsl_synctask.h

@@ -41,10 +41,11 @@ typedef void (dsl_sigfunc_t)(void *, dmu_tx_t *);
 
 typedef enum zfs_space_check {
 	/*
-	 * Normal space check: if there is less than 3.2% free space,
-	 * the operation will fail.  Operations which are logically
-	 * creating things should use this (e.g. "zfs create", "zfs snapshot").
-	 * User writes (via the ZPL / ZVOL) also fail at this point.
+	 * Normal space check: if there is less than 3.2% free space (bounded
+	 * by spa_max_slop), the operation will fail.  Operations which are
+	 * logically creating things should use this (e.g. "zfs create", "zfs
+	 * snapshot").  User writes (via the ZPL / ZVOL) also fail at this
+	 * point.
 	 */
 	ZFS_SPACE_CHECK_NORMAL,
 

include/sys/efi_partition.h

@@ -370,7 +370,6 @@ extern	int	efi_rescan(int);
 extern	void	efi_free(struct dk_gpt *);
 extern	int	efi_type(int);
 extern	void	efi_err_check(struct dk_gpt *);
-extern	int	efi_auto_sense(int fd, struct dk_gpt **);
 extern	int	efi_use_whole_disk(int fd);
 #endif
 

include/sys/fm/util.h

@@ -31,6 +31,7 @@ extern "C" {
 #endif
 
 #include <sys/nvpair.h>
+#include <sys/zfs_file.h>
 
 /*
  * Shared user/kernel definitions for class length, error channel name,
@@ -96,8 +97,8 @@ extern void fm_nvprint(nvlist_t *);
 extern void zfs_zevent_post_cb(nvlist_t *nvl, nvlist_t *detector);
 extern int zfs_zevent_post(nvlist_t *, nvlist_t *, zevent_cb_t *);
 extern void zfs_zevent_drain_all(int *);
-extern int zfs_zevent_fd_hold(int, minor_t *, zfs_zevent_t **);
-extern void zfs_zevent_fd_rele(int);
+extern zfs_file_t *zfs_zevent_fd_hold(int, minor_t *, zfs_zevent_t **);
+extern void zfs_zevent_fd_rele(zfs_file_t *);
 extern int zfs_zevent_next(zfs_zevent_t *, nvlist_t **, uint64_t *, uint64_t *);
 extern int zfs_zevent_wait(zfs_zevent_t *);
 extern int zfs_zevent_seek(zfs_zevent_t *, uint64_t);

include/sys/sa.h

@@ -158,7 +158,7 @@ void sa_handle_lock(sa_handle_t *);
 void sa_handle_unlock(sa_handle_t *);
 
 #ifdef _KERNEL
-int sa_lookup_uio(sa_handle_t *, sa_attr_type_t, uio_t *);
+int sa_lookup_uio(sa_handle_t *, sa_attr_type_t, zfs_uio_t *);
 int sa_add_projid(sa_handle_t *, dmu_tx_t *, uint64_t);
 #endif
 

include/sys/uio_impl.h

@@ -41,9 +41,28 @@
 
 #include <sys/uio.h>
 
-extern int uiomove(void *, size_t, enum uio_rw, uio_t *);
-extern int uio_prefaultpages(ssize_t, uio_t *);
-extern int uiocopy(void *, size_t, enum uio_rw, uio_t *, size_t *);
-extern void uioskip(uio_t *, size_t);
+extern int zfs_uiomove(void *, size_t, zfs_uio_rw_t, zfs_uio_t *);
+extern int zfs_uiocopy(void *, size_t, zfs_uio_rw_t, zfs_uio_t *, size_t *);
+extern void zfs_uioskip(zfs_uio_t *, size_t);
+
+static inline void
+zfs_uio_iov_at_index(zfs_uio_t *uio, uint_t idx, void **base, uint64_t *len)
+{
+	*base = zfs_uio_iovbase(uio, idx);
+	*len = zfs_uio_iovlen(uio, idx);
+}
+
+static inline offset_t
+zfs_uio_index_at_offset(zfs_uio_t *uio, offset_t off, uint_t *vec_idx)
+{
+	*vec_idx = 0;
+	while (*vec_idx < zfs_uio_iovcnt(uio) &&
+	    off >= zfs_uio_iovlen(uio, *vec_idx)) {
+		off -= zfs_uio_iovlen(uio, *vec_idx);
+		(*vec_idx)++;
+	}
+
+	return (off);
+}
 
 #endif	/* _SYS_UIO_IMPL_H */

include/sys/zfs_context.h

@@ -72,6 +72,7 @@ extern "C" {
 #include <sys/trace.h>
 #include <sys/procfs_list.h>
 #include <sys/mod.h>
+#include <sys/uio_impl.h>
 #include <sys/zfs_context_os.h>
 #else /* _KERNEL || _STANDALONE */
 
@@ -759,7 +760,6 @@ extern void spl_fstrans_unmark(fstrans_cookie_t);
 extern int __spl_pf_fstrans_check(void);
 extern int kmem_cache_reap_active(void);
 
-#define	____cacheline_aligned
 
 /*
  * Kernel modules

include/sys/zfs_file.h

@@ -22,6 +22,8 @@
 #ifndef	_SYS_ZFS_FILE_H
 #define	_SYS_ZFS_FILE_H
 
+#include <sys/zfs_context.h>
+
 #ifndef _KERNEL
 typedef struct zfs_file {
 	int f_fd;
@@ -55,8 +57,8 @@ int zfs_file_fallocate(zfs_file_t *fp, int mode, loff_t offset, loff_t len);
 loff_t zfs_file_off(zfs_file_t *fp);
 int zfs_file_unlink(const char *);
 
-int zfs_file_get(int fd, zfs_file_t **fp);
-void zfs_file_put(int fd);
+zfs_file_t *zfs_file_get(int fd);
+void zfs_file_put(zfs_file_t *fp);
 void *zfs_file_private(zfs_file_t *fp);
 
 #endif /* _SYS_ZFS_FILE_H */

include/sys/zfs_ioctl.h

@@ -566,7 +566,7 @@ typedef struct zfsdev_state {
 } zfsdev_state_t;
 
 extern void *zfsdev_get_state(minor_t minor, enum zfsdev_state_type which);
-extern int zfsdev_getminor(int fd, minor_t *minorp);
+extern int zfsdev_getminor(zfs_file_t *fp, minor_t *minorp);
 extern minor_t zfsdev_minor_alloc(void);
 
 extern uint_t zfs_fsyncer_key;

include/sys/zfs_onexit.h

@@ -51,8 +51,8 @@ extern void zfs_onexit_destroy(zfs_onexit_t *zo);
 
 #endif
 
-extern int zfs_onexit_fd_hold(int fd, minor_t *minorp);
-extern void zfs_onexit_fd_rele(int fd);
+extern zfs_file_t *zfs_onexit_fd_hold(int fd, minor_t *minorp);
+extern void zfs_onexit_fd_rele(zfs_file_t *);
 extern int zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
     uint64_t *action_handle);
 

include/sys/zfs_sa.h

@@ -134,7 +134,7 @@ typedef struct znode_phys {
 #define	DXATTR_MAX_ENTRY_SIZE	(32768)
 #define	DXATTR_MAX_SA_SIZE	(SPA_OLD_MAXBLOCKSIZE >> 1)
 
-int zfs_sa_readlink(struct znode *, uio_t *);
+int zfs_sa_readlink(struct znode *, zfs_uio_t *);
 void zfs_sa_symlink(struct znode *, char *link, int len, dmu_tx_t *);
 void zfs_sa_get_scanstamp(struct znode *, xvattr_t *);
 void zfs_sa_set_scanstamp(struct znode *, xvattr_t *, dmu_tx_t *);

include/sys/zfs_vnops.h

@@ -27,16 +27,16 @@
 #include <sys/zfs_vnops_os.h>
 
 extern int zfs_fsync(znode_t *, int, cred_t *);
-extern int zfs_read(znode_t *, uio_t *, int, cred_t *);
-extern int zfs_write(znode_t *, uio_t *, int, cred_t *);
+extern int zfs_read(znode_t *, zfs_uio_t *, int, cred_t *);
+extern int zfs_write(znode_t *, zfs_uio_t *, int, cred_t *);
 extern int zfs_holey(znode_t *, ulong_t, loff_t *);
 extern int zfs_access(znode_t *, int, int, cred_t *);
 
 extern int zfs_getsecattr(znode_t *, vsecattr_t *, int, cred_t *);
 extern int zfs_setsecattr(znode_t *, vsecattr_t *, int, cred_t *);
 
-extern int mappedread(znode_t *, int, uio_t *);
-extern int mappedread_sf(znode_t *, int, uio_t *);
+extern int mappedread(znode_t *, int, zfs_uio_t *);
+extern int mappedread_sf(znode_t *, int, zfs_uio_t *);
 extern void update_pages(znode_t *, int64_t, int, objset_t *);
 
 /*

include/sys/zstd/zstd.h

@@ -56,21 +56,24 @@ typedef struct zfs_zstd_header {
 
 	/*
 	 * Version and compression level
-	 * We use a union to be able to big endian encode a single 32 bit
-	 * unsigned integer, but still access the individual bitmasked
-	 * components easily.
+	 * We used to use a union to reference compression level
+	 * and version easily, but as it turns out, relying on the
+	 * ordering of bitfields is not remotely portable.
+	 * So now we have get/set functions in zfs_zstd.c for
+	 * manipulating this in just the right way forever.
 	 */
-	union {
-		uint32_t raw_version_level;
-		struct {
-			uint32_t version : 24;
-			uint8_t level;
-		};
-	};
-
+	uint32_t raw_version_level;
 	char data[];
 } zfs_zstdhdr_t;
 
+/*
+ * Simple struct to pass the data from raw_version_level around.
+ */
+typedef struct zfs_zstd_meta {
+	uint8_t level;
+	uint32_t version;
+} zfs_zstdmeta_t;
+
 /*
  * kstat helper macros
  */
@@ -94,6 +97,131 @@ int zfs_zstd_decompress(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n);
 void zfs_zstd_cache_reap_now(void);
 
+/*
+ * So, the reason we have all these complicated set/get functions is that
+ * originally, in the zstd "header" we wrote out to disk, we used a 32-bit
+ * bitfield to store the "level" (8 bits) and "version" (24 bits).
+ *
+ * Unfortunately, bitfields make few promises about how they're arranged in
+ * memory...
+ *
+ * By way of example, if we were using version 1.4.5 and level 3, it'd be
+ * level = 0x03, version = 10405/0x0028A5, which gets broken into Vhigh = 0x00,
+ * Vmid = 0x28, Vlow = 0xA5. We include these positions below to help follow
+ * which data winds up where.
+ *
+ * As a consequence, we wound up with little endian platforms with a layout
+ * like this in memory:
+ *
+ *      0       8      16      24      32
+ *      +-------+-------+-------+-------+
+ *      | Vlow  | Vmid  | Vhigh | level |
+ *      +-------+-------+-------+-------+
+ *        =A5     =28     =00     =03
+ *
+ * ...and then, after being run through BE_32(), serializing this out to
+ * disk:
+ *
+ *      0       8      16      24      32
+ *      +-------+-------+-------+-------+
+ *      | level | Vhigh | Vmid  | Vlow  |
+ *      +-------+-------+-------+-------+
+ *        =03     =00     =28     =A5
+ *
+ * while on big-endian systems, since BE_32() is a noop there, both in
+ * memory and on disk, we wind up with:
+ *
+ *      0       8      16      24      32
+ *      +-------+-------+-------+-------+
+ *      | Vhigh | Vmid  | Vlow  | level |
+ *      +-------+-------+-------+-------+
+ *        =00     =28     =A5     =03
+ *
+ * (Vhigh is always 0 until version exceeds 6.55.35. Vmid and Vlow are the
+ * other two bytes of the "version" data.)
+ *
+ * So now we use the BF32_SET macros to get consistent behavior (the
+ * ondisk LE encoding, since x86 currently rules the world) across
+ * platforms, but the "get" behavior requires that we check each of the
+ * bytes in the aforementioned former-bitfield for 0x00, and from there,
+ * we can know which possible layout we're dealing with. (Only the two
+ * that have been observed in the wild are illustrated above, but handlers
+ * for all 4 positions of 0x00 are implemented.
+ */
+
+static inline void
+zfs_get_hdrmeta(const zfs_zstdhdr_t *blob, zfs_zstdmeta_t *res)
+{
+	uint32_t raw = blob->raw_version_level;
+	uint8_t findme = 0xff;
+	int shift;
+	for (shift = 0; shift < 4; shift++) {
+		findme = BF32_GET(raw, 8*shift, 8);
+		if (findme == 0)
+			break;
+	}
+	switch (shift) {
+	case 0:
+		res->level = BF32_GET(raw, 24, 8);
+		res->version = BSWAP_32(raw);
+		res->version = BF32_GET(res->version, 8, 24);
+		break;
+	case 1:
+		res->level = BF32_GET(raw, 0, 8);
+		res->version = BSWAP_32(raw);
+		res->version = BF32_GET(res->version, 0, 24);
+		break;
+	case 2:
+		res->level = BF32_GET(raw, 24, 8);
+		res->version = BF32_GET(raw, 0, 24);
+		break;
+	case 3:
+		res->level = BF32_GET(raw, 0, 8);
+		res->version = BF32_GET(raw, 8, 24);
+		break;
+	default:
+		res->level = 0;
+		res->version = 0;
+		break;
+	}
+}
+
+static inline uint8_t
+zfs_get_hdrlevel(const zfs_zstdhdr_t *blob)
+{
+	uint8_t level = 0;
+	zfs_zstdmeta_t res;
+	zfs_get_hdrmeta(blob, &res);
+	level = res.level;
+	return (level);
+}
+
+static inline uint32_t
+zfs_get_hdrversion(const zfs_zstdhdr_t *blob)
+{
+	uint32_t version = 0;
+	zfs_zstdmeta_t res;
+	zfs_get_hdrmeta(blob, &res);
+	version = res.version;
+	return (version);
+
+}
+
+static inline void
+zfs_set_hdrversion(zfs_zstdhdr_t *blob, uint32_t version)
+{
+	/* cppcheck-suppress syntaxError */
+	BF32_SET(blob->raw_version_level, 0, 24, version);
+}
+
+static inline void
+zfs_set_hdrlevel(zfs_zstdhdr_t *blob, uint8_t level)
+{
+	/* cppcheck-suppress syntaxError */
+	BF32_SET(blob->raw_version_level, 24, 8, level);
+}
+
+
 #ifdef	__cplusplus
 }
 #endif

lib/libefi/rdwr_efi.c

@@ -218,22 +218,15 @@ read_disk_info(int fd, diskaddr_t *capacity, uint_t *lbsize)
 static char *
 efi_get_devname(int fd)
 {
-	char *path;
-	char *dev_name;
-
-	path = calloc(1, PATH_MAX);
-	if (path == NULL)
-		return (NULL);
+	char path[32];
 
 	/*
 	 * The libefi API only provides the open fd and not the file path.
 	 * To handle this realpath(3) is used to resolve the block device
 	 * name from /proc/self/fd/<fd>.
 	 */
-	(void) sprintf(path, "/proc/self/fd/%d", fd);
-	dev_name = realpath(path, NULL);
-	free(path);
-	return (dev_name);
+	(void) snprintf(path, sizeof (path), "/proc/self/fd/%d", fd);
+	return (realpath(path, NULL));
 }
 
 static int
@@ -468,6 +461,7 @@ efi_alloc_and_read(int fd, struct dk_gpt **vtoc)
 		    (int) sizeof (struct dk_part) * (vptr->efi_nparts - 1);
 		nparts = vptr->efi_nparts;
 		if ((tmp = realloc(vptr, length)) == NULL) {
+			/* cppcheck-suppress doubleFree */
 			free(vptr);
 			*vtoc = NULL;
 			return (VT_ERROR);
@@ -1698,57 +1692,3 @@ efi_err_check(struct dk_gpt *vtoc)
 		    "no reserved partition found\n");
 	}
 }
-
-/*
- * We need to get information necessary to construct a *new* efi
- * label type
- */
-int
-efi_auto_sense(int fd, struct dk_gpt **vtoc)
-{
-
-	int	i;
-
-	/*
-	 * Now build the default partition table
-	 */
-	if (efi_alloc_and_init(fd, EFI_NUMPAR, vtoc) != 0) {
-		if (efi_debug) {
-			(void) fprintf(stderr, "efi_alloc_and_init failed.\n");
-		}
-		return (-1);
-	}
-
-	for (i = 0; i < MIN((*vtoc)->efi_nparts, V_NUMPAR); i++) {
-		(*vtoc)->efi_parts[i].p_tag = default_vtoc_map[i].p_tag;
-		(*vtoc)->efi_parts[i].p_flag = default_vtoc_map[i].p_flag;
-		(*vtoc)->efi_parts[i].p_start = 0;
-		(*vtoc)->efi_parts[i].p_size = 0;
-	}
-	/*
-	 * Make constants first
-	 * and variable partitions later
-	 */
-
-	/* root partition - s0 128 MB */
-	(*vtoc)->efi_parts[0].p_start = 34;
-	(*vtoc)->efi_parts[0].p_size = 262144;
-
-	/* partition - s1  128 MB */
-	(*vtoc)->efi_parts[1].p_start = 262178;
-	(*vtoc)->efi_parts[1].p_size = 262144;
-
-	/* partition -s2 is NOT the Backup disk */
-	(*vtoc)->efi_parts[2].p_tag = V_UNASSIGNED;
-
-	/* partition -s6 /usr partition - HOG */
-	(*vtoc)->efi_parts[6].p_start = 524322;
-	(*vtoc)->efi_parts[6].p_size = (*vtoc)->efi_last_u_lba - 524322
-	    - (1024 * 16);
-
-	/* efi reserved partition - s9 16K */
-	(*vtoc)->efi_parts[8].p_start = (*vtoc)->efi_last_u_lba - (1024 * 16);
-	(*vtoc)->efi_parts[8].p_size = (1024 * 16);
-	(*vtoc)->efi_parts[8].p_tag = V_RESERVED;
-	return (0);
-}

lib/libspl/include/sys/feature_tests.h

@@ -27,6 +27,15 @@
 #ifndef _SYS_FEATURE_TESTS_H
 #define	_SYS_FEATURE_TESTS_H
 
-#define	__NORETURN	__attribute__((__noreturn__))
+#define	____cacheline_aligned
+#define	__NORETURN		__attribute__((__noreturn__))
+
+#if !defined(fallthrough)
+#if defined(HAVE_IMPLICIT_FALLTHROUGH)
+#define	fallthrough		__attribute__((__fallthrough__))
+#else
+#define	fallthrough		((void)0)
+#endif
+#endif
 
 #endif

lib/libspl/include/sys/uio.h

@@ -51,97 +51,58 @@
 typedef struct iovec iovec_t;
 
 #if defined(__linux__) || defined(__APPLE__)
-typedef enum uio_rw {
+typedef enum zfs_uio_rw {
 	UIO_READ =	0,
 	UIO_WRITE =	1,
-} uio_rw_t;
+} zfs_uio_rw_t;
 
-typedef enum uio_seg {
+typedef enum zfs_uio_seg {
 	UIO_USERSPACE =	0,
 	UIO_SYSSPACE =	1,
-} uio_seg_t;
+} zfs_uio_seg_t;
 
 #elif defined(__FreeBSD__)
-typedef enum uio_seg  uio_seg_t;
+typedef enum uio_seg  zfs_uio_seg_t;
 #endif
 
-typedef struct uio {
+typedef struct zfs_uio {
 	struct iovec	*uio_iov;	/* pointer to array of iovecs */
 	int		uio_iovcnt;	/* number of iovecs */
 	offset_t	uio_loffset;	/* file offset */
-	uio_seg_t	uio_segflg;	/* address space (kernel or user) */
+	zfs_uio_seg_t	uio_segflg;	/* address space (kernel or user) */
 	uint16_t	uio_fmode;	/* file mode flags */
 	uint16_t	uio_extflg;	/* extended flags */
 	ssize_t		uio_resid;	/* residual count */
-} uio_t;
+} zfs_uio_t;
 
-typedef enum xuio_type {
-	UIOTYPE_ASYNCIO,
-	UIOTYPE_ZEROCOPY,
-} xuio_type_t;
-
-#define	UIOA_IOV_MAX	16
-
-typedef struct uioa_page_s {		/* locked uio_iov state */
-	int	uioa_pfncnt;		/* count of pfn_t(s) in *uioa_ppp */
-	void	**uioa_ppp;		/* page_t or pfn_t array */
-	caddr_t	uioa_base;		/* address base */
-	size_t	uioa_len;		/* span length */
-} uioa_page_t;
-
-typedef struct xuio {
-	uio_t xu_uio;				/* embedded UIO structure */
-
-	/* Extended uio fields */
-	enum xuio_type xu_type;			/* uio type */
-	union {
-		struct {
-			uint32_t xu_a_state;	/* state of async i/o */
-			ssize_t xu_a_mbytes;	/* bytes moved */
-			uioa_page_t *xu_a_lcur;	/* uioa_locked[] pointer */
-			void **xu_a_lppp;	/* lcur->uioa_pppp[] pointer */
-			void *xu_a_hwst[4];	/* opaque hardware state */
-			uioa_page_t xu_a_locked[UIOA_IOV_MAX];
-		} xu_aio;
-
-		struct {
-			int xu_zc_rw;		/* read or write buffer */
-			void *xu_zc_priv;	/* fs specific */
-		} xu_zc;
-	} xu_ext;
-} xuio_t;
-
-#define	XUIO_XUZC_PRIV(xuio)	xuio->xu_ext.xu_zc.xu_zc_priv
-#define	XUIO_XUZC_RW(xuio)	xuio->xu_ext.xu_zc.xu_zc_rw
-
-#define	uio_segflg(uio)			(uio)->uio_segflg
-#define	uio_offset(uio)			(uio)->uio_loffset
-#define	uio_resid(uio)			(uio)->uio_resid
-#define	uio_iovcnt(uio)			(uio)->uio_iovcnt
-#define	uio_iovlen(uio, idx)		(uio)->uio_iov[(idx)].iov_len
-#define	uio_iovbase(uio, idx)		(uio)->uio_iov[(idx)].iov_base
+#define	zfs_uio_segflg(uio)		(uio)->uio_segflg
+#define	zfs_uio_offset(uio)		(uio)->uio_loffset
+#define	zfs_uio_resid(uio)		(uio)->uio_resid
+#define	zfs_uio_iovcnt(uio)		(uio)->uio_iovcnt
+#define	zfs_uio_iovlen(uio, idx)	(uio)->uio_iov[(idx)].iov_len
+#define	zfs_uio_iovbase(uio, idx)	(uio)->uio_iov[(idx)].iov_base
 
 static inline void
-uio_iov_at_index(uio_t *uio, uint_t idx, void **base, uint64_t *len)
+zfs_uio_iov_at_index(zfs_uio_t *uio, uint_t idx, void **base, uint64_t *len)
 {
-	*base = uio_iovbase(uio, idx);
-	*len = uio_iovlen(uio, idx);
+	*base = zfs_uio_iovbase(uio, idx);
+	*len = zfs_uio_iovlen(uio, idx);
 }
 
 static inline void
-uio_advance(uio_t *uio, size_t size)
+zfs_uio_advance(zfs_uio_t *uio, size_t size)
 {
 	uio->uio_resid -= size;
 	uio->uio_loffset += size;
 }
 
 static inline offset_t
-uio_index_at_offset(uio_t *uio, offset_t off, uint_t *vec_idx)
+zfs_uio_index_at_offset(zfs_uio_t *uio, offset_t off, uint_t *vec_idx)
 {
 	*vec_idx = 0;
-	while (*vec_idx < (uint_t)uio_iovcnt(uio) &&
-	    off >= (offset_t)uio_iovlen(uio, *vec_idx)) {
-		off -= uio_iovlen(uio, *vec_idx);
+	while (*vec_idx < (uint_t)zfs_uio_iovcnt(uio) &&
+	    off >= (offset_t)zfs_uio_iovlen(uio, *vec_idx)) {
+		off -= zfs_uio_iovlen(uio, *vec_idx);
 		(*vec_idx)++;
 	}
 

lib/libzfs/libzfs_dataset.c

@@ -1365,10 +1365,9 @@ badlabel:
 				(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
 				goto error;
 			}
+			fallthrough;
 		}
 
-			/*FALLTHRU*/
-
 		case ZFS_PROP_SHARESMB:
 		case ZFS_PROP_SHARENFS:
 			/*
@@ -3767,8 +3766,8 @@ zfs_create(libzfs_handle_t *hdl, const char *path, zfs_type_t type,
 			if (type == ZFS_TYPE_VOLUME)
 				return (zfs_error(hdl, EZFS_VOLTOOBIG,
 				    errbuf));
+			fallthrough;
 #endif
-			/* FALLTHROUGH */
 		default:
 			return (zfs_standard_error(hdl, errno, errbuf));
 		}

lib/libzfs/libzfs_diff.c

@@ -243,6 +243,7 @@ write_inuse_diffs_one(FILE *fp, differ_info_t *di, uint64_t dobj)
 	struct zfs_stat fsb, tsb;
 	mode_t fmode, tmode;
 	char fobjname[MAXPATHLEN], tobjname[MAXPATHLEN];
+	boolean_t already_logged = B_FALSE;
 	int fobjerr, tobjerr;
 	int change;
 
@@ -254,22 +255,35 @@ write_inuse_diffs_one(FILE *fp, differ_info_t *di, uint64_t dobj)
 	 * we get ENOENT, then the object just didn't exist in that
 	 * snapshot.  If we get ENOTSUP, then we tried to get
 	 * info on a non-ZPL object, which we don't care about anyway.
+	 * For any other error we print a warning which includes the
+	 * errno and continue.
 	 */
+
 	fobjerr = get_stats_for_obj(di, di->fromsnap, dobj, fobjname,
 	    MAXPATHLEN, &fsb);
-	if (fobjerr && di->zerr != ENOENT && di->zerr != ENOTSUP)
-		return (-1);
+	if (fobjerr && di->zerr != ENOTSUP && di->zerr != ENOENT) {
+		zfs_error_aux(di->zhp->zfs_hdl, strerror(di->zerr));
+		zfs_error(di->zhp->zfs_hdl, di->zerr, di->errbuf);
+		/*
+		 * Let's not print an error for the same object more than
+		 * once if it happens in both snapshots
+		 */
+		already_logged = B_TRUE;
+	}
 
 	tobjerr = get_stats_for_obj(di, di->tosnap, dobj, tobjname,
 	    MAXPATHLEN, &tsb);
-	if (tobjerr && di->zerr != ENOENT && di->zerr != ENOTSUP)
-		return (-1);
 
+	if (tobjerr && di->zerr != ENOTSUP && di->zerr != ENOENT) {
+		if (!already_logged) {
+			zfs_error_aux(di->zhp->zfs_hdl, strerror(di->zerr));
+			zfs_error(di->zhp->zfs_hdl, di->zerr, di->errbuf);
+		}
+	}
 	/*
 	 * Unallocated object sharing the same meta dnode block
 	 */
 	if (fobjerr && tobjerr) {
-		ASSERT(di->zerr == ENOENT || di->zerr == ENOTSUP);
 		di->zerr = 0;
 		return (0);
 	}
@@ -344,12 +358,11 @@ describe_free(FILE *fp, differ_info_t *di, uint64_t object, char *namebuf,
 {
 	struct zfs_stat sb;
 
-	if (get_stats_for_obj(di, di->fromsnap, object, namebuf,
-	    maxlen, &sb) != 0) {
-		return (-1);
-	}
+	(void) get_stats_for_obj(di, di->fromsnap, object, namebuf,
+	    maxlen, &sb);
+
 	/* Don't print if in the delete queue on from side */
-	if (di->zerr == ESTALE) {
+	if (di->zerr == ESTALE || di->zerr == ENOENT) {
 		di->zerr = 0;
 		return (0);
 	}
@@ -384,8 +397,6 @@ write_free_diffs(FILE *fp, differ_info_t *di, dmu_diff_record_t *dr)
 			}
 			err = describe_free(fp, di, zc.zc_obj, fobjname,
 			    MAXPATHLEN);
-			if (err)
-				break;
 		} else if (errno == ESRCH) {
 			break;
 		} else {

lib/libzfs/libzfs_pool.c

@@ -309,7 +309,7 @@ zpool_get_prop(zpool_handle_t *zhp, zpool_prop_t prop, char *buf,
 				    len);
 				break;
 			}
-			/* FALLTHROUGH */
+			fallthrough;
 		default:
 			(void) strlcpy(buf, "-", len);
 			break;
@@ -400,7 +400,7 @@ zpool_get_prop(zpool_handle_t *zhp, zpool_prop_t prop, char *buf,
 				(void) snprintf(buf, len, "-");
 				break;
 			}
-			/* FALLTHROUGH */
+			fallthrough;
 		default:
 			(void) snprintf(buf, len, "%llu", (u_longlong_t)intval);
 		}

lib/libzfs/libzfs_sendrecv.c

@@ -847,7 +847,8 @@ dump_ioctl(zfs_handle_t *zhp, const char *fromsnap, uint64_t fromsnap_obj,
 		case ERANGE:
 		case EFAULT:
 		case EROFS:
-			zfs_error_aux(hdl, strerror(errno));
+		case EINVAL:
+			zfs_error_aux(hdl, "%s", strerror(errno));
 			return (zfs_error(hdl, EZFS_BADBACKUP, errbuf));
 
 		default:
@@ -4883,7 +4884,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 				(void) zfs_error(hdl, EZFS_BUSY, errbuf);
 				break;
 			}
-			/* fallthru */
+			fallthrough;
 		default:
 			(void) zfs_standard_error(hdl, ioctl_errno, errbuf);
 		}

lib/libzfs/libzfs_util.c

@@ -598,7 +598,7 @@ zfs_setprop_error(libzfs_handle_t *hdl, zfs_prop_t prop, int err,
 			break;
 		}
 #endif
-		/* FALLTHROUGH */
+		fallthrough;
 	default:
 		(void) zfs_standard_error(hdl, err, errbuf);
 	}

lib/libzfsbootenv/libzfsbootenv.abi

@@ -1,4 +1,4 @@
-<abi-corpus path='libzfsbootenv.so' architecture='elf-amd-x86_64' soname='libzfsbootenv.so.1'>
+<abi-corpus version='2.0' path='libzfsbootenv.so' architecture='elf-amd-x86_64' soname='libzfsbootenv.so.1'>
   <elf-needed>
     <dependency name='libzfs.so.4'/>
     <dependency name='libzfs_core.so.3'/>
@@ -24,189 +24,555 @@
     <elf-symbol name='lzbe_remove_pair' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='lzbe_set_boot_device' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
   </elf-function-symbols>
-  <abi-instr version='1.0' address-size='64' path='lzbe_device.c' comp-dir-path='/home/fedora/zfs/lib/libzfsbootenv' language='LANG_C99'>
+  <abi-instr address-size='64' path='lzbe_device.c' comp-dir-path='/home/hutter2/release20/lib/libzfsbootenv' language='LANG_C99'>
     <type-decl name='char' size-in-bits='8' id='type-id-1'/>
-    <type-decl name='int' size-in-bits='32' id='type-id-2'/>
-    <type-decl name='unnamed-enum-underlying-type' is-anonymous='yes' size-in-bits='32' alignment-in-bits='32' id='type-id-3'/>
-    <typedef-decl name='lzbe_flags_t' type-id='type-id-4' filepath='../../include/libzfsbootenv.h' line='26' column='1' id='type-id-5'/>
-    <enum-decl name='lzbe_flags' filepath='../../include/libzfsbootenv.h' line='23' column='1' id='type-id-4'>
-      <underlying-type type-id='type-id-3'/>
+    <array-type-def dimensions='1' type-id='type-id-1' size-in-bits='8' id='type-id-2'>
+      <subrange length='1' type-id='type-id-3' id='type-id-4'/>
+    </array-type-def>
+    <array-type-def dimensions='1' type-id='type-id-1' size-in-bits='160' id='type-id-5'>
+      <subrange length='20' type-id='type-id-3' id='type-id-6'/>
+    </array-type-def>
+    <class-decl name='_IO_codecvt' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-7'/>
+    <class-decl name='_IO_marker' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-8'/>
+    <class-decl name='_IO_wide_data' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-9'/>
+    <class-decl name='libzfs_handle' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-10'/>
+    <class-decl name='zpool_handle' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-11'/>
+    <type-decl name='int' size-in-bits='32' id='type-id-12'/>
+    <type-decl name='long int' size-in-bits='64' id='type-id-13'/>
+    <type-decl name='signed char' size-in-bits='8' id='type-id-14'/>
+    <type-decl name='unnamed-enum-underlying-type-32' is-anonymous='yes' size-in-bits='32' alignment-in-bits='32' id='type-id-15'/>
+    <type-decl name='unsigned int' size-in-bits='32' id='type-id-16'/>
+    <type-decl name='unsigned long int' size-in-bits='64' id='type-id-3'/>
+    <type-decl name='unsigned short int' size-in-bits='16' id='type-id-17'/>
+    <type-decl name='variadic parameter type' id='type-id-18'/>
+    <type-decl name='void' id='type-id-19'/>
+    <typedef-decl name='zpool_handle_t' type-id='type-id-11' filepath='../../include/libzfs.h' line='195' column='1' id='type-id-20'/>
+    <typedef-decl name='libzfs_handle_t' type-id='type-id-10' filepath='../../include/libzfs.h' line='196' column='1' id='type-id-21'/>
+    <enum-decl name='lzbe_flags' filepath='../../include/libzfsbootenv.h' line='23' column='1' id='type-id-22'>
+      <underlying-type type-id='type-id-15'/>
       <enumerator name='lzbe_add' value='0'/>
       <enumerator name='lzbe_replace' value='1'/>
     </enum-decl>
-    <pointer-type-def type-id='type-id-1' size-in-bits='64' id='type-id-6'/>
-    <pointer-type-def type-id='type-id-6' size-in-bits='64' id='type-id-7'/>
-    <qualified-type-def type-id='type-id-1' const='yes' id='type-id-8'/>
-    <pointer-type-def type-id='type-id-8' size-in-bits='64' id='type-id-9'/>
-    <function-decl name='lzbe_get_boot_device' mangled-name='lzbe_get_boot_device' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_device.c' line='114' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_get_boot_device'>
-      <parameter type-id='type-id-9' name='pool' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_device.c' line='114' column='1'/>
-      <parameter type-id='type-id-7' name='device' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_device.c' line='114' column='1'/>
-      <return type-id='type-id-2'/>
-    </function-decl>
-    <function-decl name='lzbe_set_boot_device' mangled-name='lzbe_set_boot_device' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_device.c' line='28' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_set_boot_device'>
-      <parameter type-id='type-id-9' name='pool' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_device.c' line='28' column='1'/>
-      <parameter type-id='type-id-5' name='flag' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_device.c' line='28' column='1'/>
-      <parameter type-id='type-id-9' name='device' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_device.c' line='28' column='1'/>
-      <return type-id='type-id-2'/>
-    </function-decl>
-  </abi-instr>
-  <abi-instr version='1.0' address-size='64' path='lzbe_pair.c' comp-dir-path='/home/fedora/zfs/lib/libzfsbootenv' language='LANG_C99'>
-    <type-decl name='unsigned long int' size-in-bits='64' id='type-id-10'/>
-    <type-decl name='void' id='type-id-11'/>
-    <typedef-decl name='size_t' type-id='type-id-10' filepath='/usr/lib/gcc/x86_64-redhat-linux/10/include/stddef.h' line='209' column='1' id='type-id-12'/>
-    <pointer-type-def type-id='type-id-11' size-in-bits='64' id='type-id-13'/>
-    <pointer-type-def type-id='type-id-13' size-in-bits='64' id='type-id-14'/>
-    <function-decl name='lzbe_remove_pair' mangled-name='lzbe_remove_pair' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='343' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_remove_pair'>
-      <parameter type-id='type-id-13' name='ptr' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='343' column='1'/>
-      <parameter type-id='type-id-9' name='key' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='343' column='1'/>
-      <return type-id='type-id-2'/>
-    </function-decl>
-    <function-decl name='lzbe_add_pair' mangled-name='lzbe_add_pair' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='182' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_add_pair'>
-      <parameter type-id='type-id-13' name='ptr' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='182' column='1'/>
-      <parameter type-id='type-id-9' name='key' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='182' column='1'/>
-      <parameter type-id='type-id-9' name='type' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='182' column='1'/>
-      <parameter type-id='type-id-13' name='value' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='182' column='1'/>
-      <parameter type-id='type-id-12' name='size' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='183' column='1'/>
-      <return type-id='type-id-2'/>
-    </function-decl>
-    <function-decl name='lzbe_nvlist_free' mangled-name='lzbe_nvlist_free' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='131' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_nvlist_free'>
-      <parameter type-id='type-id-13' name='ptr' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='131' column='1'/>
-      <return type-id='type-id-11'/>
-    </function-decl>
-    <function-decl name='lzbe_nvlist_set' mangled-name='lzbe_nvlist_set' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='74' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_nvlist_set'>
-      <parameter type-id='type-id-9' name='pool' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='74' column='1'/>
-      <parameter type-id='type-id-9' name='key' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='74' column='1'/>
-      <parameter type-id='type-id-13' name='ptr' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='74' column='1'/>
-      <return type-id='type-id-2'/>
-    </function-decl>
-    <function-decl name='lzbe_nvlist_get' mangled-name='lzbe_nvlist_get' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='27' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_nvlist_get'>
-      <parameter type-id='type-id-9' name='pool' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='27' column='1'/>
-      <parameter type-id='type-id-9' name='key' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='27' column='1'/>
-      <parameter type-id='type-id-14' name='ptr' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_pair.c' line='27' column='1'/>
-      <return type-id='type-id-2'/>
-    </function-decl>
-  </abi-instr>
-  <abi-instr version='1.0' address-size='64' path='lzbe_util.c' comp-dir-path='/home/fedora/zfs/lib/libzfsbootenv' language='LANG_C99'>
-
-
-    <array-type-def dimensions='1' type-id='type-id-1' size-in-bits='8' id='type-id-15'>
-      <subrange length='1' type-id='type-id-10' id='type-id-16'/>
-
-    </array-type-def>
-    <array-type-def dimensions='1' type-id='type-id-1' size-in-bits='160' id='type-id-17'>
-      <subrange length='20' type-id='type-id-10' id='type-id-18'/>
-
-    </array-type-def>
-    <class-decl name='_IO_codecvt' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-19'/>
-    <class-decl name='_IO_marker' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-20'/>
-    <class-decl name='_IO_wide_data' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-21'/>
-    <type-decl name='long int' size-in-bits='64' id='type-id-22'/>
-    <type-decl name='signed char' size-in-bits='8' id='type-id-23'/>
-    <type-decl name='unsigned short int' size-in-bits='16' id='type-id-24'/>
-    <typedef-decl name='FILE' type-id='type-id-25' filepath='/usr/include/bits/types/FILE.h' line='7' column='1' id='type-id-26'/>
-    <class-decl name='_IO_FILE' size-in-bits='1728' is-struct='yes' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='49' column='1' id='type-id-25'>
+    <typedef-decl name='lzbe_flags_t' type-id='type-id-22' filepath='../../include/libzfsbootenv.h' line='26' column='1' id='type-id-23'/>
+    <class-decl name='nvlist' size-in-bits='192' is-struct='yes' visibility='default' filepath='../../include/sys/nvpair.h' line='85' column='1' id='type-id-24'>
       <data-member access='public' layout-offset-in-bits='0'>
-        <var-decl name='_flags' type-id='type-id-2' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='51' column='1'/>
+        <var-decl name='nvl_version' type-id='type-id-25' visibility='default' filepath='../../include/sys/nvpair.h' line='86' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='32'>
+        <var-decl name='nvl_nvflag' type-id='type-id-26' visibility='default' filepath='../../include/sys/nvpair.h' line='87' column='1'/>
       </data-member>
       <data-member access='public' layout-offset-in-bits='64'>
-        <var-decl name='_IO_read_ptr' type-id='type-id-6' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='54' column='1'/>
+        <var-decl name='nvl_priv' type-id='type-id-27' visibility='default' filepath='../../include/sys/nvpair.h' line='88' column='1'/>
       </data-member>
       <data-member access='public' layout-offset-in-bits='128'>
-        <var-decl name='_IO_read_end' type-id='type-id-6' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='55' column='1'/>
+        <var-decl name='nvl_flag' type-id='type-id-26' visibility='default' filepath='../../include/sys/nvpair.h' line='89' column='1'/>
       </data-member>
-      <data-member access='public' layout-offset-in-bits='192'>
-        <var-decl name='_IO_read_base' type-id='type-id-6' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='56' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='256'>
-        <var-decl name='_IO_write_base' type-id='type-id-6' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='57' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='320'>
-        <var-decl name='_IO_write_ptr' type-id='type-id-6' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='58' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='384'>
-        <var-decl name='_IO_write_end' type-id='type-id-6' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='59' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='448'>
-        <var-decl name='_IO_buf_base' type-id='type-id-6' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='60' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='512'>
-        <var-decl name='_IO_buf_end' type-id='type-id-6' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='61' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='576'>
-        <var-decl name='_IO_save_base' type-id='type-id-6' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='64' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='640'>
-        <var-decl name='_IO_backup_base' type-id='type-id-6' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='65' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='704'>
-        <var-decl name='_IO_save_end' type-id='type-id-6' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='66' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='768'>
-        <var-decl name='_markers' type-id='type-id-27' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='68' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='832'>
-        <var-decl name='_chain' type-id='type-id-28' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='70' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='896'>
-        <var-decl name='_fileno' type-id='type-id-2' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='72' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='928'>
-        <var-decl name='_flags2' type-id='type-id-2' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='73' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='960'>
-        <var-decl name='_old_offset' type-id='type-id-29' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='74' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1024'>
-        <var-decl name='_cur_column' type-id='type-id-24' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='77' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1040'>
-        <var-decl name='_vtable_offset' type-id='type-id-23' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='78' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1048'>
-        <var-decl name='_shortbuf' type-id='type-id-15' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='79' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1088'>
-        <var-decl name='_lock' type-id='type-id-30' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='81' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1152'>
-        <var-decl name='_offset' type-id='type-id-31' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='89' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1216'>
-        <var-decl name='_codecvt' type-id='type-id-32' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='91' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1280'>
-        <var-decl name='_wide_data' type-id='type-id-33' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='92' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1344'>
-        <var-decl name='_freeres_list' type-id='type-id-28' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='93' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1408'>
-        <var-decl name='_freeres_buf' type-id='type-id-13' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='94' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1472'>
-        <var-decl name='__pad5' type-id='type-id-12' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='95' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1536'>
-        <var-decl name='_mode' type-id='type-id-2' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='96' column='1'/>
-      </data-member>
-      <data-member access='public' layout-offset-in-bits='1568'>
-        <var-decl name='_unused2' type-id='type-id-17' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='98' column='1'/>
+      <data-member access='public' layout-offset-in-bits='160'>
+        <var-decl name='nvl_pad' type-id='type-id-25' visibility='default' filepath='../../include/sys/nvpair.h' line='90' column='1'/>
       </data-member>
     </class-decl>
-    <typedef-decl name='__off_t' type-id='type-id-22' filepath='/usr/include/bits/types.h' line='152' column='1' id='type-id-29'/>
-    <typedef-decl name='_IO_lock_t' type-id='type-id-11' filepath='/usr/include/bits/types/struct_FILE.h' line='43' column='1' id='type-id-34'/>
-    <typedef-decl name='__off64_t' type-id='type-id-22' filepath='/usr/include/bits/types.h' line='153' column='1' id='type-id-31'/>
-    <pointer-type-def type-id='type-id-26' size-in-bits='64' id='type-id-35'/>
-    <pointer-type-def type-id='type-id-25' size-in-bits='64' id='type-id-28'/>
-    <pointer-type-def type-id='type-id-19' size-in-bits='64' id='type-id-32'/>
-    <pointer-type-def type-id='type-id-34' size-in-bits='64' id='type-id-30'/>
-    <pointer-type-def type-id='type-id-20' size-in-bits='64' id='type-id-27'/>
-    <pointer-type-def type-id='type-id-21' size-in-bits='64' id='type-id-33'/>
-    <function-decl name='lzbe_bootenv_print' mangled-name='lzbe_bootenv_print' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_util.c' line='24' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_bootenv_print'>
-      <parameter type-id='type-id-9' name='pool' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_util.c' line='24' column='1'/>
-      <parameter type-id='type-id-9' name='nvlist' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_util.c' line='24' column='1'/>
-      <parameter type-id='type-id-35' name='of' filepath='/home/fedora/zfs/lib/libzfsbootenv/lzbe_util.c' line='24' column='1'/>
-      <return type-id='type-id-2'/>
+    <typedef-decl name='nvlist_t' type-id='type-id-24' filepath='../../include/sys/nvpair.h' line='91' column='1' id='type-id-28'/>
+    <enum-decl name='boolean_t' naming-typedef-id='type-id-29' filepath='../../lib/libspl/include/sys/stdtypes.h' line='26' column='1' id='type-id-30'>
+      <underlying-type type-id='type-id-15'/>
+      <enumerator name='B_FALSE' value='0'/>
+      <enumerator name='B_TRUE' value='1'/>
+    </enum-decl>
+    <typedef-decl name='boolean_t' type-id='type-id-30' filepath='../../lib/libspl/include/sys/stdtypes.h' line='29' column='1' id='type-id-29'/>
+    <typedef-decl name='int32_t' type-id='type-id-31' filepath='/usr/include/bits/stdint-intn.h' line='26' column='1' id='type-id-25'/>
+    <typedef-decl name='uint32_t' type-id='type-id-32' filepath='/usr/include/bits/stdint-uintn.h' line='26' column='1' id='type-id-26'/>
+    <typedef-decl name='uint64_t' type-id='type-id-33' filepath='/usr/include/bits/stdint-uintn.h' line='27' column='1' id='type-id-27'/>
+    <typedef-decl name='__int32_t' type-id='type-id-12' filepath='/usr/include/bits/types.h' line='41' column='1' id='type-id-31'/>
+    <typedef-decl name='__uint32_t' type-id='type-id-16' filepath='/usr/include/bits/types.h' line='42' column='1' id='type-id-32'/>
+    <typedef-decl name='__uint64_t' type-id='type-id-3' filepath='/usr/include/bits/types.h' line='45' column='1' id='type-id-33'/>
+    <typedef-decl name='__off_t' type-id='type-id-13' filepath='/usr/include/bits/types.h' line='152' column='1' id='type-id-34'/>
+    <typedef-decl name='__off64_t' type-id='type-id-13' filepath='/usr/include/bits/types.h' line='153' column='1' id='type-id-35'/>
+    <typedef-decl name='FILE' type-id='type-id-36' filepath='/usr/include/bits/types/FILE.h' line='7' column='1' id='type-id-37'/>
+    <typedef-decl name='_IO_lock_t' type-id='type-id-19' filepath='/usr/include/bits/types/struct_FILE.h' line='43' column='1' id='type-id-38'/>
+    <class-decl name='_IO_FILE' size-in-bits='1728' is-struct='yes' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='49' column='1' id='type-id-36'>
+      <data-member access='public' layout-offset-in-bits='0'>
+        <var-decl name='_flags' type-id='type-id-12' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='51' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='64'>
+        <var-decl name='_IO_read_ptr' type-id='type-id-39' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='54' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='128'>
+        <var-decl name='_IO_read_end' type-id='type-id-39' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='55' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='192'>
+        <var-decl name='_IO_read_base' type-id='type-id-39' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='56' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='256'>
+        <var-decl name='_IO_write_base' type-id='type-id-39' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='57' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='320'>
+        <var-decl name='_IO_write_ptr' type-id='type-id-39' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='58' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='384'>
+        <var-decl name='_IO_write_end' type-id='type-id-39' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='59' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='448'>
+        <var-decl name='_IO_buf_base' type-id='type-id-39' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='60' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='512'>
+        <var-decl name='_IO_buf_end' type-id='type-id-39' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='61' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='576'>
+        <var-decl name='_IO_save_base' type-id='type-id-39' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='64' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='640'>
+        <var-decl name='_IO_backup_base' type-id='type-id-39' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='65' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='704'>
+        <var-decl name='_IO_save_end' type-id='type-id-39' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='66' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='768'>
+        <var-decl name='_markers' type-id='type-id-40' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='68' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='832'>
+        <var-decl name='_chain' type-id='type-id-41' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='70' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='896'>
+        <var-decl name='_fileno' type-id='type-id-12' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='72' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='928'>
+        <var-decl name='_flags2' type-id='type-id-12' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='73' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='960'>
+        <var-decl name='_old_offset' type-id='type-id-34' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='74' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='1024'>
+        <var-decl name='_cur_column' type-id='type-id-17' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='77' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='1040'>
+        <var-decl name='_vtable_offset' type-id='type-id-14' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='78' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='1048'>
+        <var-decl name='_shortbuf' type-id='type-id-2' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='79' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='1088'>
+        <var-decl name='_lock' type-id='type-id-42' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='81' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='1152'>
+        <var-decl name='_offset' type-id='type-id-35' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='89' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='1216'>
+        <var-decl name='_codecvt' type-id='type-id-43' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='91' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='1280'>
+        <var-decl name='_wide_data' type-id='type-id-44' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='92' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='1344'>
+        <var-decl name='_freeres_list' type-id='type-id-41' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='93' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='1408'>
+        <var-decl name='_freeres_buf' type-id='type-id-45' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='94' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='1472'>
+        <var-decl name='__pad5' type-id='type-id-46' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='95' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='1536'>
+        <var-decl name='_mode' type-id='type-id-12' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='96' column='1'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='1568'>
+        <var-decl name='_unused2' type-id='type-id-5' visibility='default' filepath='/usr/include/bits/types/struct_FILE.h' line='98' column='1'/>
+      </data-member>
+    </class-decl>
+    <typedef-decl name='size_t' type-id='type-id-3' filepath='/usr/lib/gcc/x86_64-redhat-linux/10/include/stddef.h' line='209' column='1' id='type-id-46'/>
+    <pointer-type-def type-id='type-id-37' size-in-bits='64' id='type-id-47'/>
+    <qualified-type-def type-id='type-id-47' restrict='yes' id='type-id-48'/>
+    <pointer-type-def type-id='type-id-36' size-in-bits='64' id='type-id-41'/>
+    <pointer-type-def type-id='type-id-7' size-in-bits='64' id='type-id-43'/>
+    <pointer-type-def type-id='type-id-38' size-in-bits='64' id='type-id-42'/>
+    <pointer-type-def type-id='type-id-8' size-in-bits='64' id='type-id-40'/>
+    <pointer-type-def type-id='type-id-9' size-in-bits='64' id='type-id-44'/>
+    <pointer-type-def type-id='type-id-1' size-in-bits='64' id='type-id-39'/>
+    <pointer-type-def type-id='type-id-39' size-in-bits='64' id='type-id-49'/>
+    <qualified-type-def type-id='type-id-49' restrict='yes' id='type-id-50'/>
+    <qualified-type-def type-id='type-id-1' const='yes' id='type-id-51'/>
+    <pointer-type-def type-id='type-id-51' size-in-bits='64' id='type-id-52'/>
+    <qualified-type-def type-id='type-id-52' restrict='yes' id='type-id-53'/>
+    <qualified-type-def type-id='type-id-28' const='yes' id='type-id-54'/>
+    <pointer-type-def type-id='type-id-54' size-in-bits='64' id='type-id-55'/>
+    <pointer-type-def type-id='type-id-21' size-in-bits='64' id='type-id-56'/>
+    <pointer-type-def type-id='type-id-28' size-in-bits='64' id='type-id-57'/>
+    <pointer-type-def type-id='type-id-57' size-in-bits='64' id='type-id-58'/>
+    <pointer-type-def type-id='type-id-27' size-in-bits='64' id='type-id-59'/>
+    <pointer-type-def type-id='type-id-19' size-in-bits='64' id='type-id-45'/>
+    <pointer-type-def type-id='type-id-20' size-in-bits='64' id='type-id-60'/>
+    <class-decl name='_IO_codecvt' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-7'/>
+    <class-decl name='_IO_marker' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-8'/>
+    <class-decl name='_IO_wide_data' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-9'/>
+    <class-decl name='libzfs_handle' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-10'/>
+    <class-decl name='zpool_handle' is-struct='yes' visibility='default' is-declaration-only='yes' id='type-id-11'/>
+    <function-decl name='libzfs_init' filepath='../../include/libzfs.h' line='205' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <return type-id='type-id-56'/>
+    </function-decl>
+    <function-decl name='libzfs_fini' filepath='../../include/libzfs.h' line='206' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-56'/>
+      <return type-id='type-id-19'/>
+    </function-decl>
+    <function-decl name='libzfs_error_description' filepath='../../include/libzfs.h' line='219' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-56'/>
+      <return type-id='type-id-52'/>
+    </function-decl>
+    <function-decl name='zpool_open' filepath='../../include/libzfs.h' line='233' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-56'/>
+      <parameter type-id='type-id-52'/>
+      <return type-id='type-id-60'/>
+    </function-decl>
+    <function-decl name='zpool_close' filepath='../../include/libzfs.h' line='235' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-60'/>
+      <return type-id='type-id-19'/>
+    </function-decl>
+    <function-decl name='zpool_set_bootenv' filepath='../../include/libzfs.h' line='895' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-60'/>
+      <parameter type-id='type-id-55'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='zpool_get_bootenv' filepath='../../include/libzfs.h' line='896' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-60'/>
+      <parameter type-id='type-id-58'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_free' filepath='../../include/sys/nvpair.h' line='152' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <return type-id='type-id-19'/>
+    </function-decl>
+    <function-decl name='nvlist_lookup_uint64' filepath='../../include/sys/nvpair.h' line='212' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-59'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_lookup_string' filepath='../../include/sys/nvpair.h' line='213' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-49'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_exists' filepath='../../include/sys/nvpair.h' line='238' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <return type-id='type-id-29'/>
+    </function-decl>
+    <function-decl name='fnvlist_alloc' filepath='../../include/sys/nvpair.h' line='276' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <return type-id='type-id-57'/>
+    </function-decl>
+    <function-decl name='fnvlist_free' filepath='../../include/sys/nvpair.h' line='277' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <return type-id='type-id-19'/>
+    </function-decl>
+    <function-decl name='fnvlist_add_uint64' filepath='../../include/sys/nvpair.h' line='296' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-27'/>
+      <return type-id='type-id-19'/>
+    </function-decl>
+    <function-decl name='fnvlist_add_string' filepath='../../include/sys/nvpair.h' line='297' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-52'/>
+      <return type-id='type-id-19'/>
+    </function-decl>
+    <function-decl name='fnvlist_remove' filepath='../../include/sys/nvpair.h' line='313' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <return type-id='type-id-19'/>
+    </function-decl>
+    <function-decl name='lzbe_set_boot_device' mangled-name='lzbe_set_boot_device' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_device.c' line='28' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_set_boot_device'>
+      <parameter type-id='type-id-52' name='pool' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_device.c' line='28' column='1'/>
+      <parameter type-id='type-id-23' name='flag' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_device.c' line='28' column='1'/>
+      <parameter type-id='type-id-52' name='device' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_device.c' line='28' column='1'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='lzbe_get_boot_device' mangled-name='lzbe_get_boot_device' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_device.c' line='114' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_get_boot_device'>
+      <parameter type-id='type-id-52' name='pool' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_device.c' line='114' column='1'/>
+      <parameter type-id='type-id-49' name='device' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_device.c' line='114' column='1'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='fprintf' filepath='/usr/include/stdio.h' line='326' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-48'/>
+      <parameter type-id='type-id-53'/>
+      <parameter is-variadic='yes'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='asprintf' filepath='/usr/include/stdio.h' line='372' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-50'/>
+      <parameter type-id='type-id-53'/>
+      <parameter is-variadic='yes'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='free' filepath='/usr/include/stdlib.h' line='565' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-45'/>
+      <return type-id='type-id-19'/>
+    </function-decl>
+    <function-decl name='strncmp' filepath='/usr/include/string.h' line='143' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-46'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='strdup' filepath='/usr/include/string.h' line='171' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-52'/>
+      <return type-id='type-id-39'/>
+    </function-decl>
+    <function-decl name='strlen' filepath='/usr/include/string.h' line='391' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-52'/>
+      <return type-id='type-id-46'/>
+    </function-decl>
+  </abi-instr>
+  <abi-instr address-size='64' path='lzbe_pair.c' comp-dir-path='/home/hutter2/release20/lib/libzfsbootenv' language='LANG_C99'>
+    <type-decl name='short int' size-in-bits='16' id='type-id-61'/>
+    <type-decl name='unsigned char' size-in-bits='8' id='type-id-62'/>
+    <typedef-decl name='uchar_t' type-id='type-id-62' filepath='../../lib/libspl/include/sys/stdtypes.h' line='31' column='1' id='type-id-63'/>
+    <typedef-decl name='uint_t' type-id='type-id-16' filepath='../../lib/libspl/include/sys/stdtypes.h' line='33' column='1' id='type-id-64'/>
+    <typedef-decl name='int8_t' type-id='type-id-65' filepath='/usr/include/bits/stdint-intn.h' line='24' column='1' id='type-id-66'/>
+    <typedef-decl name='int16_t' type-id='type-id-67' filepath='/usr/include/bits/stdint-intn.h' line='25' column='1' id='type-id-68'/>
+    <typedef-decl name='int64_t' type-id='type-id-69' filepath='/usr/include/bits/stdint-intn.h' line='27' column='1' id='type-id-70'/>
+    <typedef-decl name='uint8_t' type-id='type-id-71' filepath='/usr/include/bits/stdint-uintn.h' line='24' column='1' id='type-id-72'/>
+    <typedef-decl name='uint16_t' type-id='type-id-73' filepath='/usr/include/bits/stdint-uintn.h' line='25' column='1' id='type-id-74'/>
+    <typedef-decl name='__int8_t' type-id='type-id-14' filepath='/usr/include/bits/types.h' line='37' column='1' id='type-id-65'/>
+    <typedef-decl name='__uint8_t' type-id='type-id-62' filepath='/usr/include/bits/types.h' line='38' column='1' id='type-id-71'/>
+    <typedef-decl name='__int16_t' type-id='type-id-61' filepath='/usr/include/bits/types.h' line='39' column='1' id='type-id-67'/>
+    <typedef-decl name='__uint16_t' type-id='type-id-17' filepath='/usr/include/bits/types.h' line='40' column='1' id='type-id-73'/>
+    <typedef-decl name='__int64_t' type-id='type-id-13' filepath='/usr/include/bits/types.h' line='44' column='1' id='type-id-69'/>
+    <pointer-type-def type-id='type-id-29' size-in-bits='64' id='type-id-75'/>
+    <qualified-type-def type-id='type-id-39' const='yes' id='type-id-76'/>
+    <pointer-type-def type-id='type-id-76' size-in-bits='64' id='type-id-77'/>
+    <pointer-type-def type-id='type-id-68' size-in-bits='64' id='type-id-78'/>
+    <pointer-type-def type-id='type-id-25' size-in-bits='64' id='type-id-79'/>
+    <pointer-type-def type-id='type-id-70' size-in-bits='64' id='type-id-80'/>
+    <pointer-type-def type-id='type-id-66' size-in-bits='64' id='type-id-81'/>
+    <pointer-type-def type-id='type-id-63' size-in-bits='64' id='type-id-82'/>
+    <pointer-type-def type-id='type-id-74' size-in-bits='64' id='type-id-83'/>
+    <pointer-type-def type-id='type-id-26' size-in-bits='64' id='type-id-84'/>
+    <pointer-type-def type-id='type-id-72' size-in-bits='64' id='type-id-85'/>
+    <pointer-type-def type-id='type-id-45' size-in-bits='64' id='type-id-86'/>
+    <function-decl name='nvlist_alloc' filepath='../../include/sys/nvpair.h' line='151' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-58'/>
+      <parameter type-id='type-id-64'/>
+      <parameter type-id='type-id-12'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_dup' filepath='../../include/sys/nvpair.h' line='156' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-58'/>
+      <parameter type-id='type-id-12'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_boolean_value' filepath='../../include/sys/nvpair.h' line='169' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-29'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_byte' filepath='../../include/sys/nvpair.h' line='170' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-63'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_int8' filepath='../../include/sys/nvpair.h' line='171' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-66'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_uint8' filepath='../../include/sys/nvpair.h' line='172' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-72'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_int16' filepath='../../include/sys/nvpair.h' line='173' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-68'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_uint16' filepath='../../include/sys/nvpair.h' line='174' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-74'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_int32' filepath='../../include/sys/nvpair.h' line='175' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-25'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_uint32' filepath='../../include/sys/nvpair.h' line='176' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-26'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_int64' filepath='../../include/sys/nvpair.h' line='177' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-70'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_uint64' filepath='../../include/sys/nvpair.h' line='178' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-27'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_string' filepath='../../include/sys/nvpair.h' line='179' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-52'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_nvlist' filepath='../../include/sys/nvpair.h' line='180' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-57'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_boolean_array' filepath='../../include/sys/nvpair.h' line='181' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-75'/>
+      <parameter type-id='type-id-64'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_byte_array' filepath='../../include/sys/nvpair.h' line='182' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-82'/>
+      <parameter type-id='type-id-64'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_int8_array' filepath='../../include/sys/nvpair.h' line='183' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-81'/>
+      <parameter type-id='type-id-64'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_uint8_array' filepath='../../include/sys/nvpair.h' line='184' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-85'/>
+      <parameter type-id='type-id-64'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_int16_array' filepath='../../include/sys/nvpair.h' line='185' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-78'/>
+      <parameter type-id='type-id-64'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_uint16_array' filepath='../../include/sys/nvpair.h' line='186' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-83'/>
+      <parameter type-id='type-id-64'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_int32_array' filepath='../../include/sys/nvpair.h' line='187' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-79'/>
+      <parameter type-id='type-id-64'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_uint32_array' filepath='../../include/sys/nvpair.h' line='188' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-84'/>
+      <parameter type-id='type-id-64'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_int64_array' filepath='../../include/sys/nvpair.h' line='189' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-80'/>
+      <parameter type-id='type-id-64'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_uint64_array' filepath='../../include/sys/nvpair.h' line='190' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-59'/>
+      <parameter type-id='type-id-64'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_string_array' filepath='../../include/sys/nvpair.h' line='191' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-77'/>
+      <parameter type-id='type-id-64'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_add_nvlist_array' filepath='../../include/sys/nvpair.h' line='192' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-58'/>
+      <parameter type-id='type-id-64'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_remove_all' filepath='../../include/sys/nvpair.h' line='199' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='nvlist_lookup_nvlist' filepath='../../include/sys/nvpair.h' line='214' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-57'/>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-58'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='lzbe_nvlist_get' mangled-name='lzbe_nvlist_get' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='27' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_nvlist_get'>
+      <parameter type-id='type-id-52' name='pool' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='27' column='1'/>
+      <parameter type-id='type-id-52' name='key' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='27' column='1'/>
+      <parameter type-id='type-id-86' name='ptr' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='27' column='1'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='lzbe_nvlist_set' mangled-name='lzbe_nvlist_set' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='74' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_nvlist_set'>
+      <parameter type-id='type-id-52' name='pool' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='74' column='1'/>
+      <parameter type-id='type-id-52' name='key' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='74' column='1'/>
+      <parameter type-id='type-id-45' name='ptr' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='74' column='1'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='lzbe_nvlist_free' mangled-name='lzbe_nvlist_free' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='131' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_nvlist_free'>
+      <parameter type-id='type-id-45' name='ptr' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='131' column='1'/>
+      <return type-id='type-id-19'/>
+    </function-decl>
+    <function-decl name='lzbe_add_pair' mangled-name='lzbe_add_pair' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='182' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_add_pair'>
+      <parameter type-id='type-id-45' name='ptr' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='182' column='1'/>
+      <parameter type-id='type-id-52' name='key' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='182' column='1'/>
+      <parameter type-id='type-id-52' name='type' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='182' column='1'/>
+      <parameter type-id='type-id-45' name='value' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='182' column='1'/>
+      <parameter type-id='type-id-46' name='size' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='183' column='1'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='lzbe_remove_pair' mangled-name='lzbe_remove_pair' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='343' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_remove_pair'>
+      <parameter type-id='type-id-45' name='ptr' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='343' column='1'/>
+      <parameter type-id='type-id-52' name='key' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_pair.c' line='343' column='1'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+    <function-decl name='strcmp' filepath='/usr/include/string.h' line='140' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-52'/>
+      <parameter type-id='type-id-52'/>
+      <return type-id='type-id-12'/>
+    </function-decl>
+  </abi-instr>
+  <abi-instr address-size='64' path='lzbe_util.c' comp-dir-path='/home/hutter2/release20/lib/libzfsbootenv' language='LANG_C99'>
+    <function-decl name='nvlist_print' filepath='../../include/libnvpair.h' line='49' column='1' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='type-id-47'/>
+      <parameter type-id='type-id-57'/>
+      <return type-id='type-id-19'/>
+    </function-decl>
+    <function-decl name='lzbe_bootenv_print' mangled-name='lzbe_bootenv_print' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_util.c' line='24' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzbe_bootenv_print'>
+      <parameter type-id='type-id-52' name='pool' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_util.c' line='24' column='1'/>
+      <parameter type-id='type-id-52' name='nvlist' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_util.c' line='24' column='1'/>
+      <parameter type-id='type-id-47' name='of' filepath='/home/hutter2/release20/lib/libzfsbootenv/lzbe_util.c' line='24' column='1'/>
+      <return type-id='type-id-12'/>
     </function-decl>
   </abi-instr>
 </abi-corpus>

lib/libzfsbootenv/lzbe_device.c

@@ -63,7 +63,7 @@ lzbe_set_boot_device(const char *pool, lzbe_flags_t flag, const char *device)
 			/* Drop this nvlist */
 			fnvlist_free(nv);
 		}
-		/* FALLTHROUGH */
+		fallthrough;
 	case lzbe_replace:
 		nv = fnvlist_alloc();
 		break;

lib/libzpool/kernel.c

@@ -966,16 +966,16 @@ kmem_asprintf(const char *fmt, ...)
 }
 
 /* ARGSUSED */
-int
+zfs_file_t *
 zfs_onexit_fd_hold(int fd, minor_t *minorp)
 {
 	*minorp = 0;
-	return (0);
+	return (NULL);
 }
 
 /* ARGSUSED */
 void
-zfs_onexit_fd_rele(int fd)
+zfs_onexit_fd_rele(zfs_file_t *fp)
 {
 }
 
@@ -1385,28 +1385,26 @@ zfs_file_unlink(const char *path)
  * Get reference to file pointer
  *
  * fd - input file descriptor
- * fpp - pointer to file pointer
  *
- * Returns 0 on success EBADF on failure.
+ * Returns pointer to file struct or NULL.
  * Unsupported in user space.
  */
-int
-zfs_file_get(int fd, zfs_file_t **fpp)
+zfs_file_t *
+zfs_file_get(int fd)
 {
 	abort();
 
-	return (EOPNOTSUPP);
+	return (NULL);
 }
-
 /*
  * Drop reference to file pointer
  *
- * fd - input file descriptor
+ * fp - pointer to file struct
  *
  * Unsupported in user space.
  */
 void
-zfs_file_put(int fd)
+zfs_file_put(zfs_file_t *fp)
 {
 	abort();
 }

lib/libzutil/os/freebsd/zutil_import_os.c

@@ -237,3 +237,8 @@ zfs_dev_flush(int fd __unused)
 {
 	return (0);
 }
+
+void
+update_vdevs_config_dev_sysfs_path(nvlist_t *config)
+{
+}

lib/libzutil/os/linux/zutil_device_path_os.c

@@ -154,15 +154,260 @@ zfs_strip_path(char *path)
 	return (strrchr(path, '/') + 1);
 }
 
+/*
+ * Read the contents of a sysfs file into an allocated buffer and remove the
+ * last newline.
+ *
+ * This is useful for reading sysfs files that return a single string.  Return
+ * an allocated string pointer on success, NULL otherwise.  Returned buffer
+ * must be freed by the user.
+ */
+static char *
+zfs_read_sysfs_file(char *filepath)
+{
+	char buf[4096];	/* all sysfs files report 4k size */
+	char *str = NULL;
+
+	FILE *fp = fopen(filepath, "r");
+	if (fp == NULL) {
+		return (NULL);
+	}
+	if (fgets(buf, sizeof (buf), fp) == buf) {
+		/* success */
+
+		/* Remove the last newline (if any) */
+		size_t len = strlen(buf);
+		if (buf[len - 1] == '\n') {
+			buf[len - 1] = '\0';
+		}
+		str = strdup(buf);
+	}
+
+	fclose(fp);
+
+	return (str);
+}
+
+/*
+ * Given a dev name like "nvme0n1", return the full PCI slot sysfs path to
+ * the drive (in /sys/bus/pci/slots).
+ *
+ * For example:
+ *     dev:            "nvme0n1"
+ *     returns:        "/sys/bus/pci/slots/0"
+ *
+ * 'dev' must be an NVMe device.
+ *
+ * Returned string must be freed.  Returns NULL on error or no sysfs path.
+ */
+static char *
+zfs_get_pci_slots_sys_path(const char *dev_name)
+{
+	DIR *dp = NULL;
+	struct dirent *ep;
+	char *address1 = NULL;
+	char *address2 = NULL;
+	char *path = NULL;
+	char buf[MAXPATHLEN];
+	char *tmp;
+
+	/* If they preface 'dev' with a path (like "/dev") then strip it off */
+	tmp = strrchr(dev_name, '/');
+	if (tmp != NULL)
+		dev_name = tmp + 1;    /* +1 since we want the chr after '/' */
+
+	if (strncmp("nvme", dev_name, 4) != 0)
+		return (NULL);
+
+	(void) snprintf(buf, sizeof (buf), "/sys/block/%s/device/address",
+	    dev_name);
+
+	address1 = zfs_read_sysfs_file(buf);
+	if (!address1)
+		return (NULL);
+
+	/*
+	 * /sys/block/nvme0n1/device/address format will
+	 * be "0000:01:00.0" while /sys/bus/pci/slots/0/address will be
+	 * "0000:01:00".  Just NULL terminate at the '.' so they match.
+	 */
+	tmp = strrchr(address1, '.');
+	if (tmp != NULL)
+		*tmp = '\0';
+
+	dp = opendir("/sys/bus/pci/slots/");
+	if (dp == NULL) {
+		free(address1);
+		return (NULL);
+	}
+
+	/*
+	 * Look through all the /sys/bus/pci/slots/ subdirs
+	 */
+	while ((ep = readdir(dp))) {
+		/*
+		 * We only care about directory names that are a single number.
+		 * Sometimes there's other directories like
+		 * "/sys/bus/pci/slots/0-3/" in there - skip those.
+		 */
+		if (!zfs_isnumber(ep->d_name))
+			continue;
+
+		(void) snprintf(buf, sizeof (buf),
+		    "/sys/bus/pci/slots/%s/address", ep->d_name);
+
+		address2 = zfs_read_sysfs_file(buf);
+		if (!address2)
+			continue;
+
+		if (strcmp(address1, address2) == 0) {
+			/* Addresses match, we're all done */
+			free(address2);
+			if (asprintf(&path, "/sys/bus/pci/slots/%s",
+			    ep->d_name) == -1) {
+				free(tmp);
+				continue;
+			}
+			break;
+		}
+		free(address2);
+	}
+
+	closedir(dp);
+	free(address1);
+
+	return (path);
+}
+
+/*
+ * Given a dev name like "sda", return the full enclosure sysfs path to
+ * the disk.  You can also pass in the name with "/dev" prepended
+ * to it (like /dev/sda).  This works for both JBODs and NVMe PCI devices.
+ *
+ * For example, disk "sda" in enclosure slot 1:
+ *     dev_name:       "sda"
+ *     returns:        "/sys/class/enclosure/1:0:3:0/Slot 1"
+ *
+ * Or:
+ *
+ *      dev_name:   "nvme0n1"
+ *      returns:    "/sys/bus/pci/slots/0"
+ *
+ * 'dev' must be a non-devicemapper device.
+ *
+ * Returned string must be freed.  Returns NULL on error.
+ */
+char *
+zfs_get_enclosure_sysfs_path(const char *dev_name)
+{
+	DIR *dp = NULL;
+	struct dirent *ep;
+	char buf[MAXPATHLEN];
+	char *tmp1 = NULL;
+	char *tmp2 = NULL;
+	char *tmp3 = NULL;
+	char *path = NULL;
+	size_t size;
+	int tmpsize;
+
+	if (dev_name == NULL)
+		return (NULL);
+
+	/* If they preface 'dev' with a path (like "/dev") then strip it off */
+	tmp1 = strrchr(dev_name, '/');
+	if (tmp1 != NULL)
+		dev_name = tmp1 + 1;    /* +1 since we want the chr after '/' */
+
+	tmpsize = asprintf(&tmp1, "/sys/block/%s/device", dev_name);
+	if (tmpsize == -1 || tmp1 == NULL) {
+		tmp1 = NULL;
+		goto end;
+	}
+
+	dp = opendir(tmp1);
+	if (dp == NULL)
+		goto end;
+
+	/*
+	 * Look though all sysfs entries in /sys/block/<dev>/device for
+	 * the enclosure symlink.
+	 */
+	while ((ep = readdir(dp))) {
+		/* Ignore everything that's not our enclosure_device link */
+		if (strstr(ep->d_name, "enclosure_device") == NULL)
+			continue;
+
+		if (asprintf(&tmp2, "%s/%s", tmp1, ep->d_name) == -1) {
+			tmp2 = NULL;
+			break;
+		}
+
+		size = readlink(tmp2, buf, sizeof (buf));
+
+		/* Did readlink fail or crop the link name? */
+		if (size == -1 || size >= sizeof (buf))
+			break;
+
+		/*
+		 * We got a valid link.  readlink() doesn't terminate strings
+		 * so we have to do it.
+		 */
+		buf[size] = '\0';
+
+		/*
+		 * Our link will look like:
+		 *
+		 * "../../../../port-11:1:2/..STUFF../enclosure/1:0:3:0/SLOT 1"
+		 *
+		 * We want to grab the "enclosure/1:0:3:0/SLOT 1" part
+		 */
+		tmp3 = strstr(buf, "enclosure");
+		if (tmp3 == NULL)
+			break;
+
+		if (asprintf(&path, "/sys/class/%s", tmp3) == -1) {
+			/* If asprintf() fails, 'path' is undefined */
+			path = NULL;
+			break;
+		}
+
+		if (path == NULL)
+			break;
+	}
+
+end:
+	free(tmp2);
+	free(tmp1);
+
+	if (dp != NULL)
+		closedir(dp);
+
+	if (!path) {
+		/*
+		 * This particular disk isn't in a JBOD.  It could be an NVMe
+		 * drive. If so, look up the NVMe device's path in
+		 * /sys/bus/pci/slots/. Within that directory is a 'attention'
+		 * file which controls the NVMe fault LED.
+		 */
+		path = zfs_get_pci_slots_sys_path(dev_name);
+	}
+
+	return (path);
+}
+
 /*
  * Allocate and return the underlying device name for a device mapper device.
- * If a device mapper device maps to multiple devices, return the first device.
  *
  * For example, dm_name = "/dev/dm-0" could return "/dev/sda". Symlinks to a
  * DM device (like /dev/disk/by-vdev/A0) are also allowed.
  *
- * Returns device name, or NULL on error or no match.  If dm_name is not a DM
- * device then return NULL.
+ * If the DM device has multiple underlying devices (like with multipath
+ * DM devices), then favor underlying devices that have a symlink back to their
+ * back to their enclosure device in sysfs.  This will be useful for the
+ * zedlet scripts that toggle the fault LED.
+ *
+ * Returns an underlying device name, or NULL on error or no match.  If dm_name
+ * is not a DM device then return NULL.
  *
  * NOTE: The returned name string must be *freed*.
  */
@@ -176,6 +421,8 @@ dm_get_underlying_path(const char *dm_name)
 	char *path = NULL;
 	char *dev_str;
 	int size;
+	char *first_path = NULL;
+	char *enclosure_path;
 
 	if (dm_name == NULL)
 		return (NULL);
@@ -204,13 +451,27 @@ dm_get_underlying_path(const char *dm_name)
 		goto end;
 
 	/*
-	 * Return first entry (that isn't itself a directory) in the
-	 * directory containing device-mapper dependent (underlying)
-	 * devices.
+	 * A device-mapper device can have multiple paths to it (multipath).
+	 * Favor paths that have a symlink back to their enclosure device.
+	 * We have to do this since some enclosures may only provide a symlink
+	 * back for one underlying path to a disk and not the other.
+	 *
+	 * If no paths have links back to their enclosure, then just return the
+	 * first path.
 	 */
 	while ((ep = readdir(dp))) {
 		if (ep->d_type != DT_DIR) {	/* skip "." and ".." dirs */
+			if (!first_path)
+				first_path = strdup(ep->d_name);
+
+			enclosure_path =
+			    zfs_get_enclosure_sysfs_path(ep->d_name);
+
+			if (!enclosure_path)
+				continue;
+
 			size = asprintf(&path, "/dev/%s", ep->d_name);
+			free(enclosure_path);
 			break;
 		}
 	}
@@ -220,6 +481,17 @@ end:
 		closedir(dp);
 	free(tmp);
 	free(realp);
+
+	if (!path) {
+		/*
+		 * None of the underlying paths had a link back to their
+		 * enclosure devices.  Throw up out hands and return the first
+		 * underlying path.
+		 */
+		size = asprintf(&path, "/dev/%s", first_path);
+	}
+
+	free(first_path);
 	return (path);
 }
 
@@ -331,110 +603,6 @@ zfs_get_underlying_path(const char *dev_name)
 	return (name);
 }
 
-/*
- * Given a dev name like "sda", return the full enclosure sysfs path to
- * the disk.  You can also pass in the name with "/dev" prepended
- * to it (like /dev/sda).
- *
- * For example, disk "sda" in enclosure slot 1:
- *     dev:            "sda"
- *     returns:        "/sys/class/enclosure/1:0:3:0/Slot 1"
- *
- * 'dev' must be a non-devicemapper device.
- *
- * Returned string must be freed.
- */
-char *
-zfs_get_enclosure_sysfs_path(const char *dev_name)
-{
-	DIR *dp = NULL;
-	struct dirent *ep;
-	char buf[MAXPATHLEN];
-	char *tmp1 = NULL;
-	char *tmp2 = NULL;
-	char *tmp3 = NULL;
-	char *path = NULL;
-	size_t size;
-	int tmpsize;
-
-	if (dev_name == NULL)
-		return (NULL);
-
-	/* If they preface 'dev' with a path (like "/dev") then strip it off */
-	tmp1 = strrchr(dev_name, '/');
-	if (tmp1 != NULL)
-		dev_name = tmp1 + 1;    /* +1 since we want the chr after '/' */
-
-	tmpsize = asprintf(&tmp1, "/sys/block/%s/device", dev_name);
-	if (tmpsize == -1 || tmp1 == NULL) {
-		tmp1 = NULL;
-		goto end;
-	}
-
-	dp = opendir(tmp1);
-	if (dp == NULL) {
-		tmp1 = NULL;	/* To make free() at the end a NOP */
-		goto end;
-	}
-
-	/*
-	 * Look though all sysfs entries in /sys/block/<dev>/device for
-	 * the enclosure symlink.
-	 */
-	while ((ep = readdir(dp))) {
-		/* Ignore everything that's not our enclosure_device link */
-		if (strstr(ep->d_name, "enclosure_device") == NULL)
-			continue;
-
-		if (asprintf(&tmp2, "%s/%s", tmp1, ep->d_name) == -1 ||
-		    tmp2 == NULL)
-			break;
-
-		size = readlink(tmp2, buf, sizeof (buf));
-
-		/* Did readlink fail or crop the link name? */
-		if (size == -1 || size >= sizeof (buf)) {
-			free(tmp2);
-			tmp2 = NULL;	/* To make free() at the end a NOP */
-			break;
-		}
-
-		/*
-		 * We got a valid link.  readlink() doesn't terminate strings
-		 * so we have to do it.
-		 */
-		buf[size] = '\0';
-
-		/*
-		 * Our link will look like:
-		 *
-		 * "../../../../port-11:1:2/..STUFF../enclosure/1:0:3:0/SLOT 1"
-		 *
-		 * We want to grab the "enclosure/1:0:3:0/SLOT 1" part
-		 */
-		tmp3 = strstr(buf, "enclosure");
-		if (tmp3 == NULL)
-			break;
-
-		if (asprintf(&path, "/sys/class/%s", tmp3) == -1) {
-			/* If asprintf() fails, 'path' is undefined */
-			path = NULL;
-			break;
-		}
-
-		if (path == NULL)
-			break;
-	}
-
-end:
-	free(tmp2);
-	free(tmp1);
-
-	if (dp != NULL)
-		closedir(dp);
-
-	return (path);
-}
 
 #ifdef HAVE_LIBUDEV
 

lib/libzutil/os/linux/zutil_import_os.c

@@ -66,6 +66,7 @@
 #include <thread_pool.h>
 #include <libzutil.h>
 #include <libnvpair.h>
+#include <libzfs.h>
 
 #include "zutil_import.h"
 
@@ -777,6 +778,58 @@ no_dev:
 #endif
 }
 
+/*
+ * Rescan the enclosure sysfs path for turning on enclosure LEDs and store it
+ * in the nvlist * (if applicable).  Like:
+ *    vdev_enc_sysfs_path: '/sys/class/enclosure/11:0:1:0/SLOT 4'
+ */
+static void
+update_vdev_config_dev_sysfs_path(nvlist_t *nv, char *path)
+{
+	char *upath, *spath;
+
+	/* Add enclosure sysfs path (if disk is in an enclosure). */
+	upath = zfs_get_underlying_path(path);
+	spath = zfs_get_enclosure_sysfs_path(upath);
+
+	if (spath) {
+		nvlist_add_string(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH, spath);
+	} else {
+		nvlist_remove_all(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
+	}
+
+	free(upath);
+	free(spath);
+}
+
+/*
+ * This will get called for each leaf vdev.
+ */
+static int
+sysfs_path_pool_vdev_iter_f(void *hdl_data, nvlist_t *nv, void *data)
+{
+	char *path = NULL;
+	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) != 0)
+		return (1);
+
+	/* Rescan our enclosure sysfs path for this vdev */
+	update_vdev_config_dev_sysfs_path(nv, path);
+	return (0);
+}
+
+/*
+ * Given an nvlist for our pool (with vdev tree), iterate over all the
+ * leaf vdevs and update their ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH.
+ */
+void
+update_vdevs_config_dev_sysfs_path(nvlist_t *config)
+{
+	nvlist_t *nvroot = NULL;
+	verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
+	    &nvroot) == 0);
+	for_each_vdev_in_nvlist(nvroot, sysfs_path_pool_vdev_iter_f, NULL);
+}
+
 /*
  * Update a leaf vdev's persistent device strings
  *
@@ -803,7 +856,6 @@ update_vdev_config_dev_strs(nvlist_t *nv)
 	vdev_dev_strs_t vds;
 	char *env, *type, *path;
 	uint64_t wholedisk = 0;
-	char *upath, *spath;
 
 	/*
 	 * For the benefit of legacy ZFS implementations, allow
@@ -850,18 +902,7 @@ update_vdev_config_dev_strs(nvlist_t *nv)
 			(void) nvlist_add_string(nv, ZPOOL_CONFIG_PHYS_PATH,
 			    vds.vds_devphys);
 		}
-
-		/* Add enclosure sysfs path (if disk is in an enclosure). */
-		upath = zfs_get_underlying_path(path);
-		spath = zfs_get_enclosure_sysfs_path(upath);
-		if (spath)
-			nvlist_add_string(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH,
-			    spath);
-		else
-			nvlist_remove_all(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
-
-		free(upath);
-		free(spath);
+		update_vdev_config_dev_sysfs_path(nv, path);
 	} else {
 		/* Clear out any stale entries. */
 		(void) nvlist_remove_all(nv, ZPOOL_CONFIG_DEVID);

lib/libzutil/zutil_import.c

@@ -1534,6 +1534,8 @@ zpool_find_import_cached(libpc_handle_t *hdl, importargs_t *iarg)
 			return (NULL);
 		}
 
+		update_vdevs_config_dev_sysfs_path(src);
+
 		if ((dst = zutil_refresh_config(hdl, src)) == NULL) {
 			nvlist_free(raw);
 			nvlist_free(pools);
@@ -1691,3 +1693,69 @@ zpool_find_config(void *hdl, const char *target, nvlist_t **configp,
 
 	return (0);
 }
+
+/*
+ * Internal function for iterating over the vdevs.
+ *
+ * For each vdev, func() will be called and will be passed 'zhp' (which is
+ * typically the zpool_handle_t cast as a void pointer), the vdev's nvlist, and
+ * a user-defined data pointer).
+ *
+ * The return values from all the func() calls will be OR'd together and
+ * returned.
+ */
+int
+for_each_vdev_cb(void *zhp, nvlist_t *nv, pool_vdev_iter_f func,
+    void *data)
+{
+	nvlist_t **child;
+	uint_t c, children;
+	int ret = 0;
+	int i;
+	char *type;
+
+	const char *list[] = {
+	    ZPOOL_CONFIG_SPARES,
+	    ZPOOL_CONFIG_L2CACHE,
+	    ZPOOL_CONFIG_CHILDREN
+	};
+
+	for (i = 0; i < ARRAY_SIZE(list); i++) {
+		if (nvlist_lookup_nvlist_array(nv, list[i], &child,
+		    &children) == 0) {
+			for (c = 0; c < children; c++) {
+				uint64_t ishole = 0;
+
+				(void) nvlist_lookup_uint64(child[c],
+				    ZPOOL_CONFIG_IS_HOLE, &ishole);
+
+				if (ishole)
+					continue;
+
+				ret |= for_each_vdev_cb(zhp, child[c],
+				    func, data);
+			}
+		}
+	}
+
+	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) != 0)
+		return (ret);
+
+	/* Don't run our function on root vdevs */
+	if (strcmp(type, VDEV_TYPE_ROOT) != 0) {
+		ret |= func(zhp, nv, data);
+	}
+
+	return (ret);
+}
+
+/*
+ * Given an ZPOOL_CONFIG_VDEV_TREE nvpair, iterate over all the vdevs, calling
+ * func() for each one.  func() is passed the vdev's nvlist and an optional
+ * user-defined 'data' pointer.
+ */
+int
+for_each_vdev_in_nvlist(nvlist_t *nvroot, pool_vdev_iter_f func, void *data)
+{
+	return (for_each_vdev_cb(NULL, nvroot, func, data));
+}

lib/libzutil/zutil_nicenum.c

@@ -27,6 +27,7 @@
 #include <math.h>
 #include <stdio.h>
 #include <libzutil.h>
+#include <string.h>
 
 /*
  * Return B_TRUE if "str" is a number string, B_FALSE otherwise.
@@ -42,6 +43,14 @@ zfs_isnumber(const char *str)
 		if (!(isdigit(*str) || (*str == '.')))
 			return (B_FALSE);
 
+	/*
+	 * Numbers should not end with a period ("." ".." or "5." are
+	 * not valid)
+	 */
+	if (str[strlen(str) - 1] == '.') {
+		return (B_FALSE);
+	}
+
 	return (B_TRUE);
 }
 

man/man5/vdev_id.conf.5

@@ -1,222 +1,251 @@
-.TH VDEV_ID.CONF 5 "Aug 24, 2020" OpenZFS
-.SH NAME
-vdev_id.conf \- Configuration file for vdev_id
-.SH DESCRIPTION
-.I vdev_id.conf
+.\"
+.\" This file and its contents are supplied under the terms of the
+.\" Common Development and Distribution License ("CDDL"), version 1.0.
+.\" You may only use this file in accordance with the terms of version
+.\" 1.0 of the CDDL.
+.\"
+.\" A full copy of the text of the CDDL should have accompanied this
+.\" source.  A copy of the CDDL is also available via the Internet at
+.\" http://www.illumos.org/license/CDDL.
+.\"
+.Dd May 26, 2021
+.Dt VDEV_ID.CONF 5
+.Os
+.
+.Sh NAME
+.Nm vdev_id.conf
+.Nd Configuration file for vdev_id
+.Sh DESCRIPTION
+.Nm
 is the configuration file for
-.BR vdev_id (8).
+.Nm vdev_id Ns Sy (8) .
 It controls the default behavior of
-.BR vdev_id (8)
+.Nm vdev_id Ns Sy (8)
 while it is mapping a disk device name to an alias.
-.PP
+.Pp
 The
-.I vdev_id.conf
+.Nm
 file uses a simple format consisting of a keyword followed by one or
-more values on a single line.  Any line not beginning with a recognized
-keyword is ignored.  Comments may optionally begin with a hash
-character.
-
+more values on a single line.
+Any line not beginning with a recognized keyword is ignored.
+Comments may optionally begin with a hash character.
+.Pp
 The following keywords and values are used.
-.TP
-\fIalias\fR <name> <devlink>
-Maps a device link in the /dev directory hierarchy to a new device
-name.  The udev rule defining the device link must have run prior to
-.BR vdev_id (8).
+.Bl -tag -width "-h"
+.It Sy alias Ar name Ar devlink
+Maps a device link in the
+.Pa /dev
+directory hierarchy to a new device name.
+The udev rule defining the device link must have run prior to
+.Nm vdev_id Ns Sy (8) .
 A defined alias takes precedence over a topology-derived name, but the
-two naming methods can otherwise coexist.  For example, one might name
-drives in a JBOD with the sas_direct topology while naming an internal
-L2ARC device with an alias.
-
-\fIname\fR - the name of the link to the device that will by created in
-/dev/disk/by-vdev.
-
-\fIdevlink\fR - the name of the device link that has already been
-defined by udev.  This may be an absolute path or the base filename.
-
-.TP
-\fIchannel\fR [pci_slot] <port> <name>
+two naming methods can otherwise coexist.
+For example, one might name drives in a JBOD with the
+.Sy sas_direct
+topology while naming an internal L2ARC device with an alias.
+.Pp
+.Ar name
+is the name of the link to the device that will by created under
+.Pa /dev/disk/by-vdev .
+.Pp
+.Ar devlink
+is the name of the device link that has already been
+defined by udev.
+This may be an absolute path or the base filename.
+.
+.It Sy channel [ Ns Ar pci_slot ] Ar port Ar name
 Maps a physical path to a channel name (typically representing a single
 disk enclosure).
-
-.TP
-\fIenclosure_symlinks\fR <yes|no>
-Additionally create /dev/by-enclosure symlinks to the disk enclosure
-sg devices using the naming scheme from vdev_id.conf.
-\fIenclosure_symlinks\fR is only allowed for sas_direct mode.
-.TP
-\fIenclosure_symlinks_prefix\fR <prefix>
-Specify the prefix for the enclosure symlinks in the form of:
-
-/dev/by-enclosure/<prefix>-<channel><num>
-
-Defaults to "enc" if not specified.
-.TP
-\fIpci_slot\fR - specifies the PCI SLOT of the HBA
-hosting the disk enclosure being mapped, as found in the output of
-.BR lspci (8).
-This argument is not used in sas_switch mode.
-
-\fIport\fR - specifies the numeric identifier of the HBA or SAS switch port
-connected to the disk enclosure being mapped.
-
-\fIname\fR - specifies the name of the channel.
-
-.TP
-\fIslot\fR <old> <new> [channel]
+.
+.It Sy enclosure_symlinks Sy yes Ns | Ns Sy no
+Additionally create
+.Pa /dev/by-enclosure
+symlinks to the disk enclosure
+.Em sg
+devices using the naming scheme from
+.Pa vdev_id.conf .
+.Sy enclosure_symlinks
+is only allowed for
+.Sy sas_direct
+mode.
+.
+.It Sy enclosure_symlinks_prefix Ar prefix
+Specify the prefix for the enclosure symlinks in the form
+.Pa /dev/by-enclosure/ Ns Ao Ar prefix Ac Ns - Ns Ao Ar channel Ac Ns Aq Ar num
+.Pp
+Defaults to
+.Dq Em enc .
+.
+.It Sy slot Ar prefix Ar new Op Ar channel
 Maps a disk slot number as reported by the operating system to an
-alternative slot number.  If the \fIchannel\fR parameter is specified
+alternative slot number.
+If the
+.Ar channel
+parameter is specified
 then the mapping is only applied to slots in the named channel,
-otherwise the mapping is applied to all channels. The first-specified
-\fIslot\fR rule that can match a slot takes precedence.  Therefore a
-channel-specific mapping for a given slot should generally appear before
-a generic mapping for the same slot.  In this way a custom mapping may
-be applied to a particular channel and a default mapping applied to the
-others.
-
-.TP
-\fImultipath\fR <yes|no>
+otherwise the mapping is applied to all channels.
+The first-specified
+.Ar slot
+rule that can match a slot takes precedence.
+Therefore a channel-specific mapping for a given slot should generally appear
+before a generic mapping for the same slot.
+In this way a custom mapping may be applied to a particular channel
+and a default mapping applied to the others.
+.
+.It Sy multipath Sy yes Ns | Ns Sy no
 Specifies whether
-.BR vdev_id (8)
-will handle only dm-multipath devices.  If set to "yes" then
-.BR vdev_id (8)
+.Nm vdev_id Ns Sy (8)
+will handle only dm-multipath devices.
+If set to
+.Sy yes
+then
+.Nm vdev_id Ns Sy (8)
 will examine the first running component disk of a dm-multipath
-device as listed by the
-.BR multipath (8)
-command to determine the physical path.
-.TP
-\fItopology\fR <sas_direct|sas_switch>
+device as provided by the driver command to determine the physical path.
+.
+.It Sy topology Sy sas_direct Ns | Ns Sy sas_switch Ns | Ns Sy scsi
 Identifies a physical topology that governs how physical paths are
-mapped to channels.
-
-\fIsas_direct\fR - in this mode a channel is uniquely identified by
-a PCI slot and a HBA port number
-
-\fIsas_switch\fR - in this mode a channel is uniquely identified by
-a SAS switch port number
-
-.TP
-\fIphys_per_port\fR <num>
+mapped to channels:
+.Bl -tag -compact -width "sas_direct and scsi"
+.It Sy sas_direct No and Sy scsi
+channels are uniquely identified by a PCI slot and HBA port number
+.It Sy sas_switch
+channels are uniquely identified by a SAS switch port number
+.El
+.
+.It Sy phys_per_port Ar num
 Specifies the number of PHY devices associated with a SAS HBA port or SAS
 switch port.
-.BR vdev_id (8)
+.Nm vdev_id Ns Sy (8)
 internally uses this value to determine which HBA or switch port a
-device is connected to.  The default is 4.
-
-.TP
-\fIslot\fR <bay|phy|port|id|lun|ses>
+device is connected to.
+The default is
+.Sy 4 .
+.
+.It Sy slot Sy bay Ns | Ns Sy phy Ns | Ns Sy port Ns | Ns Sy id Ns | Ns Sy lun Ns | Ns Sy ses
 Specifies from which element of a SAS identifier the slot number is
-taken.  The default is bay.
-
-\fIbay\fR - read the slot number from the bay identifier.
-
-\fIphy\fR - read the slot number from the phy identifier.
-
-\fIport\fR - use the SAS port as the slot number.
-
-\fIid\fR - use the scsi id as the slot number.
-
-\fIlun\fR - use the scsi lun as the slot number.
-
-\fIses\fR - use the SCSI Enclosure Services (SES) enclosure device slot number,
+taken.
+The default is
+.Sy bay :
+.Bl -tag -compact -width "port"
+.It Sy bay
+read the slot number from the bay identifier.
+.It Sy phy
+read the slot number from the phy identifier.
+.It Sy port
+use the SAS port as the slot number.
+.It Sy id
+use the scsi id as the slot number.
+.It Sy lun
+use the scsi lun as the slot number.
+.It Sy ses
+use the SCSI Enclosure Services (SES) enclosure device slot number,
 as reported by
-.BR sg_ses (8).
-This is intended for use only on systems where \fIbay\fR is unsupported,
-noting that \fIport\fR and \fIid\fR may be unstable across disk replacement.
-.SH EXAMPLES
-A non-multipath configuration with direct-attached SAS enclosures and an
-arbitrary slot re-mapping.
-.P
-.nf
-	multipath     no
-	topology      sas_direct
-	phys_per_port 4
-	slot          bay
-
-	#       PCI_SLOT HBA PORT  CHANNEL NAME
-	channel 85:00.0  1         A
-	channel 85:00.0  0         B
-	channel 86:00.0  1         C
-	channel 86:00.0  0         D
-
-	# Custom mapping for Channel A
-
-	#    Linux      Mapped
-	#    Slot       Slot      Channel
-	slot 1          7         A
-	slot 2          10        A
-	slot 3          3         A
-	slot 4          6         A
-
-	# Default mapping for B, C, and D
-
-	slot 1          4
-	slot 2          2
-	slot 3          1
-	slot 4          3
-.fi
-.P
-A SAS-switch topology.  Note that the
-.I channel
-keyword takes only two arguments in this example.
-.P
-.nf
-	topology      sas_switch
-
-	#       SWITCH PORT  CHANNEL NAME
-	channel 1            A
-	channel 2            B
-	channel 3            C
-	channel 4            D
-.fi
-.P
-A multipath configuration.  Note that channel names have multiple
-definitions - one per physical path.
-.P
-.nf
-	multipath yes
-
-	#       PCI_SLOT HBA PORT  CHANNEL NAME
-	channel 85:00.0  1         A
-	channel 85:00.0  0         B
-	channel 86:00.0  1         A
-	channel 86:00.0  0         B
-.fi
-.P
-A configuration with enclosure_symlinks enabled.
-.P
-.nf
-	multipath yes
-	enclosure_symlinks yes
-
-	#          PCI_ID      HBA PORT     CHANNEL NAME
-	channel    05:00.0     1            U
-	channel    05:00.0     0            L
-	channel    06:00.0     1            U
-	channel    06:00.0     0            L
-.fi
-In addition to the disks symlinks, this configuration will create:
-.P
-.nf
-	/dev/by-enclosure/enc-L0
-	/dev/by-enclosure/enc-L1
-	/dev/by-enclosure/enc-U0
-	/dev/by-enclosure/enc-U1
-.fi
-.P
-A configuration using device link aliases.
-.P
-.nf
-	#     by-vdev
-	#     name     fully qualified or base name of device link
-	alias d1       /dev/disk/by-id/wwn-0x5000c5002de3b9ca
-	alias d2       wwn-0x5000c5002def789e
-.fi
-.P
-
-.SH FILES
-.TP
-.I /etc/zfs/vdev_id.conf
+.Xr sg_ses 8 .
+Intended for use only on systems where
+.Sy bay
+is unsupported,
+noting that
+.Sy port
+and
+.Sy id
+may be unstable across disk replacement.
+.El
+.El
+.
+.Sh FILES
+.Bl -tag -width "-v v"
+.It Pa /etc/zfs/vdev_id.conf
 The configuration file for
-.BR vdev_id (8).
-.SH SEE ALSO
-.BR vdev_id (8)
+.Nm vdev_id Ns Sy (8) .
+.El
+.
+.Sh EXAMPLES
+A non-multipath configuration with direct-attached SAS enclosures and an
+arbitrary slot re-mapping:
+.Bd -literal -offset Ds
+multipath     no
+topology      sas_direct
+phys_per_port 4
+slot          bay
+
+#       PCI_SLOT HBA PORT  CHANNEL NAME
+channel 85:00.0  1         A
+channel 85:00.0  0         B
+channel 86:00.0  1         C
+channel 86:00.0  0         D
+
+# Custom mapping for Channel A
+
+#    Linux      Mapped
+#    Slot       Slot      Channel
+slot 1          7         A
+slot 2          10        A
+slot 3          3         A
+slot 4          6         A
+
+# Default mapping for B, C, and D
+
+slot 1          4
+slot 2          2
+slot 3          1
+slot 4          3
+.Ed
+.Pp
+A SAS-switch topology.
+Note, that the
+.Ar channel
+keyword takes only two arguments in this example.
+.Bd -literal -offset Ds
+topology      sas_switch
+
+#       SWITCH PORT  CHANNEL NAME
+channel 1            A
+channel 2            B
+channel 3            C
+channel 4            D
+.Ed
+.Pp
+A multipath configuration.
+Note that channel names have multiple
+definitions - one per physical path.
+.Bd -literal -offset Ds
+multipath yes
+
+#       PCI_SLOT HBA PORT  CHANNEL NAME
+channel 85:00.0  1         A
+channel 85:00.0  0         B
+channel 86:00.0  1         A
+channel 86:00.0  0         B
+.Ed
+.Pp
+A configuration with enclosure_symlinks enabled.
+.Bd -literal -offset Ds
+multipath yes
+enclosure_symlinks yes
+
+#          PCI_ID      HBA PORT     CHANNEL NAME
+channel    05:00.0     1            U
+channel    05:00.0     0            L
+channel    06:00.0     1            U
+channel    06:00.0     0            L
+.Ed
+.Pp
+In addition to the disks symlinks, this configuration will create:
+.Bd -literal -offset Ds
+/dev/by-enclosure/enc-L0
+/dev/by-enclosure/enc-L1
+/dev/by-enclosure/enc-U0
+/dev/by-enclosure/enc-U1
+.Ed
+.Pp
+A configuration using device link aliases.
+.Bd -literal -offset Ds
+#     by-vdev
+#     name     fully qualified or base name of device link
+alias d1       /dev/disk/by-id/wwn-0x5000c5002de3b9ca
+alias d2       wwn-0x5000c5002def789e
+.Ed
+.
+.Sh SEE ALSO
+.Xr vdev_id 8

man/man8/vdev_id.8

@@ -1,77 +1,93 @@
-.TH VDEV_ID 8 "Aug 24, 2020" OpenZFS
-.SH NAME
-vdev_id \- generate user-friendly names for JBOD disks
-.SH SYNOPSIS
-.LP
-.nf
-\fBvdev_id\fR <-d dev> [-c config_file] [-g sas_direct|sas_switch]
-                 [-m] [-p phys_per_port]
-\fBvdev_id\fR -h
-.fi
-.SH DESCRIPTION
-The \fBvdev_id\fR command is a udev helper which parses the file
-.BR /etc/zfs/vdev_id.conf (5)
-to map a physical path in a storage topology to a channel name.  The
-channel name is combined with a disk enclosure slot number to create an
-alias that reflects the physical location of the drive.  This is
-particularly helpful when it comes to tasks like replacing failed
-drives.  Slot numbers may also be re-mapped in case the default
-numbering is unsatisfactory.  The drive aliases will be created as
-symbolic links in /dev/disk/by-vdev.
-
-The currently supported topologies are sas_direct and sas_switch.  A
-multipath mode is supported in which dm-mpath devices are handled by
-examining the first-listed running component disk as reported by the
-.BR multipath (8)
-command.  In multipath mode the configuration file should contain a
+.\"
+.\" This file and its contents are supplied under the terms of the
+.\" Common Development and Distribution License ("CDDL"), version 1.0.
+.\" You may only use this file in accordance with the terms of version
+.\" 1.0 of the CDDL.
+.\"
+.\" A full copy of the text of the CDDL should have accompanied this
+.\" source.  A copy of the CDDL is also available via the Internet at
+.\" http://www.illumos.org/license/CDDL.
+.\"
+.Dd May 26, 2021
+.Dt VDEV_ID 8
+.Os
+.
+.Sh NAME
+.Nm vdev_id
+.Nd generate user-friendly names for JBOD disks
+.Sh SYNOPSIS
+.Nm
+.Fl d Ar dev
+.Fl c Ar config_file
+.Fl g Sy sas_direct Ns | Ns Sy sas_switch Ns | Ns Sy scsi
+.Fl m
+.Fl p Ar phys_per_port
+.
+.Sh DESCRIPTION
+.Nm
+is an udev helper which parses
+.Xr vdev_id.conf 5
+to map a physical path in a storage topology to a channel name.
+The channel name is combined with a disk enclosure slot number to create
+an alias that reflects the physical location of the drive.
+This is particularly helpful when it comes to tasks like replacing failed drives.
+Slot numbers may also be remapped in case the default numbering is unsatisfactory.
+The drive aliases will be created as symbolic links in
+.Pa /dev/disk/by-vdev .
+.Pp
+The currently supported topologies are
+.Sy sas_direct ,
+.Sy sas_switch ,
+and
+.Sy scsi .
+A multipath mode is supported in which dm-mpath devices are handled by
+examining the first running component disk as reported by the driver.
+In multipath mode the configuration file should contain a
 channel definition with the same name for each path to a given
 enclosure.
-
-.BR vdev_id
+.Pp
+.Nm
 also supports creating aliases based on existing udev links in the /dev
-hierarchy using the \fIalias\fR configuration file keyword.  See the
-.BR vdev_id.conf (5)
-man page for details.
-
-.SH OPTIONS
-.TP
-\fB\-c\fR <config_file>
-Specifies the path to an alternate configuration file.  The default is
-/etc/zfs/vdev_id.conf.
-.TP
-\fB\-d\fR <device>
-This is the only mandatory argument.  Specifies the name of a device
-in /dev, i.e. "sda".
-.TP
-\fB\-g\fR <sas_direct|sas_switch>
+hierarchy using the
+.Sy alias
+configuration file keyword.
+See
+.Xr vdev_id.conf 5
+for details.
+.
+.Sh OPTIONS
+.Bl -tag -width "-m"
+.It Fl d Ar device
+The device node to classify, like
+.Pa /dev/sda .
+.It Fl c Ar config_file
+Specifies the path to an alternate configuration file.
+The default is
+.Pa /etc/zfs/vdev_id.conf .
+.It Fl g Sy sas_direct Ns | Ns Sy sas_switch Ns | Ns Sy scsi
 Identifies a physical topology that governs how physical paths are
-mapped to channels.
-
-\fIsas_direct\fR - in this mode a channel is uniquely identified by
-a PCI slot and a HBA port number
-
-\fIsas_switch\fR - in this mode a channel is uniquely identified by
-a SAS switch port number
-.TP
-\fB\-m\fR
-Specifies that
-.BR vdev_id (8)
-will handle only dm-multipath devices.  If set to "yes" then
-.BR vdev_id (8)
-will examine the first running component disk of a dm-multipath
-device as listed by the
-.BR multipath (8)
-command to determine the physical path.
-.TP
-\fB\-p\fR <phys_per_port>
+mapped to channels:
+.Bl -tag -compact -width "sas_direct and scsi"
+.It Sy sas_direct No and Sy scsi
+channels are uniquely identified by a PCI slot and HBA port number
+.It Sy sas_switch
+channels are uniquely identified by a SAS switch port number
+.El
+.It Fl m
+Only handle dm-multipath devices.
+If specified, examine the first running component disk of a dm-multipath
+device as provided by the driver to determine the physical path.
+.It Fl p Ar phys_per_port
 Specifies the number of PHY devices associated with a SAS HBA port or SAS
 switch port.
-.BR vdev_id (8)
+.Nm
 internally uses this value to determine which HBA or switch port a
-device is connected to.  The default is 4.
-.TP
-\fB\-h\fR
+device is connected to.
+The default is
+.Sy 4 .
+.It Fl h
 Print a usage summary.
-.SH SEE ALSO
-.LP
-\fBvdev_id.conf\fR(5)
+.El
+.
+.Sh SEE ALSO
+.Xr vdev_id.conf 5

man/man8/zfsprops.8

@@ -38,7 +38,7 @@
 .\" Copyright 2019 Joyent, Inc.
 .\" Copyright (c) 2019, Kjeld Schouten-Lebbing
 .\"
-.Dd September 1, 2020
+.Dd May 5, 2021
 .Dt ZFSPROPS 8
 .Os
 .Sh NAME
@@ -1778,7 +1778,7 @@ The default value is
 This property is not used on Linux.
 .It Sy xattr Ns = Ns Sy on Ns | Ns Sy off Ns | Ns Sy sa
 Controls whether extended attributes are enabled for this file system. Two
-styles of extended attributes are supported either directory based or system
+styles of extended attributes are supported: either directory based or system
 attribute based.
 .Pp
 The default value of
@@ -1790,7 +1790,7 @@ can be set on a file. Although under Linux the
 and
 .Xr setxattr 2
 system calls limit the maximum size to 64K. This is the most compatible
-style of extended attribute and is supported by all OpenZFS implementations.
+style of extended attribute and is supported by all ZFS implementations.
 .Pp
 System attribute based xattrs can be enabled by setting the value to
 .Sy sa .
@@ -1803,6 +1803,9 @@ based xattr. System attribute based extended attributes are not accessible
 on platforms which do not support the
 .Sy xattr=sa
 feature.
+OpenZFS supports
+.Sy xattr=sa
+on both FreeBSD and Linux.
 .Pp
 The use of system attribute based xattrs is strongly encouraged for users of
 SELinux or POSIX ACLs. Both of these features heavily rely on extended

module/Makefile.in

@@ -109,7 +109,9 @@ modules_uninstall: modules_uninstall-@ac_system@
 
 cppcheck-Linux:
 	@CPPCHECK@ -j@CPU_COUNT@ --std=c99 --quiet --force --error-exitcode=2 \
-		--inline-suppr --suppress=noValidConfiguration \
+		--inline-suppr \
+		--suppress=unmatchedSuppression \
+		--suppress=noValidConfiguration \
 		--enable=warning,information -D_KERNEL \
 		--include=@LINUX_OBJ@/include/generated/autoconf.h \
 		--include=@top_srcdir@/zfs_config.h \