Merge branch 'zfs-2.2.3-staging' into truenas/zfs-2.2-release

Signed-off-by: Ameer Hamza <ahamza@ixsystems.com>

.github/codeql-cpp.yml

@@ -0,0 +1,4 @@
+name: "Custom CodeQL Analysis"
+
+queries:
+  - uses: ./.github/codeql/custom-queries/cpp/deprecatedFunctionUsage.ql

.github/codeql-python.yml

@@ -0,0 +1,4 @@
+name: "Custom CodeQL Analysis"
+
+paths-ignore:
+  - tests

.github/codeql/custom-queries/cpp/deprecatedFunctionUsage.ql

@@ -0,0 +1,59 @@
+/**
+ * @name Deprecated function usage detection
+ * @description Detects functions whose usage is banned from the OpenZFS
+ *              codebase due to QA concerns.
+ * @kind problem
+ * @severity error
+ * @id cpp/deprecated-function-usage
+*/
+
+import cpp
+
+predicate isDeprecatedFunction(Function f) {
+  f.getName() = "strtok" or
+  f.getName() = "__xpg_basename" or
+  f.getName() = "basename" or
+  f.getName() = "dirname" or
+  f.getName() = "bcopy" or
+  f.getName() = "bcmp" or
+  f.getName() = "bzero" or
+  f.getName() = "asctime" or
+  f.getName() = "asctime_r" or
+  f.getName() = "gmtime" or
+  f.getName() = "localtime" or
+  f.getName() = "strncpy"
+
+}
+
+string getReplacementMessage(Function f) {
+  if f.getName() = "strtok" then
+    result = "Use strtok_r(3) instead!"
+  else if f.getName() = "__xpg_basename" then
+    result = "basename(3) is underspecified. Use zfs_basename() instead!"
+  else if f.getName() = "basename" then
+    result = "basename(3) is underspecified. Use zfs_basename() instead!"
+  else if f.getName() = "dirname" then
+    result = "dirname(3) is underspecified. Use zfs_dirnamelen() instead!"
+  else if f.getName() = "bcopy" then
+    result = "bcopy(3) is deprecated. Use memcpy(3)/memmove(3) instead!"
+  else if f.getName() = "bcmp" then
+    result = "bcmp(3) is deprecated. Use memcmp(3) instead!"
+  else if f.getName() = "bzero" then
+    result = "bzero(3) is deprecated. Use memset(3) instead!"
+  else if f.getName() = "asctime" then
+    result = "Use strftime(3) instead!"
+  else if f.getName() = "asctime_r" then
+    result = "Use strftime(3) instead!"
+  else if f.getName() = "gmtime" then
+    result = "gmtime(3) isn't thread-safe. Use gmtime_r(3) instead!"
+  else if f.getName() = "localtime" then
+    result = "localtime(3) isn't thread-safe. Use localtime_r(3) instead!"
+  else
+    result = "strncpy(3) is deprecated. Use strlcpy(3) instead!"
+}
+
+from FunctionCall fc, Function f
+where
+  fc.getTarget() = f and
+  isDeprecatedFunction(f)
+select fc, getReplacementMessage(f)

.github/codeql/custom-queries/cpp/qlpack.yml

@@ -0,0 +1,4 @@
+name: openzfs-cpp-queries
+version: 0.0.0
+libraryPathDependencies: codeql-cpp
+suites: openzfs-cpp-suite

.github/workflows/README.md

@@ -4,44 +4,54 @@
 ```mermaid
 flowchart TB
 subgraph CleanUp and Summary
-  Part1-20.04-->CleanUp+nice+Summary
+  CleanUp+Summary
-  Part2-20.04-->CleanUp+nice+Summary
-  PartN-20.04-->CleanUp+nice+Summary
-  Part1-22.04-->CleanUp+nice+Summary
-  Part2-22.04-->CleanUp+nice+Summary
-  PartN-22.04-->CleanUp+nice+Summary
 end
 
 subgraph Functional Testings
+  sanity-checks-20.04
+  zloop-checks-20.04
   functional-testing-20.04-->Part1-20.04
   functional-testing-20.04-->Part2-20.04
-  functional-testing-20.04-->PartN-20.04
+  functional-testing-20.04-->Part3-20.04
+  functional-testing-20.04-->Part4-20.04
   functional-testing-22.04-->Part1-22.04
   functional-testing-22.04-->Part2-22.04
-  functional-testing-22.04-->PartN-22.04
+  functional-testing-22.04-->Part3-22.04
-end
+  functional-testing-22.04-->Part4-22.04
-
+  sanity-checks-22.04
-subgraph Sanity and zloop Testings
+  zloop-checks-22.04
-  sanity-checks-20.04-->functional-testing-20.04
-  sanity-checks-22.04-->functional-testing-22.04
-  zloop-checks-20.04-->functional
-  zloop-checks-22.04-->functional
 end
 
 subgraph Code Checking + Building
+  Build-Ubuntu-20.04
   codeql.yml
   checkstyle.yml
-  Build-Ubuntu-20.04-->sanity-checks-20.04
+  Build-Ubuntu-22.04
-  Build-Ubuntu-22.04-->sanity-checks-22.04
-  Build-Ubuntu-20.04-->zloop-checks-20.04
-  Build-Ubuntu-22.04-->zloop-checks-22.04
 end
+
+  Build-Ubuntu-20.04-->sanity-checks-20.04
+  Build-Ubuntu-20.04-->zloop-checks-20.04
+  Build-Ubuntu-20.04-->functional-testing-20.04
+  Build-Ubuntu-22.04-->sanity-checks-22.04
+  Build-Ubuntu-22.04-->zloop-checks-22.04
+  Build-Ubuntu-22.04-->functional-testing-22.04
+
+  sanity-checks-20.04-->CleanUp+Summary
+  Part1-20.04-->CleanUp+Summary
+  Part2-20.04-->CleanUp+Summary
+  Part3-20.04-->CleanUp+Summary
+  Part4-20.04-->CleanUp+Summary
+  Part1-22.04-->CleanUp+Summary
+  Part2-22.04-->CleanUp+Summary
+  Part3-22.04-->CleanUp+Summary
+  Part4-22.04-->CleanUp+Summary
+  sanity-checks-22.04-->CleanUp+Summary
 ```
 
 
 1) build zfs modules for Ubuntu 20.04 and 22.04 (~15m)
 2) 2x zloop test (~10m) + 2x sanity test (~25m)
-3) functional testings in parts 1..5 (each ~1h)
+3) 4x functional testings in parts 1..4 (each ~1h)
 4) cleanup and create summary
    - content of summary depends on the results of the steps
 

.github/workflows/checkstyle.yaml

@@ -8,7 +8,7 @@ jobs:
   checkstyle:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
     - name: Install dependencies
@@ -52,7 +52,7 @@ jobs:
       if: failure() && steps.CheckABI.outcome == 'failure'
       run: |
         find -name *.abi | tar -cf abi_files.tar -T -
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       if: failure() && steps.CheckABI.outcome == 'failure'
       with:
         name: New ABI files (use only if you're sure about interface changes)

.github/workflows/codeql.yml

@@ -24,11 +24,12 @@ jobs:
         echo "MAKEFLAGS=-j$(nproc)" >> $GITHUB_ENV
 
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2
       with:
+        config-file: .github/codeql-${{ matrix.language }}.yml
         languages: ${{ matrix.language }}
 
     - name: Autobuild

.github/workflows/scripts/generate-summary.sh

@@ -87,7 +87,7 @@ function summarize_f() {
   output "\n## $headline\n"
   rm -rf testfiles
   for i in $(seq 1 $FUNCTIONAL_PARTS); do
-    tarfile="$2/part$i.tar"
+    tarfile="$2-part$i/part$i.tar"
     check_tarfile "$tarfile"
     check_logfile "testfiles/log"
   done

.github/workflows/scripts/setup-dependencies.sh

@@ -55,29 +55,24 @@ function mod_install() {
   cat /proc/spl/kstat/zfs/chksum_bench
   echo "::endgroup::"
 
-  echo "::group::Reclaim and report disk space"
+  echo "::group::Optimize storage for ZFS testings"
-  # remove 4GiB of images
+  # remove swap and umount fast storage
-  sudo systemd-run docker system prune --force --all --volumes
+  # 89GiB -> rootfs + bootfs with ~80MB/s -> don't care
+  # 64GiB -> /mnt with 420MB/s -> new testing ssd
+  sudo swapoff -a
 
-  # remove unused software
+  # this one is fast and mounted @ /mnt
-  sudo systemd-run --wait rm -rf \
+  # -> we reformat with ext4 + move it to /var/tmp
-    "$AGENT_TOOLSDIRECTORY" \
+  DEV="/dev/disk/azure/resource-part1"
-    /opt/* \
+  sudo umount /mnt
-    /usr/local/* \
+  sudo mkfs.ext4 -O ^has_journal -F $DEV
-    /usr/share/az* \
+  sudo mount -o noatime,barrier=0 $DEV /var/tmp
-    /usr/share/dotnet \
+  sudo chmod 1777 /var/tmp
-    /usr/share/gradle* \
-    /usr/share/miniconda \
-    /usr/share/swift \
-    /var/lib/gems \
-    /var/lib/mysql \
-    /var/lib/snapd
-
-  # trim the cleaned space
-  sudo fstrim /
 
   # disk usage afterwards
-  df -h /
+  sudo df -h /
+  sudo df -h /var/tmp
+  sudo fstrim -a
   echo "::endgroup::"
 }
 

.github/workflows/zfs-linux-tests.yml

@@ -13,10 +13,10 @@ jobs:
   zloop:
     runs-on: ubuntu-${{ inputs.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       with:
         name: modules-${{ inputs.os }}
     - name: Install modules
@@ -34,7 +34,7 @@ jobs:
       if: failure()
       run: |
         sudo chmod +r -R /var/tmp/zloop/
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: Zpool-logs-${{ inputs.os }}
@@ -43,7 +43,7 @@ jobs:
           !/var/tmp/zloop/*/vdev/
         retention-days: 14
         if-no-files-found: ignore
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: Zpool-files-${{ inputs.os }}
@@ -55,10 +55,10 @@ jobs:
   sanity:
     runs-on: ubuntu-${{ inputs.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       with:
         name: modules-${{ inputs.os }}
     - name: Install modules
@@ -77,7 +77,7 @@ jobs:
         RESPATH="/var/tmp/test_results"
         mv -f $RESPATH/current $RESPATH/testfiles
         tar cf $RESPATH/sanity.tar -h -C $RESPATH testfiles
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       if: success() || failure()
       with:
         name: Logs-${{ inputs.os }}-sanity
@@ -91,10 +91,10 @@ jobs:
       matrix:
         tests: [ part1, part2, part3, part4 ]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       with:
         name: modules-${{ inputs.os }}
     - name: Install modules
@@ -116,9 +116,9 @@ jobs:
         RESPATH="/var/tmp/test_results"
         mv -f $RESPATH/current $RESPATH/testfiles
         tar cf $RESPATH/${{ matrix.tests }}.tar -h -C $RESPATH testfiles
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       if: success() || failure()
       with:
-        name: Logs-${{ inputs.os }}-functional
+        name: Logs-${{ inputs.os }}-functional-${{ matrix.tests }}
         path: /var/tmp/test_results/${{ matrix.tests }}.tar
         if-no-files-found: ignore

.github/workflows/zfs-linux.yml

@@ -14,14 +14,14 @@ jobs:
         os: [22.04]
     runs-on: ubuntu-${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
     - name: Build modules
       run: .github/workflows/scripts/setup-dependencies.sh build
     - name: Prepare modules upload
       run: tar czf modules-${{ matrix.os }}.tgz *.deb .github tests/test-runner tests/ImageOS.txt
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: modules-${{ matrix.os }}
         path: modules-${{ matrix.os }}.tgz
@@ -44,7 +44,7 @@ jobs:
     runs-on: ubuntu-22.04
     needs: testings
     steps:
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
     - name: Generating summary
       run: |
         tar xzf modules-22.04/modules-22.04.tgz .github tests
@@ -58,7 +58,7 @@ jobs:
       run: .github/workflows/scripts/generate-summary.sh 3
     - name: Summary for errors #4
       run: .github/workflows/scripts/generate-summary.sh 4
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: Summary Files
         path: Summary/

META

@@ -6,5 +6,5 @@ Release:       1
 Release-Tags:  relext
 License:       CDDL
 Author:        OpenZFS
-Linux-Maximum: 6.6
+Linux-Maximum: 6.7
 Linux-Minimum: 3.10

cmd/zdb/zdb.c

@@ -2360,7 +2360,7 @@ static void
 snprintf_zstd_header(spa_t *spa, char *blkbuf, size_t buflen,
     const blkptr_t *bp)
 {
-	abd_t *pabd;
+	static abd_t *pabd = NULL;
 	void *buf;
 	zio_t *zio;
 	zfs_zstdhdr_t zstd_hdr;
@@ -2391,7 +2391,8 @@ snprintf_zstd_header(spa_t *spa, char *blkbuf, size_t buflen,
 		return;
 	}
 
-	pabd = abd_alloc_for_io(SPA_MAXBLOCKSIZE, B_FALSE);
+	if (!pabd)
+		pabd = abd_alloc_for_io(SPA_MAXBLOCKSIZE, B_FALSE);
 	zio = zio_root(spa, NULL, NULL, 0);
 
 	/* Decrypt but don't decompress so we can read the compression header */
@@ -8490,6 +8491,14 @@ zdb_decompress_block(abd_t *pabd, void *buf, void *lbuf, uint64_t lsize,
 	*cfuncp++ = ZIO_COMPRESS_LZ4;
 	*cfuncp++ = ZIO_COMPRESS_LZJB;
 	mask |= ZIO_COMPRESS_MASK(LZ4) | ZIO_COMPRESS_MASK(LZJB);
+	/*
+	 * Every gzip level has the same decompressor, no need to
+	 * run it 9 times per bruteforce attempt.
+	 */
+	mask |= ZIO_COMPRESS_MASK(GZIP_2) | ZIO_COMPRESS_MASK(GZIP_3);
+	mask |= ZIO_COMPRESS_MASK(GZIP_4) | ZIO_COMPRESS_MASK(GZIP_5);
+	mask |= ZIO_COMPRESS_MASK(GZIP_6) | ZIO_COMPRESS_MASK(GZIP_7);
+	mask |= ZIO_COMPRESS_MASK(GZIP_8) | ZIO_COMPRESS_MASK(GZIP_9);
 	for (int c = 0; c < ZIO_COMPRESS_FUNCTIONS; c++)
 		if (((1ULL << c) & mask) == 0)
 			*cfuncp++ = c;

cmd/zed/agents/zfs_mod.c

@@ -233,6 +233,9 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	}
 
 	(void) nvlist_lookup_string(vdev, ZPOOL_CONFIG_PHYS_PATH, &physpath);
+
+	update_vdev_config_dev_sysfs_path(vdev, path,
+	    ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
 	(void) nvlist_lookup_string(vdev, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH,
 	    &enc_sysfs_path);
 #ifndef TRUENAS_SCALE_NEVER_WHOLEDISK

cmd/zed/zed_event.c

@@ -35,6 +35,7 @@
 #include "zed_strings.h"
 
 #include "agents/zfs_agents.h"
+#include <libzutil.h>
 
 #define	MAXBUF	4096
 
@@ -922,6 +923,25 @@ _zed_event_add_time_strings(uint64_t eid, zed_strings_t *zsp, int64_t etime[])
 	}
 }
 
+
+static void
+_zed_event_update_enc_sysfs_path(nvlist_t *nvl)
+{
+	const char *vdev_path;
+
+	if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_PATH,
+	    &vdev_path) != 0) {
+		return; /* some other kind of event, ignore it */
+	}
+
+	if (vdev_path == NULL) {
+		return;
+	}
+
+	update_vdev_config_dev_sysfs_path(nvl, vdev_path,
+	    FM_EREPORT_PAYLOAD_ZFS_VDEV_ENC_SYSFS_PATH);
+}
+
 /*
  * Service the next zevent, blocking until one is available.
  */
@@ -969,6 +989,17 @@ zed_event_service(struct zed_conf *zcp)
 		zed_log_msg(LOG_WARNING,
 		    "Failed to lookup zevent class (eid=%llu)", eid);
 	} else {
+		/*
+		 * Special case: If we can dynamically detect an enclosure sysfs
+		 * path, then use that value rather than the one stored in the
+		 * vd->vdev_enc_sysfs_path.  There have been rare cases where
+		 * vd->vdev_enc_sysfs_path becomes outdated.  However, there
+		 * will be other times when we can not dynamically detect the
+		 * sysfs path (like if a disk disappears) and have to rely on
+		 * the old value for things like turning on the fault LED.
+		 */
+		_zed_event_update_enc_sysfs_path(nvl);
+
 		/* let internal modules see this event first */
 		zfs_agent_post_event(class, NULL, nvl);
 

cmd/zfs/zfs_main.c

@@ -7230,7 +7230,8 @@ share_mount(int op, int argc, char **argv)
 		pthread_mutex_init(&share_mount_state.sm_lock, NULL);
 
 		/* For a 'zfs share -a' operation start with a clean slate. */
-		zfs_truncate_shares(NULL);
+		if (op == OP_SHARE)
+			zfs_truncate_shares(NULL);
 
 		/*
 		 * libshare isn't mt-safe, so only do the operation in parallel

cmd/zpool/os/freebsd/zpool_vdev_os.c

@@ -124,3 +124,17 @@ check_file(const char *file, boolean_t force, boolean_t isspare)
 {
 	return (check_file_generic(file, force, isspare));
 }
+
+int
+zpool_power_current_state(zpool_handle_t *zhp, char *vdev)
+{
+	/* Enclosure slot power not supported on FreeBSD yet */
+	return (-1);
+}
+
+int
+zpool_power(zpool_handle_t *zhp, char *vdev, boolean_t turn_on)
+{
+	/* Enclosure slot power not supported on FreeBSD yet */
+	return (ENOTSUP);
+}

cmd/zpool/os/linux/zpool_vdev_os.c

@@ -416,3 +416,258 @@ check_file(const char *file, boolean_t force, boolean_t isspare)
 {
 	return (check_file_generic(file, force, isspare));
 }
+
+/*
+ * Read from a sysfs file and return an allocated string.  Removes
+ * the newline from the end of the string if there is one.
+ *
+ * Returns a string on success (which must be freed), or NULL on error.
+ */
+static char *zpool_sysfs_gets(char *path)
+{
+	int fd;
+	struct stat statbuf;
+	char *buf = NULL;
+	ssize_t count = 0;
+	fd = open(path, O_RDONLY);
+	if (fd < 0)
+		return (NULL);
+
+	if (fstat(fd, &statbuf) != 0) {
+		close(fd);
+		return (NULL);
+	}
+
+	buf = calloc(sizeof (*buf), statbuf.st_size + 1);
+	if (buf == NULL) {
+		close(fd);
+		return (NULL);
+	}
+
+	/*
+	 * Note, we can read less bytes than st_size, and that's ok.  Sysfs
+	 * files will report their size is 4k even if they only return a small
+	 * string.
+	 */
+	count = read(fd, buf, statbuf.st_size);
+	if (count < 0) {
+		/* Error doing read() or we overran the buffer */
+		close(fd);
+		free(buf);
+		return (NULL);
+	}
+
+	/* Remove trailing newline */
+	if (buf[count - 1] == '\n')
+		buf[count - 1] = 0;
+
+	close(fd);
+
+	return (buf);
+}
+
+/*
+ * Write a string to a sysfs file.
+ *
+ * Returns 0 on success, non-zero otherwise.
+ */
+static int zpool_sysfs_puts(char *path, char *str)
+{
+	FILE *file;
+
+	file = fopen(path, "w");
+	if (!file) {
+		return (-1);
+	}
+
+	if (fputs(str, file) < 0) {
+		fclose(file);
+		return (-2);
+	}
+	fclose(file);
+	return (0);
+}
+
+/* Given a vdev nvlist_t, rescan its enclosure sysfs path */
+static void
+rescan_vdev_config_dev_sysfs_path(nvlist_t *vdev_nv)
+{
+	update_vdev_config_dev_sysfs_path(vdev_nv,
+	    fnvlist_lookup_string(vdev_nv, ZPOOL_CONFIG_PATH),
+	    ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
+}
+
+/*
+ * Given a power string: "on", "off", "1", or "0", return 0 if it's an
+ * off value, 1 if it's an on value, and -1 if the value is unrecognized.
+ */
+static int zpool_power_parse_value(char *str)
+{
+	if ((strcmp(str, "off") == 0) || (strcmp(str, "0") == 0))
+		return (0);
+
+	if ((strcmp(str, "on") == 0) || (strcmp(str, "1") == 0))
+		return (1);
+
+	return (-1);
+}
+
+/*
+ * Given a vdev string return an allocated string containing the sysfs path to
+ * its power control file.  Also do a check if the power control file really
+ * exists and has correct permissions.
+ *
+ * Example returned strings:
+ *
+ * /sys/class/enclosure/0:0:122:0/10/power_status
+ * /sys/bus/pci/slots/10/power
+ *
+ * Returns allocated string on success (which must be freed), NULL on failure.
+ */
+static char *
+zpool_power_sysfs_path(zpool_handle_t *zhp, char *vdev)
+{
+	const char *enc_sysfs_dir = NULL;
+	char *path = NULL;
+	nvlist_t *vdev_nv = zpool_find_vdev(zhp, vdev, NULL, NULL, NULL);
+
+	if (vdev_nv == NULL) {
+		return (NULL);
+	}
+
+	/* Make sure we're getting the updated enclosure sysfs path */
+	rescan_vdev_config_dev_sysfs_path(vdev_nv);
+
+	if (nvlist_lookup_string(vdev_nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH,
+	    &enc_sysfs_dir) != 0) {
+		return (NULL);
+	}
+
+	if (asprintf(&path, "%s/power_status", enc_sysfs_dir) == -1)
+		return (NULL);
+
+	if (access(path, W_OK) != 0) {
+		free(path);
+		path = NULL;
+		/* No HDD 'power_control' file, maybe it's NVMe? */
+		if (asprintf(&path, "%s/power", enc_sysfs_dir) == -1) {
+			return (NULL);
+		}
+
+		if (access(path, R_OK | W_OK) != 0) {
+			/* Not NVMe either */
+			free(path);
+			return (NULL);
+		}
+	}
+
+	return (path);
+}
+
+/*
+ * Given a path to a sysfs power control file, return B_TRUE if you should use
+ * "on/off" words to control it, or B_FALSE otherwise ("0/1" to control).
+ */
+static boolean_t
+zpool_power_use_word(char *sysfs_path)
+{
+	if (strcmp(&sysfs_path[strlen(sysfs_path) - strlen("power_status")],
+	    "power_status") == 0) {
+		return (B_TRUE);
+	}
+	return (B_FALSE);
+}
+
+/*
+ * Check the sysfs power control value for a vdev.
+ *
+ * Returns:
+ *  0 - Power is off
+ *  1 - Power is on
+ * -1 - Error or unsupported
+ */
+int
+zpool_power_current_state(zpool_handle_t *zhp, char *vdev)
+{
+	char *val;
+	int rc;
+
+	char *path = zpool_power_sysfs_path(zhp, vdev);
+	if (path == NULL)
+		return (-1);
+
+	val = zpool_sysfs_gets(path);
+	if (val == NULL) {
+		free(path);
+		return (-1);
+	}
+
+	rc = zpool_power_parse_value(val);
+	free(val);
+	free(path);
+	return (rc);
+}
+
+/*
+ * Turn on or off the slot to a device
+ *
+ * Device path is the full path to the device (like /dev/sda or /dev/sda1).
+ *
+ * Return code:
+ * 0:		Success
+ * ENOTSUP:	Power control not supported for OS
+ * EBADSLT:	Couldn't read current power state
+ * ENOENT:	No sysfs path to power control
+ * EIO:	Couldn't write sysfs power value
+ * EBADE:	Sysfs power value didn't change
+ */
+int
+zpool_power(zpool_handle_t *zhp, char *vdev, boolean_t turn_on)
+{
+	char *sysfs_path;
+	const char *val;
+	int rc;
+	int timeout_ms;
+
+	rc = zpool_power_current_state(zhp, vdev);
+	if (rc == -1) {
+		return (EBADSLT);
+	}
+
+	/* Already correct value? */
+	if (rc == (int)turn_on)
+		return (0);
+
+	sysfs_path = zpool_power_sysfs_path(zhp, vdev);
+	if (sysfs_path == NULL)
+		return (ENOENT);
+
+	if (zpool_power_use_word(sysfs_path)) {
+		val = turn_on ? "on" : "off";
+	} else {
+		val = turn_on ? "1" : "0";
+	}
+
+	rc = zpool_sysfs_puts(sysfs_path, (char *)val);
+
+	free(sysfs_path);
+	if (rc != 0) {
+		return (EIO);
+	}
+
+	/*
+	 * Wait up to 30 seconds for sysfs power value to change after
+	 * writing it.
+	 */
+	timeout_ms = zpool_getenv_int("ZPOOL_POWER_ON_SLOT_TIMEOUT_MS", 30000);
+	for (int i = 0; i < MAX(1, timeout_ms / 200); i++) {
+		rc = zpool_power_current_state(zhp, vdev);
+		if (rc == (int)turn_on)
+			return (0);	/* success */
+
+		fsleep(0.200);	/* 200ms */
+	}
+
+	/* sysfs value never changed */
+	return (EBADE);
+}

cmd/zpool/zpool.d/ses

@@ -33,10 +33,18 @@ for i in $scripts ; do
 	val=""
 	case $i in
 	enc)
-		val=$(ls "$VDEV_ENC_SYSFS_PATH/../../" 2>/dev/null)
+		if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then
+			val="$VDEV_ENC_SYSFS_PATH"
+		else
+			val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)"
+		fi
 		;;
 	slot)
-		val=$(cat "$VDEV_ENC_SYSFS_PATH/slot" 2>/dev/null)
+		if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then
+			val="$(basename """$VDEV_ENC_SYSFS_PATH""")"
+		else
+			val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)"
+		fi
 		;;
 	encdev)
 		val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null)

cmd/zpool/zpool_iter.c

@@ -554,6 +554,10 @@ for_each_vdev_run_cb(void *zhp_data, nvlist_t *nv, void *cb_vcdl)
 	if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) != 0)
 		return (1);
 
+	/* Make sure we're getting the updated enclosure sysfs path */
+	update_vdev_config_dev_sysfs_path(nv, path,
+	    ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
+
 	nvlist_lookup_string(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH,
 	    &vdev_enc_sysfs_path);
 

cmd/zpool/zpool_main.c

@@ -353,7 +353,7 @@ get_usage(zpool_help_t idx)
 		return (gettext("\tattach [-fsw] [-o property=value] "
 		    "<pool> <device> <new-device>\n"));
 	case HELP_CLEAR:
-		return (gettext("\tclear [-nF] <pool> [device]\n"));
+		return (gettext("\tclear [[--power]|[-nF]] <pool> [device]\n"));
 	case HELP_CREATE:
 		return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
 		    "\t    [-O file-system-property=value] ... \n"
@@ -389,9 +389,11 @@ get_usage(zpool_help_t idx)
 		    "[-T d|u] [pool] ... \n"
 		    "\t    [interval [count]]\n"));
 	case HELP_OFFLINE:
-		return (gettext("\toffline [-f] [-t] <pool> <device> ...\n"));
+		return (gettext("\toffline [--power]|[[-f][-t]] <pool> "
+		    "<device> ...\n"));
 	case HELP_ONLINE:
-		return (gettext("\tonline [-e] <pool> <device> ...\n"));
+		return (gettext("\tonline [--power][-e] <pool> <device> "
+		    "...\n"));
 	case HELP_REPLACE:
 		return (gettext("\treplace [-fsw] [-o property=value] "
 		    "<pool> <device> [new-device]\n"));
@@ -410,7 +412,7 @@ get_usage(zpool_help_t idx)
 		return (gettext("\ttrim [-dw] [-r <rate>] [-c | -s] <pool> "
 		    "[<device> ...]\n"));
 	case HELP_STATUS:
-		return (gettext("\tstatus [-c [script1,script2,...]] "
+		return (gettext("\tstatus [--power] [-c [script1,script2,...]] "
 		    "[-igLpPstvxD]  [-T d|u] [pool] ... \n"
 		    "\t    [interval [count]]\n"));
 	case HELP_UPGRADE:
@@ -516,6 +518,77 @@ print_vdev_prop_cb(int prop, void *cb)
 	return (ZPROP_CONT);
 }
 
+/*
+ * Given a leaf vdev name like 'L5' return its VDEV_CONFIG_PATH like
+ * '/dev/disk/by-vdev/L5'.
+ */
+static const char *
+vdev_name_to_path(zpool_handle_t *zhp, char *vdev)
+{
+	nvlist_t *vdev_nv = zpool_find_vdev(zhp, vdev, NULL, NULL, NULL);
+	if (vdev_nv == NULL) {
+		return (NULL);
+	}
+	return (fnvlist_lookup_string(vdev_nv, ZPOOL_CONFIG_PATH));
+}
+
+static int
+zpool_power_on(zpool_handle_t *zhp, char *vdev)
+{
+	return (zpool_power(zhp, vdev, B_TRUE));
+}
+
+static int
+zpool_power_on_and_disk_wait(zpool_handle_t *zhp, char *vdev)
+{
+	int rc;
+
+	rc = zpool_power_on(zhp, vdev);
+	if (rc != 0)
+		return (rc);
+
+	zpool_disk_wait(vdev_name_to_path(zhp, vdev));
+
+	return (0);
+}
+
+static int
+zpool_power_on_pool_and_wait_for_devices(zpool_handle_t *zhp)
+{
+	nvlist_t *nv;
+	const char *path = NULL;
+	int rc;
+
+	/* Power up all the devices first */
+	FOR_EACH_REAL_LEAF_VDEV(zhp, nv) {
+		path = fnvlist_lookup_string(nv, ZPOOL_CONFIG_PATH);
+		if (path != NULL) {
+			rc = zpool_power_on(zhp, (char *)path);
+			if (rc != 0) {
+				return (rc);
+			}
+		}
+	}
+
+	/*
+	 * Wait for their devices to show up.  Since we powered them on
+	 * at roughly the same time, they should all come online around
+	 * the same time.
+	 */
+	FOR_EACH_REAL_LEAF_VDEV(zhp, nv) {
+		path = fnvlist_lookup_string(nv, ZPOOL_CONFIG_PATH);
+		zpool_disk_wait(path);
+	}
+
+	return (0);
+}
+
+static int
+zpool_power_off(zpool_handle_t *zhp, char *vdev)
+{
+	return (zpool_power(zhp, vdev, B_FALSE));
+}
+
 /*
  * Display usage message.  If we're inside a command, display only the usage for
  * that command.  Otherwise, iterate over the entire command table and display
@@ -2093,6 +2166,7 @@ typedef struct status_cbdata {
 	boolean_t	cb_print_vdev_init;
 	boolean_t	cb_print_vdev_trim;
 	vdev_cmd_data_list_t	*vcdl;
+	boolean_t	cb_print_power;
 } status_cbdata_t;
 
 /* Return 1 if string is NULL, empty, or whitespace; return 0 otherwise. */
@@ -2378,6 +2452,26 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
 			else
 				printf(" %5s", rbuf);
 		}
+		if (cb->cb_print_power) {
+			if (children == 0)  {
+				/* Only leaf vdevs have physical slots */
+				switch (zpool_power_current_state(zhp, (char *)
+				    fnvlist_lookup_string(nv,
+				    ZPOOL_CONFIG_PATH))) {
+				case 0:
+					printf_color(ANSI_RED, " %5s",
+					    gettext("off"));
+					break;
+				case 1:
+					printf(" %5s", gettext("on"));
+					break;
+				default:
+					printf(" %5s", "-");
+				}
+			} else {
+				printf(" %5s", "-");
+			}
+		}
 	}
 
 	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
@@ -5428,19 +5522,6 @@ get_interval_count_filter_guids(int *argc, char **argv, float *interval,
 	    interval, count);
 }
 
-/*
- * Floating point sleep().  Allows you to pass in a floating point value for
- * seconds.
- */
-static void
-fsleep(float sec)
-{
-	struct timespec req;
-	req.tv_sec = floor(sec);
-	req.tv_nsec = (sec - (float)req.tv_sec) * NANOSEC;
-	nanosleep(&req, NULL);
-}
-
 /*
  * Terminal height, in rows. Returns -1 if stdout is not connected to a TTY or
  * if we were unable to determine its size.
@@ -6939,10 +7020,12 @@ zpool_do_split(int argc, char **argv)
 	return (ret);
 }
 
-
+#define	POWER_OPT 1024
 
 /*
- * zpool online <pool> <device> ...
+ * zpool online [--power] <pool> <device> ...
+ *
+ * --power: Power on the enclosure slot to the drive (if possible)
  */
 int
 zpool_do_online(int argc, char **argv)
@@ -6953,13 +7036,21 @@ zpool_do_online(int argc, char **argv)
 	int ret = 0;
 	vdev_state_t newstate;
 	int flags = 0;
+	boolean_t is_power_on = B_FALSE;
+	struct option long_options[] = {
+		{"power", no_argument, NULL, POWER_OPT},
+		{0, 0, 0, 0}
+	};
 
 	/* check options */
-	while ((c = getopt(argc, argv, "e")) != -1) {
+	while ((c = getopt_long(argc, argv, "e", long_options, NULL)) != -1) {
 		switch (c) {
 		case 'e':
 			flags |= ZFS_ONLINE_EXPAND;
 			break;
+		case POWER_OPT:
+			is_power_on = B_TRUE;
+			break;
 		case '?':
 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
 			    optopt);
@@ -6967,6 +7058,9 @@ zpool_do_online(int argc, char **argv)
 		}
 	}
 
+	if (libzfs_envvar_is_set("ZPOOL_AUTO_POWER_ON_SLOT"))
+		is_power_on = B_TRUE;
+
 	argc -= optind;
 	argv += optind;
 
@@ -6988,6 +7082,18 @@ zpool_do_online(int argc, char **argv)
 	for (i = 1; i < argc; i++) {
 		vdev_state_t oldstate;
 		boolean_t avail_spare, l2cache;
+		int rc;
+
+		if (is_power_on) {
+			rc = zpool_power_on_and_disk_wait(zhp, argv[i]);
+			if (rc == ENOTSUP) {
+				(void) fprintf(stderr,
+				    gettext("Power control not supported\n"));
+			}
+			if (rc != 0)
+				return (rc);
+		}
+
 		nvlist_t *tgt = zpool_find_vdev(zhp, argv[i], &avail_spare,
 		    &l2cache, NULL);
 		if (tgt == NULL) {
@@ -7033,12 +7139,15 @@ zpool_do_online(int argc, char **argv)
 }
 
 /*
- * zpool offline [-ft] <pool> <device> ...
+ * zpool offline [-ft]|[--power] <pool> <device> ...
+ *
  *
  *	-f	Force the device into a faulted state.
  *
  *	-t	Only take the device off-line temporarily.  The offline/faulted
  *		state will not be persistent across reboots.
+ *
+ *	--power Power off the enclosure slot to the drive (if possible)
  */
 int
 zpool_do_offline(int argc, char **argv)
@@ -7049,9 +7158,15 @@ zpool_do_offline(int argc, char **argv)
 	int ret = 0;
 	boolean_t istmp = B_FALSE;
 	boolean_t fault = B_FALSE;
+	boolean_t is_power_off = B_FALSE;
+
+	struct option long_options[] = {
+		{"power", no_argument, NULL, POWER_OPT},
+		{0, 0, 0, 0}
+	};
 
 	/* check options */
-	while ((c = getopt(argc, argv, "ft")) != -1) {
+	while ((c = getopt_long(argc, argv, "ft", long_options, NULL)) != -1) {
 		switch (c) {
 		case 'f':
 			fault = B_TRUE;
@@ -7059,6 +7174,9 @@ zpool_do_offline(int argc, char **argv)
 		case 't':
 			istmp = B_TRUE;
 			break;
+		case POWER_OPT:
+			is_power_off = B_TRUE;
+			break;
 		case '?':
 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
 			    optopt);
@@ -7066,6 +7184,20 @@ zpool_do_offline(int argc, char **argv)
 		}
 	}
 
+	if (is_power_off && fault) {
+		(void) fprintf(stderr,
+		    gettext("-0 and -f cannot be used together\n"));
+		usage(B_FALSE);
+		return (1);
+	}
+
+	if (is_power_off && istmp) {
+		(void) fprintf(stderr,
+		    gettext("-0 and -t cannot be used together\n"));
+		usage(B_FALSE);
+		return (1);
+	}
+
 	argc -= optind;
 	argv += optind;
 
@@ -7085,8 +7217,22 @@ zpool_do_offline(int argc, char **argv)
 		return (1);
 
 	for (i = 1; i < argc; i++) {
-		if (fault) {
+		uint64_t guid = zpool_vdev_path_to_guid(zhp, argv[i]);
-			uint64_t guid = zpool_vdev_path_to_guid(zhp, argv[i]);
+		if (is_power_off) {
+			/*
+			 * Note: we have to power off first, then set REMOVED,
+			 * or else zpool_vdev_set_removed_state() returns
+			 * EAGAIN.
+			 */
+			ret = zpool_power_off(zhp, argv[i]);
+			if (ret != 0) {
+				(void) fprintf(stderr, "%s %s %d\n",
+				    gettext("unable to power off slot for"),
+				    argv[i], ret);
+			}
+			zpool_vdev_set_removed_state(zhp, guid, VDEV_AUX_NONE);
+
+		} else if (fault) {
 			vdev_aux_t aux;
 			if (istmp == B_FALSE) {
 				/* Force the fault to persist across imports */
@@ -7109,7 +7255,7 @@ zpool_do_offline(int argc, char **argv)
 }
 
 /*
- * zpool clear <pool> [device]
+ * zpool clear [-nF]|[--power] <pool> [device]
  *
  * Clear all errors associated with a pool or a particular device.
  */
@@ -7121,13 +7267,20 @@ zpool_do_clear(int argc, char **argv)
 	boolean_t dryrun = B_FALSE;
 	boolean_t do_rewind = B_FALSE;
 	boolean_t xtreme_rewind = B_FALSE;
+	boolean_t is_power_on = B_FALSE;
 	uint32_t rewind_policy = ZPOOL_NO_REWIND;
 	nvlist_t *policy = NULL;
 	zpool_handle_t *zhp;
 	char *pool, *device;
 
+	struct option long_options[] = {
+		{"power", no_argument, NULL, POWER_OPT},
+		{0, 0, 0, 0}
+	};
+
 	/* check options */
-	while ((c = getopt(argc, argv, "FnX")) != -1) {
+	while ((c = getopt_long(argc, argv, "FnX", long_options,
+	    NULL)) != -1) {
 		switch (c) {
 		case 'F':
 			do_rewind = B_TRUE;
@@ -7138,6 +7291,9 @@ zpool_do_clear(int argc, char **argv)
 		case 'X':
 			xtreme_rewind = B_TRUE;
 			break;
+		case POWER_OPT:
+			is_power_on = B_TRUE;
+			break;
 		case '?':
 			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
 			    optopt);
@@ -7145,6 +7301,9 @@ zpool_do_clear(int argc, char **argv)
 		}
 	}
 
+	if (libzfs_envvar_is_set("ZPOOL_AUTO_POWER_ON_SLOT"))
+		is_power_on = B_TRUE;
+
 	argc -= optind;
 	argv += optind;
 
@@ -7185,6 +7344,14 @@ zpool_do_clear(int argc, char **argv)
 		return (1);
 	}
 
+	if (is_power_on) {
+		if (device == NULL) {
+			zpool_power_on_pool_and_wait_for_devices(zhp);
+		} else {
+			zpool_power_on_and_disk_wait(zhp, device);
+		}
+	}
+
 	if (zpool_clear(zhp, device, policy) != 0)
 		ret = 1;
 
@@ -8801,6 +8968,10 @@ status_callback(zpool_handle_t *zhp, void *data)
 			printf_color(ANSI_BOLD, " %5s", gettext("SLOW"));
 		}
 
+		if (cbp->cb_print_power) {
+			printf_color(ANSI_BOLD, " %5s", gettext("POWER"));
+		}
+
 		if (cbp->vcdl != NULL)
 			print_cmd_columns(cbp->vcdl, 0);
 
@@ -8847,8 +9018,8 @@ status_callback(zpool_handle_t *zhp, void *data)
 }
 
 /*
- * zpool status [-c [script1,script2,...]] [-igLpPstvx] [-T d|u] [pool] ...
+ * zpool status [-c [script1,script2,...]] [-igLpPstvx] [--power] [-T d|u] ...
- *              [interval [count]]
+ *              [pool] [interval [count]]
  *
  *	-c CMD	For each vdev, run command CMD
  *	-i	Display vdev initialization status.
@@ -8862,6 +9033,7 @@ status_callback(zpool_handle_t *zhp, void *data)
  *	-D	Display dedup status (undocumented)
  *	-t	Display vdev TRIM status.
  *	-T	Display a timestamp in date(1) or Unix format
+ *	--power	Display vdev enclosure slot power status
  *
  * Describes the health status of all pools or some subset.
  */
@@ -8875,8 +9047,14 @@ zpool_do_status(int argc, char **argv)
 	status_cbdata_t cb = { 0 };
 	char *cmd = NULL;
 
+	struct option long_options[] = {
+		{"power", no_argument, NULL, POWER_OPT},
+		{0, 0, 0, 0}
+	};
+
 	/* check options */
-	while ((c = getopt(argc, argv, "c:igLpPsvxDtT:")) != -1) {
+	while ((c = getopt_long(argc, argv, "c:igLpPsvxDtT:", long_options,
+	    NULL)) != -1) {
 		switch (c) {
 		case 'c':
 			if (cmd != NULL) {
@@ -8935,6 +9113,9 @@ zpool_do_status(int argc, char **argv)
 		case 'T':
 			get_timestamp_arg(*optarg);
 			break;
+		case POWER_OPT:
+			cb.cb_print_power = B_TRUE;
+			break;
 		case '?':
 			if (optopt == 'c') {
 				print_zpool_script_list("status");
@@ -10752,6 +10933,9 @@ print_wait_status_row(wait_data_t *wd, zpool_handle_t *zhp, int row)
 		col_widths[i] = MAX(strlen(headers[i]), 6) + 2;
 	}
 
+	if (timestamp_fmt != NODATE)
+		print_timestamp(timestamp_fmt);
+
 	/* Print header if appropriate */
 	int term_height = terminal_height();
 	boolean_t reprint_header = (!wd->wd_headers_once && term_height > 0 &&
@@ -10819,9 +11003,6 @@ print_wait_status_row(wait_data_t *wd, zpool_handle_t *zhp, int row)
 	if (vdev_any_spare_replacing(nvroot))
 		bytes_rem[ZPOOL_WAIT_REPLACE] =  bytes_rem[ZPOOL_WAIT_RESILVER];
 
-	if (timestamp_fmt != NODATE)
-		print_timestamp(timestamp_fmt);
-
 	for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
 		char buf[64];
 		if (!wd->wd_enabled[i])

cmd/zpool/zpool_util.h

@@ -138,6 +138,9 @@ int check_file(const char *file, boolean_t force, boolean_t isspare);
 void after_zpool_upgrade(zpool_handle_t *zhp);
 int check_file_generic(const char *file, boolean_t force, boolean_t isspare);
 
+int zpool_power(zpool_handle_t *zhp, char *vdev, boolean_t turn_on);
+int zpool_power_current_state(zpool_handle_t *zhp, char *vdev);
+
 #ifdef	__cplusplus
 }
 #endif

cmd/zpool/zpool_vdev.c

@@ -372,6 +372,10 @@ make_leaf_vdev(nvlist_t *props, const char *arg, boolean_t is_primary)
 	verify(nvlist_add_string(vdev, ZPOOL_CONFIG_PATH, path) == 0);
 	verify(nvlist_add_string(vdev, ZPOOL_CONFIG_TYPE, type) == 0);
 
+	/* Lookup and add the enclosure sysfs path (if exists) */
+	update_vdev_config_dev_sysfs_path(vdev, path,
+	    ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
+
 	if (strcmp(type, VDEV_TYPE_DISK) == 0)
 		verify(nvlist_add_uint64(vdev, ZPOOL_CONFIG_WHOLE_DISK,
 		    (uint64_t)wholedisk) == 0);

config/Rules.am

@@ -57,21 +57,6 @@ if BUILD_LINUX
 AM_CPPFLAGS += -DTRUENAS_SCALE_NEVER_WHOLEDISK
 endif
 
-AM_CPPFLAGS_NOCHECK  = -D"strtok(...)=strtok(__VA_ARGS__) __attribute__((deprecated(\"Use strtok_r(3) instead!\")))"
-AM_CPPFLAGS_NOCHECK += -D"__xpg_basename(...)=__xpg_basename(__VA_ARGS__) __attribute__((deprecated(\"basename(3) is underspecified. Use zfs_basename() instead!\")))"
-AM_CPPFLAGS_NOCHECK += -D"basename(...)=basename(__VA_ARGS__) __attribute__((deprecated(\"basename(3) is underspecified. Use zfs_basename() instead!\")))"
-AM_CPPFLAGS_NOCHECK += -D"dirname(...)=dirname(__VA_ARGS__) __attribute__((deprecated(\"dirname(3) is underspecified. Use zfs_dirnamelen() instead!\")))"
-AM_CPPFLAGS_NOCHECK += -D"bcopy(...)=__attribute__((deprecated(\"bcopy(3) is deprecated. Use memcpy(3)/memmove(3) instead!\"))) bcopy(__VA_ARGS__)"
-AM_CPPFLAGS_NOCHECK += -D"bcmp(...)=__attribute__((deprecated(\"bcmp(3) is deprecated. Use memcmp(3) instead!\"))) bcmp(__VA_ARGS__)"
-AM_CPPFLAGS_NOCHECK += -D"bzero(...)=__attribute__((deprecated(\"bzero(3) is deprecated. Use memset(3) instead!\"))) bzero(__VA_ARGS__)"
-AM_CPPFLAGS_NOCHECK += -D"asctime(...)=__attribute__((deprecated(\"Use strftime(3) instead!\"))) asctime(__VA_ARGS__)"
-AM_CPPFLAGS_NOCHECK += -D"asctime_r(...)=__attribute__((deprecated(\"Use strftime(3) instead!\"))) asctime_r(__VA_ARGS__)"
-AM_CPPFLAGS_NOCHECK += -D"gmtime(...)=__attribute__((deprecated(\"gmtime(3) isn't thread-safe. Use gmtime_r(3) instead!\"))) gmtime(__VA_ARGS__)"
-AM_CPPFLAGS_NOCHECK += -D"localtime(...)=__attribute__((deprecated(\"localtime(3) isn't thread-safe. Use localtime_r(3) instead!\"))) localtime(__VA_ARGS__)"
-AM_CPPFLAGS_NOCHECK += -D"strncpy(...)=__attribute__((deprecated(\"strncpy(3) is deprecated. Use strlcpy(3) instead!\"))) strncpy(__VA_ARGS__)"
-
-AM_CPPFLAGS += $(AM_CPPFLAGS_NOCHECK)
-
 if ASAN_ENABLED
 AM_CPPFLAGS += -DZFS_ASAN_ENABLED
 endif

config/kernel-acl.m4

@@ -172,7 +172,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_GET_ACL], [
 	ZFS_LINUX_TEST_SRC([inode_operations_get_acl], [
 		#include <linux/fs.h>
 
-		struct posix_acl *get_acl_fn(struct inode *inode, int type)
+		static struct posix_acl *get_acl_fn(struct inode *inode, int type)
 		    { return NULL; }
 
 		static const struct inode_operations
@@ -184,7 +184,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_GET_ACL], [
 	ZFS_LINUX_TEST_SRC([inode_operations_get_acl_rcu], [
 		#include <linux/fs.h>
 
-		struct posix_acl *get_acl_fn(struct inode *inode, int type,
+		static struct posix_acl *get_acl_fn(struct inode *inode, int type,
 		    bool rcu) { return NULL; }
 
 		static const struct inode_operations
@@ -196,7 +196,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_GET_ACL], [
 	ZFS_LINUX_TEST_SRC([inode_operations_get_inode_acl], [
 		#include <linux/fs.h>
 
-		struct posix_acl *get_inode_acl_fn(struct inode *inode, int type,
+		static struct posix_acl *get_inode_acl_fn(struct inode *inode, int type,
 		    bool rcu) { return NULL; }
 
 		static const struct inode_operations
@@ -243,7 +243,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_SET_ACL], [
 	ZFS_LINUX_TEST_SRC([inode_operations_set_acl_mnt_idmap_dentry], [
 		#include <linux/fs.h>
 
-		int set_acl_fn(struct mnt_idmap *idmap,
+		static int set_acl_fn(struct mnt_idmap *idmap,
 		    struct dentry *dent, struct posix_acl *acl,
 		    int type) { return 0; }
 
@@ -255,7 +255,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_SET_ACL], [
 	ZFS_LINUX_TEST_SRC([inode_operations_set_acl_userns_dentry], [
 		#include <linux/fs.h>
 
-		int set_acl_fn(struct user_namespace *userns,
+		static int set_acl_fn(struct user_namespace *userns,
 		    struct dentry *dent, struct posix_acl *acl,
 		    int type) { return 0; }
 
@@ -267,7 +267,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_SET_ACL], [
 	ZFS_LINUX_TEST_SRC([inode_operations_set_acl_userns], [
 		#include <linux/fs.h>
 
-		int set_acl_fn(struct user_namespace *userns,
+		static int set_acl_fn(struct user_namespace *userns,
 		    struct inode *inode, struct posix_acl *acl,
 		    int type) { return 0; }
 
@@ -279,7 +279,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_SET_ACL], [
 	ZFS_LINUX_TEST_SRC([inode_operations_set_acl], [
 		#include <linux/fs.h>
 
-		int set_acl_fn(struct inode *inode, struct posix_acl *acl,
+		static int set_acl_fn(struct inode *inode, struct posix_acl *acl,
 		    int type) { return 0; }
 
 		static const struct inode_operations

config/kernel-automount.m4

@@ -8,7 +8,7 @@ dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_AUTOMOUNT], [
 	ZFS_LINUX_TEST_SRC([dentry_operations_d_automount], [
 		#include <linux/dcache.h>
-		struct vfsmount *d_automount(struct path *p) { return NULL; }
+		static struct vfsmount *d_automount(struct path *p) { return NULL; }
 		struct dentry_operations dops __attribute__ ((unused)) = {
 			.d_automount = d_automount,
 		};

config/kernel-bio.m4

@@ -247,7 +247,7 @@ dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_BIO_END_IO_T_ARGS], [
 	ZFS_LINUX_TEST_SRC([bio_end_io_t_args], [
 		#include <linux/bio.h>
-		void wanted_end_io(struct bio *bio) { return; }
+		static void wanted_end_io(struct bio *bio) { return; }
 		bio_end_io_t *end_io __attribute__ ((unused)) = wanted_end_io;
 	], [])
 ])

config/kernel-blkdev.m4

@@ -35,6 +35,25 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_GET_BY_PATH_4ARG], [
 	])
 ])
 
+dnl #
+dnl # 6.8.x API change
+dnl # bdev_open_by_path() replaces blkdev_get_by_path()
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_OPEN_BY_PATH], [
+	ZFS_LINUX_TEST_SRC([bdev_open_by_path], [
+		#include <linux/fs.h>
+		#include <linux/blkdev.h>
+	], [
+		struct bdev_handle *bdh __attribute__ ((unused)) = NULL;
+		const char *path = "path";
+		fmode_t mode = 0;
+		void *holder = NULL;
+		struct blk_holder_ops h;
+
+		bdh = bdev_open_by_path(path, mode, holder, &h);
+	])
+])
+
 AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_GET_BY_PATH], [
 	AC_MSG_CHECKING([whether blkdev_get_by_path() exists and takes 3 args])
 	ZFS_LINUX_TEST_RESULT([blkdev_get_by_path], [
@@ -47,7 +66,15 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_GET_BY_PATH], [
 				[blkdev_get_by_path() exists and takes 4 args])
 			AC_MSG_RESULT(yes)
 		], [
-			ZFS_LINUX_TEST_ERROR([blkdev_get_by_path()])
+			AC_MSG_RESULT(no)
+			AC_MSG_CHECKING([whether bdev_open_by_path() exists])
+			ZFS_LINUX_TEST_RESULT([bdev_open_by_path], [
+				AC_DEFINE(HAVE_BDEV_OPEN_BY_PATH, 1,
+					[bdev_open_by_path() exists])
+				AC_MSG_RESULT(yes)
+			], [
+				ZFS_LINUX_TEST_ERROR([blkdev_get_by_path()])
+			])
 		])
 	])
 ])
@@ -108,18 +135,41 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_PUT_HOLDER], [
 	])
 ])
 
+dnl #
+dnl # 6.8.x API change
+dnl # bdev_release() replaces blkdev_put()
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_RELEASE], [
+	ZFS_LINUX_TEST_SRC([bdev_release], [
+		#include <linux/fs.h>
+		#include <linux/blkdev.h>
+	], [
+		struct bdev_handle *bdh = NULL;
+		bdev_release(bdh);
+	])
+])
+
 AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_PUT], [
 	AC_MSG_CHECKING([whether blkdev_put() exists])
 	ZFS_LINUX_TEST_RESULT([blkdev_put], [
 		AC_MSG_RESULT(yes)
 	], [
+		AC_MSG_RESULT(no)
 		AC_MSG_CHECKING([whether blkdev_put() accepts void* as arg 2])
 		ZFS_LINUX_TEST_RESULT([blkdev_put_holder], [
 			AC_MSG_RESULT(yes)
 			AC_DEFINE(HAVE_BLKDEV_PUT_HOLDER, 1,
 				[blkdev_put() accepts void* as arg 2])
 		], [
-			ZFS_LINUX_TEST_ERROR([blkdev_put()])
+			AC_MSG_RESULT(no)
+			AC_MSG_CHECKING([whether bdev_release() exists])
+			ZFS_LINUX_TEST_RESULT([bdev_release], [
+				AC_MSG_RESULT(yes)
+				AC_DEFINE(HAVE_BDEV_RELEASE, 1,
+					[bdev_release() exists])
+			], [
+				ZFS_LINUX_TEST_ERROR([blkdev_put()])
+			])
 		])
 	])
 ])
@@ -570,8 +620,10 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BLK_STS_RESV_CONFLICT], [
 AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV], [
 	ZFS_AC_KERNEL_SRC_BLKDEV_GET_BY_PATH
 	ZFS_AC_KERNEL_SRC_BLKDEV_GET_BY_PATH_4ARG
+	ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_OPEN_BY_PATH
 	ZFS_AC_KERNEL_SRC_BLKDEV_PUT
 	ZFS_AC_KERNEL_SRC_BLKDEV_PUT_HOLDER
+	ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_RELEASE
 	ZFS_AC_KERNEL_SRC_BLKDEV_REREAD_PART
 	ZFS_AC_KERNEL_SRC_BLKDEV_INVALIDATE_BDEV
 	ZFS_AC_KERNEL_SRC_BLKDEV_LOOKUP_BDEV

config/kernel-block-device-operations.m4

@@ -5,7 +5,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_CHECK_EVENTS], [
 	ZFS_LINUX_TEST_SRC([block_device_operations_check_events], [
 		#include <linux/blkdev.h>
 
-		unsigned int blk_check_events(struct gendisk *disk,
+		static unsigned int blk_check_events(struct gendisk *disk,
 		    unsigned int clearing) {
 			(void) disk, (void) clearing;
 			return (0);
@@ -34,7 +34,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_RELEASE_VOID], [
 	ZFS_LINUX_TEST_SRC([block_device_operations_release_void], [
 		#include <linux/blkdev.h>
 
-		void blk_release(struct gendisk *g, fmode_t mode) {
+		static void blk_release(struct gendisk *g, fmode_t mode) {
 			(void) g, (void) mode;
 			return;
 		}
@@ -56,7 +56,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_RELEASE_1ARG], [
 	ZFS_LINUX_TEST_SRC([block_device_operations_release_void_1arg], [
 		#include <linux/blkdev.h>
 
-		void blk_release(struct gendisk *g) {
+		static void blk_release(struct gendisk *g) {
 			(void) g;
 			return;
 		}
@@ -96,7 +96,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK], [
 	ZFS_LINUX_TEST_SRC([block_device_operations_revalidate_disk], [
 		#include <linux/blkdev.h>
 
-		int blk_revalidate_disk(struct gendisk *disk) {
+		static int blk_revalidate_disk(struct gendisk *disk) {
 			(void) disk;
 			return(0);
 		}

config/kernel-commit-metadata.m4

@@ -7,7 +7,7 @@ dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_COMMIT_METADATA], [
 	ZFS_LINUX_TEST_SRC([export_operations_commit_metadata], [
 		#include <linux/exportfs.h>
-		int commit_metadata(struct inode *inode) { return 0; }
+		static int commit_metadata(struct inode *inode) { return 0; }
 		static struct export_operations eops __attribute__ ((unused))={
 			.commit_metadata = commit_metadata,
 		};

config/kernel-dentry-operations.m4

@@ -98,7 +98,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_D_REVALIDATE_NAMEIDATA], [
 		#include <linux/dcache.h>
 		#include <linux/sched.h>
 
-		int revalidate (struct dentry *dentry,
+		static int revalidate (struct dentry *dentry,
 		    struct nameidata *nidata) { return 0; }
 
 		static const struct dentry_operations

config/kernel-dirty-inode.m4

@@ -8,7 +8,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_DIRTY_INODE], [
 	ZFS_LINUX_TEST_SRC([dirty_inode_with_flags], [
 		#include <linux/fs.h>
 
-		void dirty_inode(struct inode *a, int b) { return; }
+		static void dirty_inode(struct inode *a, int b) { return; }
 
 		static const struct super_operations
 		    sops __attribute__ ((unused)) = {

config/kernel-encode-fh-inode.m4

@@ -7,7 +7,7 @@ dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_ENCODE_FH_WITH_INODE], [
 	ZFS_LINUX_TEST_SRC([export_operations_encode_fh], [
 		#include <linux/exportfs.h>
-		int encode_fh(struct inode *inode, __u32 *fh, int *max_len,
+		static int encode_fh(struct inode *inode, __u32 *fh, int *max_len,
 		              struct inode *parent) { return 0; }
 		static struct export_operations eops __attribute__ ((unused))={
 			.encode_fh = encode_fh,

config/kernel-evict-inode.m4

@@ -6,7 +6,7 @@ dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_EVICT_INODE], [
 	ZFS_LINUX_TEST_SRC([evict_inode], [
 		#include <linux/fs.h>
-		void evict_inode (struct inode * t) { return; }
+		static void evict_inode (struct inode * t) { return; }
 		static struct super_operations sops __attribute__ ((unused)) = {
 			.evict_inode = evict_inode,
 		};

config/kernel-fallocate.m4

@@ -11,7 +11,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FALLOCATE], [
 	ZFS_LINUX_TEST_SRC([file_fallocate], [
 		#include <linux/fs.h>
 
-		long test_fallocate(struct file *file, int mode,
+		static long test_fallocate(struct file *file, int mode,
 		    loff_t offset, loff_t len) { return 0; }
 
 		static const struct file_operations

config/kernel-fsync.m4

@@ -5,7 +5,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FSYNC], [
 	ZFS_LINUX_TEST_SRC([fsync_without_dentry], [
 		#include <linux/fs.h>
 
-		int test_fsync(struct file *f, int x) { return 0; }
+		static int test_fsync(struct file *f, int x) { return 0; }
 
 		static const struct file_operations
 		    fops __attribute__ ((unused)) = {
@@ -16,7 +16,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FSYNC], [
 	ZFS_LINUX_TEST_SRC([fsync_range], [
 		#include <linux/fs.h>
 
-		int test_fsync(struct file *f, loff_t a, loff_t b, int c)
+		static int test_fsync(struct file *f, loff_t a, loff_t b, int c)
 		    { return 0; }
 
 		static const struct file_operations

config/kernel-get-link.m4

@@ -5,7 +5,7 @@ dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_GET_LINK], [
 	ZFS_LINUX_TEST_SRC([inode_operations_get_link], [
 		#include <linux/fs.h>
-		const char *get_link(struct dentry *de, struct inode *ip,
+		static const char *get_link(struct dentry *de, struct inode *ip,
 		    struct delayed_call *done) { return "symlink"; }
 		static struct inode_operations
 		     iops __attribute__ ((unused)) = {
@@ -15,7 +15,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_GET_LINK], [
 
 	ZFS_LINUX_TEST_SRC([inode_operations_get_link_cookie], [
 		#include <linux/fs.h>
-		const char *get_link(struct dentry *de, struct
+		static const char *get_link(struct dentry *de, struct
 		    inode *ip, void **cookie) { return "symlink"; }
 		static struct inode_operations
 		     iops __attribute__ ((unused)) = {
@@ -25,7 +25,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_GET_LINK], [
 
 	ZFS_LINUX_TEST_SRC([inode_operations_follow_link], [
 		#include <linux/fs.h>
-		const char *follow_link(struct dentry *de,
+		static const char *follow_link(struct dentry *de,
 		    void **cookie) { return "symlink"; }
 		static struct inode_operations
 		    iops __attribute__ ((unused)) = {
@@ -35,7 +35,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_GET_LINK], [
 
 	ZFS_LINUX_TEST_SRC([inode_operations_follow_link_nameidata], [
 		#include <linux/fs.h>
-		void *follow_link(struct dentry *de, struct
+		static void *follow_link(struct dentry *de, struct
 		    nameidata *nd) { return (void *)NULL; }
 		static struct inode_operations
 		    iops __attribute__ ((unused)) = {

config/kernel-idmap_mnt_api.m4

@@ -23,3 +23,28 @@ AC_DEFUN([ZFS_AC_KERNEL_IDMAP_MNT_API], [
         ])
 ])
 
+dnl #
+dnl # 6.8 decouples mnt_idmap from user_namespace. This is all internal
+dnl # to mnt_idmap so we can't detect it directly, but we detect a related
+dnl # change as use that as a signal.
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_IDMAP_NO_USERNS], [
+	ZFS_LINUX_TEST_SRC([idmap_no_userns], [
+		#include <linux/uidgid.h>
+	], [
+		struct uid_gid_map *map = NULL;
+		map_id_down(map, 0);
+	])
+])
+
+
+AC_DEFUN([ZFS_AC_KERNEL_IDMAP_NO_USERNS], [
+	AC_MSG_CHECKING([whether idmapped mounts have a user namespace])
+	ZFS_LINUX_TEST_RESULT([idmap_no_userns], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE(HAVE_IDMAP_NO_USERNS, 1,
+			[mnt_idmap does not have user_namespace])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])

config/kernel-inode-create.m4

@@ -7,7 +7,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_CREATE], [
 		#include <linux/fs.h>
 		#include <linux/sched.h>
 
-		int inode_create(struct mnt_idmap *idmap,
+		static int inode_create(struct mnt_idmap *idmap,
 		    struct inode *inode ,struct dentry *dentry,
 		    umode_t umode, bool flag) { return 0; }
 
@@ -25,7 +25,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_CREATE], [
 		#include <linux/fs.h>
 		#include <linux/sched.h>
 
-		int inode_create(struct user_namespace *userns,
+		static int inode_create(struct user_namespace *userns,
 		    struct inode *inode ,struct dentry *dentry,
 		    umode_t umode, bool flag) { return 0; }
 
@@ -42,7 +42,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_CREATE], [
 		#include <linux/fs.h>
 		#include <linux/sched.h>
 
-		int inode_create(struct inode *inode ,struct dentry *dentry,
+		static int inode_create(struct inode *inode ,struct dentry *dentry,
 		    umode_t umode, bool flag) { return 0; }
 
 		static const struct inode_operations

config/kernel-inode-getattr.m4

@@ -7,7 +7,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_GETATTR], [
 	ZFS_LINUX_TEST_SRC([inode_operations_getattr_mnt_idmap], [
 		#include <linux/fs.h>
 
-		int test_getattr(
+		static int test_getattr(
 		    struct mnt_idmap *idmap,
 		    const struct path *p, struct kstat *k,
 		    u32 request_mask, unsigned int query_flags)
@@ -28,7 +28,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_GETATTR], [
 	ZFS_LINUX_TEST_SRC([inode_operations_getattr_userns], [
 		#include <linux/fs.h>
 
-		int test_getattr(
+		static int test_getattr(
 			struct user_namespace *userns,
 		    const struct path *p, struct kstat *k,
 		    u32 request_mask, unsigned int query_flags)
@@ -47,7 +47,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_GETATTR], [
 	ZFS_LINUX_TEST_SRC([inode_operations_getattr_path], [
 		#include <linux/fs.h>
 
-		int test_getattr(
+		static int test_getattr(
 		    const struct path *p, struct kstat *k,
 		    u32 request_mask, unsigned int query_flags)
 		    { return 0; }
@@ -61,7 +61,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_GETATTR], [
 	ZFS_LINUX_TEST_SRC([inode_operations_getattr_vfsmount], [
 		#include <linux/fs.h>
 
-		int test_getattr(
+		static int test_getattr(
 		    struct vfsmount *mnt, struct dentry *d,
 		    struct kstat *k)
 		    { return 0; }

config/kernel-inode-lookup.m4

@@ -6,7 +6,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_LOOKUP_FLAGS], [
 		#include <linux/fs.h>
 		#include <linux/sched.h>
 
-		struct dentry *inode_lookup(struct inode *inode,
+		static struct dentry *inode_lookup(struct inode *inode,
 		    struct dentry *dentry, unsigned int flags) { return NULL; }
 
 		static const struct inode_operations iops

config/kernel-inode-permission.m4

@@ -8,12 +8,12 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_PERMISSION], [
 		#include <linux/fs.h>
 		#include <linux/sched.h>
 
-		int inode_permission(struct mnt_idmap *idmap,
+		static int test_permission(struct mnt_idmap *idmap,
 		    struct inode *inode, int mask) { return 0; }
 
 		static const struct inode_operations
 			iops __attribute__ ((unused)) = {
-			.permission             = inode_permission,
+			.permission             = test_permission,
 		};
 	],[])
 
@@ -25,12 +25,12 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_PERMISSION], [
 		#include <linux/fs.h>
 		#include <linux/sched.h>
 
-		int inode_permission(struct user_namespace *userns,
+		static int test_permission(struct user_namespace *userns,
 		    struct inode *inode, int mask) { return 0; }
 
 		static const struct inode_operations
 			iops __attribute__ ((unused)) = {
-			.permission		= inode_permission,
+			.permission		= test_permission,
 		};
 	],[])
 ])

config/kernel-inode-setattr.m4

@@ -7,7 +7,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_SETATTR], [
 	ZFS_LINUX_TEST_SRC([inode_operations_setattr_mnt_idmap], [
 		#include <linux/fs.h>
 
-		int test_setattr(
+		static int test_setattr(
 		    struct mnt_idmap *idmap,
 		    struct dentry *de, struct iattr *ia)
 		    { return 0; }
@@ -27,7 +27,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_SETATTR], [
 	ZFS_LINUX_TEST_SRC([inode_operations_setattr_userns], [
 		#include <linux/fs.h>
 
-		int test_setattr(
+		static int test_setattr(
 		    struct user_namespace *userns,
 		    struct dentry *de, struct iattr *ia)
 		    { return 0; }
@@ -41,7 +41,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_SETATTR], [
 	ZFS_LINUX_TEST_SRC([inode_operations_setattr], [
 		#include <linux/fs.h>
 
-		int test_setattr(
+		static int test_setattr(
 		    struct dentry *de, struct iattr *ia)
 		    { return 0; }
 

config/kernel-make-request-fn.m4

@@ -4,7 +4,7 @@ dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_MAKE_REQUEST_FN], [
 	ZFS_LINUX_TEST_SRC([make_request_fn_void], [
 		#include <linux/blkdev.h>
-		void make_request(struct request_queue *q,
+		static void make_request(struct request_queue *q,
 		    struct bio *bio) { return; }
 	],[
 		blk_queue_make_request(NULL, &make_request);
@@ -12,7 +12,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MAKE_REQUEST_FN], [
 
 	ZFS_LINUX_TEST_SRC([make_request_fn_blk_qc_t], [
 		#include <linux/blkdev.h>
-		blk_qc_t make_request(struct request_queue *q,
+		static blk_qc_t make_request(struct request_queue *q,
 		    struct bio *bio) { return (BLK_QC_T_NONE); }
 	],[
 		blk_queue_make_request(NULL, &make_request);
@@ -20,7 +20,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MAKE_REQUEST_FN], [
 
 	ZFS_LINUX_TEST_SRC([blk_alloc_queue_request_fn], [
 		#include <linux/blkdev.h>
-		blk_qc_t make_request(struct request_queue *q,
+		static blk_qc_t make_request(struct request_queue *q,
 		    struct bio *bio) { return (BLK_QC_T_NONE); }
 	],[
 		struct request_queue *q __attribute__ ((unused));
@@ -29,7 +29,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MAKE_REQUEST_FN], [
 
 	ZFS_LINUX_TEST_SRC([blk_alloc_queue_request_fn_rh], [
 		#include <linux/blkdev.h>
-		blk_qc_t make_request(struct request_queue *q,
+		static blk_qc_t make_request(struct request_queue *q,
 		    struct bio *bio) { return (BLK_QC_T_NONE); }
 	],[
 		struct request_queue *q __attribute__ ((unused));

config/kernel-mkdir.m4

@@ -9,7 +9,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MKDIR], [
 	ZFS_LINUX_TEST_SRC([mkdir_mnt_idmap], [
 		#include <linux/fs.h>
 
-		int mkdir(struct mnt_idmap *idmap,
+		static int mkdir(struct mnt_idmap *idmap,
 			struct inode *inode, struct dentry *dentry,
 			umode_t umode) { return 0; }
 		static const struct inode_operations
@@ -26,7 +26,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MKDIR], [
 	ZFS_LINUX_TEST_SRC([mkdir_user_namespace], [
 		#include <linux/fs.h>
 
-		int mkdir(struct user_namespace *userns,
+		static int mkdir(struct user_namespace *userns,
 			struct inode *inode, struct dentry *dentry,
 		    umode_t umode) { return 0; }
 
@@ -47,7 +47,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MKDIR], [
 	ZFS_LINUX_TEST_SRC([inode_operations_mkdir], [
 		#include <linux/fs.h>
 
-		int mkdir(struct inode *inode, struct dentry *dentry,
+		static int mkdir(struct inode *inode, struct dentry *dentry,
 		    umode_t umode) { return 0; }
 
 		static const struct inode_operations

config/kernel-mknod.m4

@@ -7,7 +7,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MKNOD], [
 		#include <linux/fs.h>
 		#include <linux/sched.h>
 
-		int tmp_mknod(struct mnt_idmap *idmap,
+		static int tmp_mknod(struct mnt_idmap *idmap,
 		    struct inode *inode ,struct dentry *dentry,
 		    umode_t u, dev_t d) { return 0; }
 
@@ -25,7 +25,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MKNOD], [
 		#include <linux/fs.h>
 		#include <linux/sched.h>
 
-		int tmp_mknod(struct user_namespace *userns,
+		static int tmp_mknod(struct user_namespace *userns,
 		    struct inode *inode ,struct dentry *dentry,
 		    umode_t u, dev_t d) { return 0; }
 

config/kernel-proc-operations.m4

@@ -7,14 +7,14 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_PROC_OPERATIONS], [
 	ZFS_LINUX_TEST_SRC([proc_ops_struct], [
 		#include <linux/proc_fs.h>
 
-		int test_open(struct inode *ip, struct file *fp) { return 0; }
+		static int test_open(struct inode *ip, struct file *fp) { return 0; }
-		ssize_t test_read(struct file *fp, char __user *ptr,
+		static ssize_t test_read(struct file *fp, char __user *ptr,
 		    size_t size, loff_t *offp) { return 0; }
-		ssize_t test_write(struct file *fp, const char __user *ptr,
+		static ssize_t test_write(struct file *fp, const char __user *ptr,
 		    size_t size, loff_t *offp) { return 0; }
-		loff_t test_lseek(struct file *fp, loff_t off, int flag)
+		static loff_t test_lseek(struct file *fp, loff_t off, int flag)
 		    { return 0; }
-		int test_release(struct inode *ip, struct file *fp)
+		static int test_release(struct inode *ip, struct file *fp)
 		    { return 0; }
 
 		const struct proc_ops test_ops __attribute__ ((unused)) = {

config/kernel-put-link.m4

@@ -4,7 +4,7 @@ dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_PUT_LINK], [
 	ZFS_LINUX_TEST_SRC([put_link_cookie], [
 		#include <linux/fs.h>
-		void put_link(struct inode *ip, void *cookie)
+		static void put_link(struct inode *ip, void *cookie)
 		    { return; }
 		static struct inode_operations
 		    iops __attribute__ ((unused)) = {
@@ -14,7 +14,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_PUT_LINK], [
 
 	ZFS_LINUX_TEST_SRC([put_link_nameidata], [
 		#include <linux/fs.h>
-		void put_link(struct dentry *de, struct
+		static void put_link(struct dentry *de, struct
 		    nameidata *nd, void *ptr) { return; }
 		static struct inode_operations
 		    iops __attribute__ ((unused)) = {

config/kernel-rename.m4

@@ -8,7 +8,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_RENAME], [
 	dnl #
 	ZFS_LINUX_TEST_SRC([inode_operations_rename2], [
 		#include <linux/fs.h>
-		int rename2_fn(struct inode *sip, struct dentry *sdp,
+		static int rename2_fn(struct inode *sip, struct dentry *sdp,
 			struct inode *tip, struct dentry *tdp,
 			unsigned int flags) { return 0; }
 
@@ -26,7 +26,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_RENAME], [
 	dnl #
 	ZFS_LINUX_TEST_SRC([inode_operations_rename_flags], [
 		#include <linux/fs.h>
-		int rename_fn(struct inode *sip, struct dentry *sdp,
+		static int rename_fn(struct inode *sip, struct dentry *sdp,
 			struct inode *tip, struct dentry *tdp,
 			unsigned int flags) { return 0; }
 
@@ -44,7 +44,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_RENAME], [
 	dnl #
 	ZFS_LINUX_TEST_SRC([dir_inode_operations_wrapper_rename2], [
 		#include <linux/fs.h>
-		int rename2_fn(struct inode *sip, struct dentry *sdp,
+		static int rename2_fn(struct inode *sip, struct dentry *sdp,
 			struct inode *tip, struct dentry *tdp,
 			unsigned int flags) { return 0; }
 
@@ -62,7 +62,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_RENAME], [
 	dnl #
 	ZFS_LINUX_TEST_SRC([inode_operations_rename_userns], [
 		#include <linux/fs.h>
-		int rename_fn(struct user_namespace *user_ns, struct inode *sip,
+		static int rename_fn(struct user_namespace *user_ns, struct inode *sip,
 			struct dentry *sdp, struct inode *tip, struct dentry *tdp,
 			unsigned int flags) { return 0; }
 
@@ -77,7 +77,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_RENAME], [
 	dnl #
 	ZFS_LINUX_TEST_SRC([inode_operations_rename_mnt_idmap], [
 		#include <linux/fs.h>
-		int rename_fn(struct mnt_idmap *idmap, struct inode *sip,
+		static int rename_fn(struct mnt_idmap *idmap, struct inode *sip,
 			struct dentry *sdp, struct inode *tip, struct dentry *tdp,
 			unsigned int flags) { return 0; }
 

config/kernel-show-options.m4

@@ -5,7 +5,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_SHOW_OPTIONS], [
 	ZFS_LINUX_TEST_SRC([super_operations_show_options], [
 		#include <linux/fs.h>
 
-		int show_options(struct seq_file * x, struct dentry * y) {
+		static int show_options(struct seq_file * x, struct dentry * y) {
 			return 0;
 		};
 

config/kernel-shrink.m4

@@ -8,9 +8,6 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_SUPER_BLOCK_S_SHRINK], [
 	ZFS_LINUX_TEST_SRC([super_block_s_shrink], [
 		#include <linux/fs.h>
 
-		int shrink(struct shrinker *s, struct shrink_control *sc)
-		    { return 0; }
-
 		static const struct super_block
 		    sb __attribute__ ((unused)) = {
 			.s_shrink.seeks = DEFAULT_SEEKS,
@@ -26,7 +23,7 @@ dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_SUPER_BLOCK_S_SHRINK_PTR], [
 	ZFS_LINUX_TEST_SRC([super_block_s_shrink_ptr], [
 		#include <linux/fs.h>
-		unsigned long shrinker_cb(struct shrinker *shrink,
+		static unsigned long shrinker_cb(struct shrinker *shrink,
 		    struct shrink_control *sc) { return 0; }
 		static struct shrinker shrinker = {
 			.count_objects = shrinker_cb,
@@ -89,7 +86,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SHRINK_CONTROL_HAS_NID], [
 AC_DEFUN([ZFS_AC_KERNEL_SRC_REGISTER_SHRINKER_VARARG], [
 	ZFS_LINUX_TEST_SRC([register_shrinker_vararg], [
 		#include <linux/mm.h>
-		unsigned long shrinker_cb(struct shrinker *shrink,
+		static unsigned long shrinker_cb(struct shrinker *shrink,
 		    struct shrink_control *sc) { return 0; }
 	],[
 		struct shrinker cache_shrinker = {
@@ -104,7 +101,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_REGISTER_SHRINKER_VARARG], [
 AC_DEFUN([ZFS_AC_KERNEL_SRC_SHRINKER_CALLBACK], [
 	ZFS_LINUX_TEST_SRC([shrinker_cb_shrink_control], [
 		#include <linux/mm.h>
-		int shrinker_cb(struct shrinker *shrink,
+		static int shrinker_cb(struct shrinker *shrink,
 		    struct shrink_control *sc) { return 0; }
 	],[
 		struct shrinker cache_shrinker = {
@@ -116,7 +113,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_SHRINKER_CALLBACK], [
 
 	ZFS_LINUX_TEST_SRC([shrinker_cb_shrink_control_split], [
 		#include <linux/mm.h>
-		unsigned long shrinker_cb(struct shrinker *shrink,
+		static unsigned long shrinker_cb(struct shrinker *shrink,
 		    struct shrink_control *sc) { return 0; }
 	],[
 		struct shrinker cache_shrinker = {
@@ -135,7 +132,7 @@ dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_SHRINKER_REGISTER], [
 	ZFS_LINUX_TEST_SRC([shrinker_register], [
 		#include <linux/shrinker.h>
-		unsigned long shrinker_cb(struct shrinker *shrink,
+		static unsigned long shrinker_cb(struct shrinker *shrink,
 		    struct shrink_control *sc) { return 0; }
 	],[
 		struct shrinker cache_shrinker = {

config/kernel-strlcpy.m4

@@ -0,0 +1,47 @@
+dnl #
+dnl # 6.8.x replaced strlcpy with strscpy. Check for both so we can provide
+dnl # appropriate fallbacks.
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_STRLCPY], [
+	ZFS_LINUX_TEST_SRC([kernel_has_strlcpy], [
+		#include <linux/string.h>
+	], [
+		const char *src = "goodbye";
+		char dst[32];
+		size_t len;
+		len = strlcpy(dst, src, sizeof (dst));
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SRC_STRSCPY], [
+	ZFS_LINUX_TEST_SRC([kernel_has_strscpy], [
+		#include <linux/string.h>
+	], [
+		const char *src = "goodbye";
+		char dst[32];
+		ssize_t len;
+		len = strscpy(dst, src, sizeof (dst));
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_STRLCPY], [
+	AC_MSG_CHECKING([whether strlcpy() exists])
+	ZFS_LINUX_TEST_RESULT([kernel_has_strlcpy], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE(HAVE_KERNEL_STRLCPY, 1,
+			[strlcpy() exists])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_STRSCPY], [
+	AC_MSG_CHECKING([whether strscpy() exists])
+	ZFS_LINUX_TEST_RESULT([kernel_has_strscpy], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE(HAVE_KERNEL_STRSCPY, 1,
+			[strscpy() exists])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])

config/kernel-symlink.m4

@@ -6,7 +6,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_SYMLINK], [
 	ZFS_LINUX_TEST_SRC([symlink_mnt_idmap], [
 		#include <linux/fs.h>
 		#include <linux/sched.h>
-		int tmp_symlink(struct mnt_idmap *idmap,
+		static int tmp_symlink(struct mnt_idmap *idmap,
 		    struct inode *inode ,struct dentry *dentry,
 		    const char *path) { return 0; }
 
@@ -23,7 +23,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_SYMLINK], [
 		#include <linux/fs.h>
 		#include <linux/sched.h>
 
-		int tmp_symlink(struct user_namespace *userns,
+		static int tmp_symlink(struct user_namespace *userns,
 		    struct inode *inode ,struct dentry *dentry,
 		    const char *path) { return 0; }
 

config/kernel-timer.m4

@@ -18,7 +18,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_TIMER_SETUP], [
 			int data;
 		};
 
-		void task_expire(struct timer_list *tl)
+		static void task_expire(struct timer_list *tl)
 		{
 			struct my_task_timer *task_timer =
 			    from_timer(task_timer, tl, timer);
@@ -31,7 +31,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_TIMER_SETUP], [
 
 	ZFS_LINUX_TEST_SRC([timer_list_function], [
 		#include <linux/timer.h>
-		void task_expire(struct timer_list *tl) {}
+		static void task_expire(struct timer_list *tl) {}
 	],[
 		struct timer_list tl;
 		tl.function = task_expire;

config/kernel-tmpfile.m4

@@ -9,7 +9,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_TMPFILE], [
 	dnl #
 	ZFS_LINUX_TEST_SRC([inode_operations_tmpfile_mnt_idmap], [
 		#include <linux/fs.h>
-		int tmpfile(struct mnt_idmap *idmap,
+		static int tmpfile(struct mnt_idmap *idmap,
 		    struct inode *inode, struct file *file,
 		    umode_t mode) { return 0; }
 		static struct inode_operations
@@ -22,7 +22,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_TMPFILE], [
 	dnl #
 	ZFS_LINUX_TEST_SRC([inode_operations_tmpfile], [
 		#include <linux/fs.h>
-		int tmpfile(struct user_namespace *userns,
+		static int tmpfile(struct user_namespace *userns,
 		    struct inode *inode, struct file *file,
 		    umode_t mode) { return 0; }
 		static struct inode_operations
@@ -36,7 +36,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_TMPFILE], [
 	dnl #
 	ZFS_LINUX_TEST_SRC([inode_operations_tmpfile_dentry_userns], [
 		#include <linux/fs.h>
-		int tmpfile(struct user_namespace *userns,
+		static int tmpfile(struct user_namespace *userns,
 		    struct inode *inode, struct dentry *dentry,
 		    umode_t mode) { return 0; }
 		static struct inode_operations
@@ -46,7 +46,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_TMPFILE], [
 	],[])
 	ZFS_LINUX_TEST_SRC([inode_operations_tmpfile_dentry], [
 			#include <linux/fs.h>
-			int tmpfile(struct inode *inode, struct dentry *dentry,
+			static int tmpfile(struct inode *inode, struct dentry *dentry,
 			    umode_t mode) { return 0; }
 			static struct inode_operations
 			    iops __attribute__ ((unused)) = {

config/kernel-vfs-direct_IO.m4

@@ -5,7 +5,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_DIRECT_IO], [
 	ZFS_LINUX_TEST_SRC([direct_io_iter], [
 		#include <linux/fs.h>
 
-		ssize_t test_direct_IO(struct kiocb *kiocb,
+		static ssize_t test_direct_IO(struct kiocb *kiocb,
 		    struct iov_iter *iter) { return 0; }
 
 		static const struct address_space_operations
@@ -17,7 +17,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_DIRECT_IO], [
 	ZFS_LINUX_TEST_SRC([direct_io_iter_offset], [
 		#include <linux/fs.h>
 
-		ssize_t test_direct_IO(struct kiocb *kiocb,
+		static ssize_t test_direct_IO(struct kiocb *kiocb,
 		    struct iov_iter *iter, loff_t offset) { return 0; }
 
 		static const struct address_space_operations
@@ -29,7 +29,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_DIRECT_IO], [
 	ZFS_LINUX_TEST_SRC([direct_io_iter_rw_offset], [
 		#include <linux/fs.h>
 
-		ssize_t test_direct_IO(int rw, struct kiocb *kiocb,
+		static ssize_t test_direct_IO(int rw, struct kiocb *kiocb,
 		    struct iov_iter *iter, loff_t offset) { return 0; }
 
 		static const struct address_space_operations
@@ -41,7 +41,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_DIRECT_IO], [
 	ZFS_LINUX_TEST_SRC([direct_io_iovec], [
 		#include <linux/fs.h>
 
-		ssize_t test_direct_IO(int rw, struct kiocb *kiocb,
+		static ssize_t test_direct_IO(int rw, struct kiocb *kiocb,
 		    const struct iovec *iov, loff_t offset,
 		    unsigned long nr_segs) { return 0; }
 

config/kernel-vfs-iterate.m4

@@ -1,7 +1,7 @@
 AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_ITERATE], [
 	ZFS_LINUX_TEST_SRC([file_operations_iterate_shared], [
 		#include <linux/fs.h>
-		int iterate(struct file *filp, struct dir_context * context)
+		static int iterate(struct file *filp, struct dir_context * context)
 		    { return 0; }
 
 		static const struct file_operations fops
@@ -12,7 +12,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_ITERATE], [
 
 	ZFS_LINUX_TEST_SRC([file_operations_iterate], [
 		#include <linux/fs.h>
-		int iterate(struct file *filp,
+		static int iterate(struct file *filp,
 		    struct dir_context *context) { return 0; }
 
 		static const struct file_operations fops
@@ -27,7 +27,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_ITERATE], [
 
 	ZFS_LINUX_TEST_SRC([file_operations_readdir], [
 		#include <linux/fs.h>
-		int readdir(struct file *filp, void *entry,
+		static int readdir(struct file *filp, void *entry,
 		    filldir_t func) { return 0; }
 
 		static const struct file_operations fops

config/kernel-vfs-rw-iterate.m4

@@ -5,9 +5,9 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_RW_ITERATE], [
 	ZFS_LINUX_TEST_SRC([file_operations_rw], [
 		#include <linux/fs.h>
 
-		ssize_t test_read(struct kiocb *kiocb, struct iov_iter *to)
+		static ssize_t test_read(struct kiocb *kiocb, struct iov_iter *to)
 		    { return 0; }
-		ssize_t test_write(struct kiocb *kiocb, struct iov_iter *from)
+		static ssize_t test_write(struct kiocb *kiocb, struct iov_iter *from)
 		    { return 0; }
 
 		static const struct file_operations

config/kernel-writepage_t.m4

@@ -6,7 +6,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_WRITEPAGE_T], [
 	dnl #
 	ZFS_LINUX_TEST_SRC([writepage_t_folio], [
 		#include <linux/writeback.h>
-		int putpage(struct folio *folio,
+		static int putpage(struct folio *folio,
 		    struct writeback_control *wbc, void *data)
 		{ return 0; }
 		writepage_t func = putpage;

config/kernel-xattr-handler.m4

@@ -68,7 +68,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_GET], [
 	ZFS_LINUX_TEST_SRC([xattr_handler_get_dentry_inode], [
 		#include <linux/xattr.h>
 
-		int get(const struct xattr_handler *handler,
+		static int get(const struct xattr_handler *handler,
 		    struct dentry *dentry, struct inode *inode,
 		    const char *name, void *buffer, size_t size) { return 0; }
 		static const struct xattr_handler
@@ -80,7 +80,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_GET], [
 	ZFS_LINUX_TEST_SRC([xattr_handler_get_xattr_handler], [
 		#include <linux/xattr.h>
 
-		int get(const struct xattr_handler *handler,
+		static int get(const struct xattr_handler *handler,
 		    struct dentry *dentry, const char *name,
 		    void *buffer, size_t size) { return 0; }
 		static const struct xattr_handler
@@ -92,7 +92,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_GET], [
 	ZFS_LINUX_TEST_SRC([xattr_handler_get_dentry], [
 		#include <linux/xattr.h>
 
-		int get(struct dentry *dentry, const char *name,
+		static int get(struct dentry *dentry, const char *name,
 		    void *buffer, size_t size, int handler_flags)
 		    { return 0; }
 		static const struct xattr_handler
@@ -104,7 +104,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_GET], [
 	ZFS_LINUX_TEST_SRC([xattr_handler_get_dentry_inode_flags], [
 		#include <linux/xattr.h>
 
-		int get(const struct xattr_handler *handler,
+		static int get(const struct xattr_handler *handler,
 		    struct dentry *dentry, struct inode *inode,
 		    const char *name, void *buffer,
 		    size_t size, int flags) { return 0; }
@@ -182,7 +182,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_SET], [
 	ZFS_LINUX_TEST_SRC([xattr_handler_set_mnt_idmap], [
 		#include <linux/xattr.h>
 
-		int set(const struct xattr_handler *handler,
+		static int set(const struct xattr_handler *handler,
 			struct mnt_idmap *idmap,
 			struct dentry *dentry, struct inode *inode,
 			const char *name, const void *buffer,
@@ -197,7 +197,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_SET], [
 	ZFS_LINUX_TEST_SRC([xattr_handler_set_userns], [
 		#include <linux/xattr.h>
 
-		int set(const struct xattr_handler *handler,
+		static int set(const struct xattr_handler *handler,
 			struct user_namespace *mnt_userns,
 			struct dentry *dentry, struct inode *inode,
 			const char *name, const void *buffer,
@@ -212,7 +212,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_SET], [
 	ZFS_LINUX_TEST_SRC([xattr_handler_set_dentry_inode], [
 		#include <linux/xattr.h>
 
-		int set(const struct xattr_handler *handler,
+		static int set(const struct xattr_handler *handler,
 		    struct dentry *dentry, struct inode *inode,
 		    const char *name, const void *buffer,
 		    size_t size, int flags)
@@ -226,7 +226,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_SET], [
 	ZFS_LINUX_TEST_SRC([xattr_handler_set_xattr_handler], [
 		#include <linux/xattr.h>
 
-		int set(const struct xattr_handler *handler,
+		static int set(const struct xattr_handler *handler,
 		    struct dentry *dentry, const char *name,
 		    const void *buffer, size_t size, int flags)
 		    { return 0; }
@@ -239,7 +239,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_SET], [
 	ZFS_LINUX_TEST_SRC([xattr_handler_set_dentry], [
 		#include <linux/xattr.h>
 
-		int set(struct dentry *dentry, const char *name,
+		static int set(struct dentry *dentry, const char *name,
 		    const void *buffer, size_t size, int flags,
 		    int handler_flags) { return 0; }
 		static const struct xattr_handler
@@ -325,7 +325,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_LIST], [
 	ZFS_LINUX_TEST_SRC([xattr_handler_list_simple], [
 		#include <linux/xattr.h>
 
-		bool list(struct dentry *dentry) { return 0; }
+		static bool list(struct dentry *dentry) { return 0; }
 		static const struct xattr_handler
 		    xops __attribute__ ((unused)) = {
 			.list = list,
@@ -335,7 +335,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_LIST], [
 	ZFS_LINUX_TEST_SRC([xattr_handler_list_xattr_handler], [
 		#include <linux/xattr.h>
 
-		size_t list(const struct xattr_handler *handler,
+		static size_t list(const struct xattr_handler *handler,
 		    struct dentry *dentry, char *list, size_t list_size,
 		    const char *name, size_t name_len) { return 0; }
 		static const struct xattr_handler
@@ -347,7 +347,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_LIST], [
 	ZFS_LINUX_TEST_SRC([xattr_handler_list_dentry], [
 		#include <linux/xattr.h>
 
-		size_t list(struct dentry *dentry,
+		static size_t list(struct dentry *dentry,
 		    char *list, size_t list_size,
 		    const char *name, size_t name_len,
 		    int handler_flags) { return 0; }

config/kernel.m4

@@ -149,6 +149,8 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [
 	ZFS_AC_KERNEL_SRC_SYSFS
 	ZFS_AC_KERNEL_SRC_SET_SPECIAL_STATE
 	ZFS_AC_KERNEL_SRC_STANDALONE_LINUX_STDARG
+	ZFS_AC_KERNEL_SRC_STRLCPY
+	ZFS_AC_KERNEL_SRC_STRSCPY
 	ZFS_AC_KERNEL_SRC_PAGEMAP_FOLIO_WAIT_BIT
 	ZFS_AC_KERNEL_SRC_ADD_DISK
 	ZFS_AC_KERNEL_SRC_KTHREAD
@@ -156,6 +158,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [
 	ZFS_AC_KERNEL_SRC___COPY_FROM_USER_INATOMIC
 	ZFS_AC_KERNEL_SRC_USER_NS_COMMON_INUM
 	ZFS_AC_KERNEL_SRC_IDMAP_MNT_API
+	ZFS_AC_KERNEL_SRC_IDMAP_NO_USERNS
 	ZFS_AC_KERNEL_SRC_IATTR_VFSID
 	ZFS_AC_KERNEL_SRC_FILEMAP
 	ZFS_AC_KERNEL_SRC_WRITEPAGE_T
@@ -294,6 +297,8 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [
 	ZFS_AC_KERNEL_SYSFS
 	ZFS_AC_KERNEL_SET_SPECIAL_STATE
 	ZFS_AC_KERNEL_STANDALONE_LINUX_STDARG
+	ZFS_AC_KERNEL_STRLCPY
+	ZFS_AC_KERNEL_STRSCPY
 	ZFS_AC_KERNEL_PAGEMAP_FOLIO_WAIT_BIT
 	ZFS_AC_KERNEL_ADD_DISK
 	ZFS_AC_KERNEL_KTHREAD
@@ -301,6 +306,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [
 	ZFS_AC_KERNEL___COPY_FROM_USER_INATOMIC
 	ZFS_AC_KERNEL_USER_NS_COMMON_INUM
 	ZFS_AC_KERNEL_IDMAP_MNT_API
+	ZFS_AC_KERNEL_IDMAP_NO_USERNS
 	ZFS_AC_KERNEL_IATTR_VFSID
 	ZFS_AC_KERNEL_FILEMAP
 	ZFS_AC_KERNEL_WRITEPAGE_T

config/rpm.am

@@ -83,6 +83,11 @@ srpm-common:
 		rpm-local || exit 1; \
 	LANG=C $(RPMBUILD) \
 		--define "_tmppath $$rpmbuild/TMP" \
+		--define "_builddir $$rpmbuild/BUILD" \
+		--define "_rpmdir $$rpmbuild/RPMS" \
+		--define "_srcrpmdir $$rpmbuild/SRPMS" \
+		--define "_specdir $$rpmbuild/SPECS" \
+		--define "_sourcedir $$rpmbuild/SOURCES" \
 		--define "_topdir $$rpmbuild" \
 		$(def) -bs $$rpmbuild/SPECS/$$rpmspec || exit 1; \
 	cp $$rpmbuild/SRPMS/$$rpmpkg . || exit 1; \
@@ -99,6 +104,11 @@ rpm-common:
 		rpm-local || exit 1; \
 	LANG=C ${RPMBUILD} \
 		--define "_tmppath $$rpmbuild/TMP" \
+		--define "_builddir $$rpmbuild/BUILD" \
+		--define "_rpmdir $$rpmbuild/RPMS" \
+		--define "_srcrpmdir $$rpmbuild/SRPMS" \
+		--define "_specdir $$rpmbuild/SPECS" \
+		--define "_sourcedir $$rpmbuild/SOURCES" \
 		--define "_topdir $$rpmbuild" \
 		$(def) --rebuild $$rpmpkg || exit 1; \
 	cp $$rpmbuild/RPMS/*/* . || exit 1; \

include/libzfs.h

@@ -318,6 +318,9 @@ _LIBZFS_H int zpool_vdev_remove_wanted(zpool_handle_t *, const char *);
 
 _LIBZFS_H int zpool_vdev_fault(zpool_handle_t *, uint64_t, vdev_aux_t);
 _LIBZFS_H int zpool_vdev_degrade(zpool_handle_t *, uint64_t, vdev_aux_t);
+_LIBZFS_H int zpool_vdev_set_removed_state(zpool_handle_t *, uint64_t,
+    vdev_aux_t);
+
 _LIBZFS_H int zpool_vdev_clear(zpool_handle_t *, uint64_t);
 
 _LIBZFS_H nvlist_t *zpool_find_vdev(zpool_handle_t *, const char *, boolean_t *,

include/libzutil.h

@@ -97,6 +97,7 @@ _LIBZUTIL_H int zpool_find_config(libpc_handle_t *, const char *, nvlist_t **,
 _LIBZUTIL_H const char * const * zpool_default_search_paths(size_t *count);
 _LIBZUTIL_H int zpool_read_label(int, nvlist_t **, int *);
 _LIBZUTIL_H int zpool_label_disk_wait(const char *, int);
+_LIBZUTIL_H int zpool_disk_wait(const char *);
 
 struct udev_device;
 
@@ -163,6 +164,8 @@ _LIBZUTIL_H void zfs_niceraw(uint64_t, char *, size_t);
 _LIBZUTIL_H void zpool_dump_ddt(const ddt_stat_t *, const ddt_histogram_t *);
 _LIBZUTIL_H int zpool_history_unpack(char *, uint64_t, uint64_t *, nvlist_t ***,
     uint_t *);
+_LIBZUTIL_H void fsleep(float sec);
+_LIBZUTIL_H int zpool_getenv_int(const char *env, int default_val);
 
 struct zfs_cmd;
 
@@ -205,9 +208,65 @@ _LIBZUTIL_H void zfs_setproctitle(const char *fmt, ...);
 typedef int (*pool_vdev_iter_f)(void *, nvlist_t *, void *);
 int for_each_vdev_cb(void *zhp, nvlist_t *nv, pool_vdev_iter_f func,
     void *data);
+int for_each_vdev_macro_helper_func(void *zhp_data, nvlist_t *nv, void *data);
+int for_each_real_leaf_vdev_macro_helper_func(void *zhp_data, nvlist_t *nv,
+    void *data);
+/*
+ * Often you'll want to iterate over all the vdevs in the pool, but don't want
+ * to use for_each_vdev() since it requires a callback function.
+ *
+ * Instead you can use FOR_EACH_VDEV():
+ *
+ *     zpool_handle_t *zhp      // Assume this is initialized
+ *     nvlist_t *nv
+ *     ...
+ *     FOR_EACH_VDEV(zhp, nv) {
+ *	 const char *path = NULL;
+ *	 nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path);
+ *	 printf("Looking at vdev %s\n", path);
+ *     }
+ *
+ * Note: FOR_EACH_VDEV runs in O(n^2) time where n = number of vdevs.  However,
+ * there's an upper limit of 256 vdevs per dRAID top-level vdevs (TLDs), 255 for
+ * raidz2 TLDs, a real world limit of ~500 vdevs for mirrors, so this shouldn't
+ * really be an issue.
+ *
+ * Here are some micro-benchmarks of a complete FOR_EACH_VDEV loop on a RAID0
+ * pool:
+ *
+ * 100  vdevs = 0.7ms
+ * 500  vdevs = 17ms
+ * 750  vdevs = 40ms
+ * 1000 vdevs = 82ms
+ *
+ * The '__nv += 0' at the end of the for() loop gets around a "comma or
+ * semicolon followed by non-blank" checkstyle error.  Note on most compliers
+ * the '__nv += 0' can just be replaced with 'NULL', but gcc on Centos 7
+ * will give a 'warning: statement with no effect' error if you do that.
+ */
+#define	__FOR_EACH_VDEV(__zhp, __nv, __func) { \
+	__nv = zpool_get_config(__zhp, NULL); \
+	VERIFY0(nvlist_lookup_nvlist(__nv, ZPOOL_CONFIG_VDEV_TREE, &__nv)); \
+	} \
+	for (nvlist_t *__root_nv = __nv, *__state = (nvlist_t *)0; \
+	    for_each_vdev_cb(&__state, __root_nv, __func, &__nv) == 1; \
+	    __nv += 0)
+
+#define	FOR_EACH_VDEV(__zhp, __nv) \
+	__FOR_EACH_VDEV(__zhp, __nv, for_each_vdev_macro_helper_func)
+
+/*
+ * "real leaf" vdevs are leaf vdevs that are real devices (disks or files).
+ * This excludes leaf vdevs like like draid spares.
+ */
+#define	FOR_EACH_REAL_LEAF_VDEV(__zhp, __nv) \
+	__FOR_EACH_VDEV(__zhp, __nv, for_each_real_leaf_vdev_macro_helper_func)
+
 int for_each_vdev_in_nvlist(nvlist_t *nvroot, pool_vdev_iter_f func,
     void *data);
 void update_vdevs_config_dev_sysfs_path(nvlist_t *config);
+_LIBZUTIL_H void update_vdev_config_dev_sysfs_path(nvlist_t *nv,
+    const char *path, const char *key);
 #ifdef	__cplusplus
 }
 #endif

include/os/linux/spl/sys/cred.h

@@ -73,13 +73,25 @@ static inline struct user_namespace *zfs_i_user_ns(struct inode *inode)
 static inline boolean_t zfs_no_idmapping(struct user_namespace *mnt_userns,
     struct user_namespace *fs_userns)
 {
-	return (zfs_is_init_userns(mnt_userns) || mnt_userns == fs_userns);
+	return (zfs_is_init_userns(mnt_userns) ||
+	    mnt_userns == fs_userns);
 }
 
 static inline uid_t zfs_uid_to_vfsuid(zidmap_t *mnt_userns,
     struct user_namespace *fs_userns, uid_t uid)
 {
-	struct user_namespace *owner = idmap_owner(mnt_userns);
+	struct user_namespace *owner;
+#ifdef HAVE_IOPS_CREATE_IDMAP
+	if (mnt_userns == zfs_init_idmap)
+		return (uid);
+#endif
+#ifdef HAVE_IDMAP_NO_USERNS
+	struct user_namespace ns;
+	ns.uid_map = mnt_userns->uid_map;
+	owner = &ns;
+#else
+	owner = idmap_owner(mnt_userns);
+#endif
 	if (zfs_no_idmapping(owner, fs_userns))
 		return (uid);
 	if (!zfs_is_init_userns(fs_userns))
@@ -92,7 +104,18 @@ static inline uid_t zfs_uid_to_vfsuid(zidmap_t *mnt_userns,
 static inline gid_t zfs_gid_to_vfsgid(zidmap_t *mnt_userns,
     struct user_namespace *fs_userns, gid_t gid)
 {
-	struct user_namespace *owner = idmap_owner(mnt_userns);
+	struct user_namespace *owner;
+#ifdef HAVE_IOPS_CREATE_IDMAP
+	if (mnt_userns == zfs_init_idmap)
+		return (gid);
+#endif
+#ifdef HAVE_IDMAP_NO_USERNS
+	struct user_namespace ns;
+	ns.gid_map = mnt_userns->gid_map;
+	owner = &ns;
+#else
+	owner = idmap_owner(mnt_userns);
+#endif
 	if (zfs_no_idmapping(owner, fs_userns))
 		return (gid);
 	if (!zfs_is_init_userns(fs_userns))
@@ -105,7 +128,18 @@ static inline gid_t zfs_gid_to_vfsgid(zidmap_t *mnt_userns,
 static inline uid_t zfs_vfsuid_to_uid(zidmap_t *mnt_userns,
     struct user_namespace *fs_userns, uid_t uid)
 {
-	struct user_namespace *owner = idmap_owner(mnt_userns);
+	struct user_namespace *owner;
+#ifdef HAVE_IOPS_CREATE_IDMAP
+	if (mnt_userns == zfs_init_idmap)
+		return (uid);
+#endif
+#ifdef HAVE_IDMAP_NO_USERNS
+	struct user_namespace ns;
+	ns.uid_map = mnt_userns->uid_map;
+	owner = &ns;
+#else
+	owner = idmap_owner(mnt_userns);
+#endif
 	if (zfs_no_idmapping(owner, fs_userns))
 		return (uid);
 	uid = from_kuid(owner, KUIDT_INIT(uid));
@@ -119,7 +153,18 @@ static inline uid_t zfs_vfsuid_to_uid(zidmap_t *mnt_userns,
 static inline gid_t zfs_vfsgid_to_gid(zidmap_t *mnt_userns,
     struct user_namespace *fs_userns, gid_t gid)
 {
-	struct user_namespace *owner = idmap_owner(mnt_userns);
+	struct user_namespace *owner;
+#ifdef HAVE_IOPS_CREATE_IDMAP
+	if (mnt_userns == zfs_init_idmap)
+		return (gid);
+#endif
+#ifdef HAVE_IDMAP_NO_USERNS
+	struct user_namespace ns;
+	ns.gid_map = mnt_userns->gid_map;
+	owner = &ns;
+#else
+	owner = idmap_owner(mnt_userns);
+#endif
 	if (zfs_no_idmapping(owner, fs_userns))
 		return (gid);
 	gid = from_kgid(owner, KGIDT_INIT(gid));

include/os/linux/spl/sys/string.h

@@ -1 +1,50 @@
+/*
+ *  Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
+ *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
+ *  Written by Brian Behlendorf <behlendorf1@llnl.gov>.
+ *  UCRL-CODE-235197
+ *
+ *  This file is part of the SPL, Solaris Porting Layer.
+ *
+ *  The SPL is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or (at your
+ *  option) any later version.
+ *
+ *  The SPL is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _SPL_STRING_H
+#define	_SPL_STRING_H
+
 #include <linux/string.h>
+
+/* Fallbacks for kernel missing strlcpy */
+#ifndef HAVE_KERNEL_STRLCPY
+
+#if defined(HAVE_KERNEL_STRSCPY)
+/*
+ * strscpy is strlcpy, but returns an error on truncation. strlcpy is defined
+ * to return strlen(src), so detect error and override it.
+ */
+static inline size_t
+strlcpy(char *dest, const char *src, size_t size)
+{
+	ssize_t ret = strscpy(dest, src, size);
+	if (likely(ret > 0))
+		return ((size_t)ret);
+	return (strlen(src));
+}
+#else
+#error "no strlcpy fallback available"
+#endif
+
+#endif /* HAVE_KERNEL_STRLCPY */
+
+#endif /* _SPL_STRING_H */

include/os/linux/spl/sys/types.h

@@ -57,12 +57,23 @@ typedef int			minor_t;
 struct user_namespace;
 #ifdef HAVE_IOPS_CREATE_IDMAP
 #include <linux/refcount.h>
+#ifdef HAVE_IDMAP_NO_USERNS
+#include <linux/user_namespace.h>
+struct mnt_idmap {
+	struct uid_gid_map uid_map;
+	struct uid_gid_map gid_map;
+	refcount_t count;
+};
+typedef struct mnt_idmap	zidmap_t;
+#define	idmap_owner(p)	(NULL)
+#else
 struct mnt_idmap {
 	struct user_namespace *owner;
 	refcount_t count;
 };
 typedef struct mnt_idmap	zidmap_t;
 #define	idmap_owner(p)	(((struct mnt_idmap *)p)->owner)
+#endif
 #else
 typedef struct user_namespace	zidmap_t;
 #define	idmap_owner(p)	((struct user_namespace *)p)

include/sys/spa_impl.h

@@ -266,6 +266,7 @@ struct spa {
 
 	spa_aux_vdev_t	spa_spares;		/* hot spares */
 	spa_aux_vdev_t	spa_l2cache;		/* L2ARC cache devices */
+	boolean_t	spa_aux_sync_uber;	/* need to sync aux uber */
 	nvlist_t	*spa_label_features;	/* Features for reading MOS */
 	uint64_t	spa_config_object;	/* MOS object for pool config */
 	uint64_t	spa_config_generation;	/* config generation number */

lib/libspl/include/os/freebsd/sys/stat.h

@@ -76,8 +76,12 @@ fstat64_blk(int fd, struct stat64 *st)
 /*
  * Only Intel-based Macs have a separate stat64; Arm-based Macs are like
  * FreeBSD and have a full 64-bit stat from the start.
+ *
+ * On Linux, musl libc is full 64-bit too and has deprecated its own version
+ * of these defines since version 1.2.4.
  */
-#if defined(__APPLE__) && !(defined(__i386__) || defined(__x86_64__))
+#if (defined(__APPLE__) && !(defined(__i386__) || defined(__x86_64__))) || \
+	(defined(__linux__) && !defined(__GLIBC__))
 #define	stat64	stat
 #define	fstat64	fstat
 #endif

lib/libzfs/libzfs.abi

@@ -179,6 +179,7 @@
     <elf-symbol name='fletcher_4_native' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='fletcher_4_native_varsize' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='fletcher_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='fsleep' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='get_dataset_depth' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='get_system_hostid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='getexecname' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -260,6 +261,7 @@
     <elf-symbol name='tpool_suspended' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='tpool_wait' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='update_vdev_config_dev_strs' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='update_vdev_config_dev_sysfs_path' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='use_color' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='vdev_expand_proplist' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='vdev_name_to_prop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -465,6 +467,7 @@
     <elf-symbol name='zpool_disable_datasets_os' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_disable_volume_os' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_discard_checkpoint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='zpool_disk_wait' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_dump_ddt' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_enable_datasets' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_events_clear' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -496,6 +499,7 @@
     <elf-symbol name='zpool_get_userprop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_get_vdev_prop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_get_vdev_prop_value' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='zpool_getenv_int' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_history_unpack' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_import' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_import_props' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -566,6 +570,7 @@
     <elf-symbol name='zpool_vdev_remove_wanted' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_vdev_script_alloc_env' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_vdev_script_free_env' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='zpool_vdev_set_removed_state' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_vdev_split' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_wait' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_wait_status' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -1401,8 +1406,6 @@
     <qualified-type-def type-id='0897719a' const='yes' id='c4a7b189'/>
     <pointer-type-def type-id='c4a7b189' size-in-bits='64' id='36fca399'/>
     <qualified-type-def type-id='36fca399' restrict='yes' id='37e4897b'/>
-    <qualified-type-def type-id='a9c79a1f' const='yes' id='cd087e36'/>
-    <pointer-type-def type-id='cd087e36' size-in-bits='64' id='e05e8614'/>
     <qualified-type-def type-id='e05e8614' restrict='yes' id='0be2e71c'/>
     <pointer-type-def type-id='8037c762' size-in-bits='64' id='d74a6869'/>
     <qualified-type-def type-id='7292109c' restrict='yes' id='6942f6a4'/>
@@ -6355,6 +6358,12 @@
       <parameter type-id='9d774e0b' name='aux'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='zpool_vdev_set_removed_state' mangled-name='zpool_vdev_set_removed_state' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_set_removed_state'>
+      <parameter type-id='4c81de99' name='zhp'/>
+      <parameter type-id='9c313c2d' name='guid'/>
+      <parameter type-id='9d774e0b' name='aux'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='zpool_vdev_attach' mangled-name='zpool_vdev_attach' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_attach'>
       <parameter type-id='4c81de99' name='zhp'/>
       <parameter type-id='80f4b756' name='old_disk'/>
@@ -7588,6 +7597,12 @@
     <qualified-type-def type-id='d33f11cb' restrict='yes' id='5c53ba29'/>
     <pointer-type-def type-id='ffa52b96' size-in-bits='64' id='76c8174b'/>
     <pointer-type-def type-id='f3d87113' size-in-bits='64' id='0d2a0670'/>
+    <function-decl name='zpool_label_disk' mangled-name='zpool_label_disk' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_label_disk'>
+      <parameter type-id='b0382bb3'/>
+      <parameter type-id='4c81de99'/>
+      <parameter type-id='80f4b756'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='zfs_version_kernel' mangled-name='zfs_version_kernel' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_version_kernel'>
       <return type-id='26a90f95'/>
     </function-decl>
@@ -7597,6 +7612,10 @@
     <function-decl name='libzfs_core_fini' visibility='default' binding='global' size-in-bits='64'>
       <return type-id='48b5725f'/>
     </function-decl>
+    <function-decl name='zfs_get_underlying_path' mangled-name='zfs_get_underlying_path' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_underlying_path'>
+      <parameter type-id='80f4b756'/>
+      <return type-id='26a90f95'/>
+    </function-decl>
     <function-decl name='zpool_prop_unsupported' mangled-name='zpool_prop_unsupported' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prop_unsupported'>
       <parameter type-id='80f4b756'/>
       <return type-id='c19b74c3'/>
@@ -7714,6 +7733,11 @@
       <parameter type-id='b59d7dce'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='access' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='80f4b756'/>
+      <parameter type-id='95e97e5e'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='dup2' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='95e97e5e'/>
       <parameter type-id='95e97e5e'/>
@@ -7881,6 +7905,37 @@
       <parameter is-variadic='yes'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='zpool_vdev_script_alloc_env' mangled-name='zpool_vdev_script_alloc_env' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_script_alloc_env'>
+      <parameter type-id='80f4b756' name='pool_name'/>
+      <parameter type-id='80f4b756' name='vdev_path'/>
+      <parameter type-id='80f4b756' name='vdev_upath'/>
+      <parameter type-id='80f4b756' name='vdev_enc_sysfs_path'/>
+      <parameter type-id='80f4b756' name='opt_key'/>
+      <parameter type-id='80f4b756' name='opt_val'/>
+      <return type-id='9b23c9ad'/>
+    </function-decl>
+    <function-decl name='zpool_vdev_script_free_env' mangled-name='zpool_vdev_script_free_env' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_script_free_env'>
+      <parameter type-id='9b23c9ad' name='env'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
+    <function-decl name='zpool_prepare_disk' mangled-name='zpool_prepare_disk' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prepare_disk'>
+      <parameter type-id='4c81de99' name='zhp'/>
+      <parameter type-id='5ce45b60' name='vdev_nv'/>
+      <parameter type-id='80f4b756' name='prepare_str'/>
+      <parameter type-id='c0563f85' name='lines'/>
+      <parameter type-id='7292109c' name='lines_cnt'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
+    <function-decl name='zpool_prepare_and_label_disk' mangled-name='zpool_prepare_and_label_disk' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prepare_and_label_disk'>
+      <parameter type-id='b0382bb3' name='hdl'/>
+      <parameter type-id='4c81de99' name='zhp'/>
+      <parameter type-id='80f4b756' name='name'/>
+      <parameter type-id='5ce45b60' name='vdev_nv'/>
+      <parameter type-id='80f4b756' name='prepare_str'/>
+      <parameter type-id='c0563f85' name='lines'/>
+      <parameter type-id='7292109c' name='lines_cnt'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='lib/libzfs/os/linux/libzfs_mount_os.c' language='LANG_C99'>
     <pointer-type-def type-id='7359adad' size-in-bits='64' id='1d2c2b85'/>
@@ -8070,12 +8125,6 @@
       <parameter type-id='95e97e5e'/>
       <return type-id='95e97e5e'/>
     </function-decl>
-    <function-decl name='zpool_label_disk' mangled-name='zpool_label_disk' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_label_disk'>
-      <parameter type-id='b0382bb3' name='hdl'/>
-      <parameter type-id='4c81de99' name='zhp'/>
-      <parameter type-id='80f4b756' name='name'/>
-      <return type-id='95e97e5e'/>
-    </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='lib/libzfs/os/linux/libzfs_util_os.c' language='LANG_C99'>
     <class-decl name='itimerspec' size-in-bits='256' is-struct='yes' visibility='default' id='acbdbcc6'>
@@ -8102,11 +8151,6 @@
     <pointer-type-def type-id='4ba62af7' size-in-bits='64' id='f39579e7'/>
     <pointer-type-def type-id='acbdbcc6' size-in-bits='64' id='116842ac'/>
     <pointer-type-def type-id='b440e872' size-in-bits='64' id='3ac36db0'/>
-    <function-decl name='access' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='80f4b756'/>
-      <parameter type-id='95e97e5e'/>
-      <return type-id='95e97e5e'/>
-    </function-decl>
     <function-decl name='__poll_chk' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='3ac36db0'/>
       <parameter type-id='555eef66'/>
@@ -8191,10 +8235,6 @@
       <parameter type-id='80f4b756' name='dev_name'/>
       <return type-id='c19b74c3'/>
     </function-decl>
-    <function-decl name='zfs_get_underlying_path' mangled-name='zfs_get_underlying_path' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_get_underlying_path'>
-      <parameter type-id='80f4b756' name='dev_name'/>
-      <return type-id='26a90f95'/>
-    </function-decl>
     <function-decl name='is_mpath_whole_disk' mangled-name='is_mpath_whole_disk' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='is_mpath_whole_disk'>
       <parameter type-id='80f4b756' name='path'/>
       <return type-id='c19b74c3'/>
@@ -8330,6 +8370,16 @@
       <parameter type-id='b59d7dce' name='buflen'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='zpool_disk_wait' mangled-name='zpool_disk_wait' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_disk_wait'>
+      <parameter type-id='80f4b756' name='path'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
+    <function-decl name='update_vdev_config_dev_sysfs_path' mangled-name='update_vdev_config_dev_sysfs_path' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='update_vdev_config_dev_sysfs_path'>
+      <parameter type-id='5ce45b60' name='nv'/>
+      <parameter type-id='80f4b756' name='path'/>
+      <parameter type-id='80f4b756' name='key'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-type size-in-bits='64' id='2ec2411e'>
       <parameter type-id='eaa32e2f'/>
       <parameter type-id='5ce45b60'/>
@@ -8349,6 +8399,9 @@
       <parameter type-id='95e97e5e'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='clearenv' visibility='default' binding='global' size-in-bits='64'>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='zfs_setproctitle_init' mangled-name='zfs_setproctitle_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_setproctitle_init'>
       <parameter type-id='95e97e5e' name='argc'/>
       <parameter type-id='9b23c9ad' name='argv'/>
@@ -8622,6 +8675,7 @@
     <array-type-def dimensions='1' type-id='853fd5dc' size-in-bits='32768' id='b505fc2f'>
       <subrange length='64' type-id='7359adad' id='b10be967'/>
     </array-type-def>
+    <type-decl name='float' size-in-bits='32' id='a6c45d85'/>
     <class-decl name='ddt_stat' size-in-bits='512' is-struct='yes' visibility='default' id='65242dfe'>
       <data-member access='public' layout-offset-in-bits='0'>
         <var-decl name='dds_blocks' type-id='9c313c2d' visibility='default'/>
@@ -8659,11 +8713,27 @@
     <pointer-type-def type-id='ec92d602' size-in-bits='64' id='932720f8'/>
     <qualified-type-def type-id='853fd5dc' const='yes' id='764c298c'/>
     <pointer-type-def type-id='764c298c' size-in-bits='64' id='dfe59052'/>
+    <qualified-type-def type-id='a9c79a1f' const='yes' id='cd087e36'/>
+    <pointer-type-def type-id='cd087e36' size-in-bits='64' id='e05e8614'/>
+    <function-decl name='nanosleep' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='e05e8614'/>
+      <parameter type-id='3d83ba87'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='zpool_dump_ddt' mangled-name='zpool_dump_ddt' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_dump_ddt'>
       <parameter type-id='dfe59052' name='dds_total'/>
       <parameter type-id='932720f8' name='ddh'/>
       <return type-id='48b5725f'/>
     </function-decl>
+    <function-decl name='fsleep' mangled-name='fsleep' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fsleep'>
+      <parameter type-id='a6c45d85' name='sec'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
+    <function-decl name='zpool_getenv_int' mangled-name='zpool_getenv_int' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_getenv_int'>
+      <parameter type-id='80f4b756' name='env'/>
+      <parameter type-id='95e97e5e' name='default_val'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='module/avl/avl.c' language='LANG_C99'>
     <function-decl name='avl_last' mangled-name='avl_last' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='avl_last'>

lib/libzfs/libzfs_import.c

@@ -291,10 +291,8 @@ zpool_in_use(libzfs_handle_t *hdl, int fd, pool_state_t *state, char **namestr,
 
 	*inuse = B_FALSE;
 
-	if (zpool_read_label(fd, &config, NULL) != 0) {
+	if (zpool_read_label(fd, &config, NULL) != 0)
-		(void) no_memory(hdl);
 		return (-1);
-	}
 
 	if (config == NULL)
 		return (0);

lib/libzfs/libzfs_pool.c

@@ -3036,6 +3036,9 @@ zpool_vdev_is_interior(const char *name)
 	return (B_FALSE);
 }
 
+/*
+ * Lookup the nvlist for a given vdev.
+ */
 nvlist_t *
 zpool_find_vdev(zpool_handle_t *zhp, const char *path, boolean_t *avail_spare,
     boolean_t *l2cache, boolean_t *log)
@@ -3043,6 +3046,7 @@ zpool_find_vdev(zpool_handle_t *zhp, const char *path, boolean_t *avail_spare,
 	char *end;
 	nvlist_t *nvroot, *search, *ret;
 	uint64_t guid;
+	boolean_t __avail_spare, __l2cache, __log;
 
 	search = fnvlist_alloc();
 
@@ -3058,6 +3062,18 @@ zpool_find_vdev(zpool_handle_t *zhp, const char *path, boolean_t *avail_spare,
 	nvroot = fnvlist_lookup_nvlist(zhp->zpool_config,
 	    ZPOOL_CONFIG_VDEV_TREE);
 
+	/*
+	 * User can pass NULL for avail_spare, l2cache, and log, but
+	 * we still need to provide variables to vdev_to_nvlist_iter(), so
+	 * just point them to junk variables here.
+	 */
+	if (!avail_spare)
+		avail_spare = &__avail_spare;
+	if (!l2cache)
+		l2cache = &__l2cache;
+	if (!log)
+		log = &__log;
+
 	*avail_spare = B_FALSE;
 	*l2cache = B_FALSE;
 	if (log != NULL)
@@ -3315,21 +3331,23 @@ zpool_vdev_fault(zpool_handle_t *zhp, uint64_t guid, vdev_aux_t aux)
 }
 
 /*
- * Mark the given vdev degraded.
+ * Generic set vdev state function
  */
-int
+static int
-zpool_vdev_degrade(zpool_handle_t *zhp, uint64_t guid, vdev_aux_t aux)
+zpool_vdev_set_state(zpool_handle_t *zhp, uint64_t guid, vdev_aux_t aux,
+    vdev_state_t state)
 {
 	zfs_cmd_t zc = {"\0"};
 	char errbuf[ERRBUFLEN];
 	libzfs_handle_t *hdl = zhp->zpool_hdl;
 
 	(void) snprintf(errbuf, sizeof (errbuf),
-	    dgettext(TEXT_DOMAIN, "cannot degrade %llu"), (u_longlong_t)guid);
+	    dgettext(TEXT_DOMAIN, "cannot set %s %llu"),
+	    zpool_state_to_name(state, aux), (u_longlong_t)guid);
 
 	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
 	zc.zc_guid = guid;
-	zc.zc_cookie = VDEV_STATE_DEGRADED;
+	zc.zc_cookie = state;
 	zc.zc_obj = aux;
 
 	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
@@ -3338,6 +3356,27 @@ zpool_vdev_degrade(zpool_handle_t *zhp, uint64_t guid, vdev_aux_t aux)
 	return (zpool_standard_error(hdl, errno, errbuf));
 }
 
+/*
+ * Mark the given vdev degraded.
+ */
+int
+zpool_vdev_degrade(zpool_handle_t *zhp, uint64_t guid, vdev_aux_t aux)
+{
+	return (zpool_vdev_set_state(zhp, guid, aux, VDEV_STATE_DEGRADED));
+}
+
+/*
+ * Mark the given vdev as in a removed state (as if the device does not exist).
+ *
+ * This is different than zpool_vdev_remove() which does a removal of a device
+ * from the pool (but the device does exist).
+ */
+int
+zpool_vdev_set_removed_state(zpool_handle_t *zhp, uint64_t guid, vdev_aux_t aux)
+{
+	return (zpool_vdev_set_state(zhp, guid, aux, VDEV_STATE_REMOVED));
+}
+
 /*
  * Returns TRUE if the given nvlist is a vdev that was originally swapped in as
  * a hot spare.

lib/libzutil/os/freebsd/zutil_import_os.c

@@ -249,6 +249,15 @@ zfs_dev_flush(int fd)
 	return (0);
 }
 
+void
+update_vdev_config_dev_sysfs_path(nvlist_t *nv, const char *path,
+    const char *key)
+{
+	(void) nv;
+	(void) path;
+	(void) key;
+}
+
 void
 update_vdevs_config_dev_sysfs_path(nvlist_t *config)
 {

lib/libzutil/os/linux/zutil_import_os.c

@@ -170,25 +170,17 @@ zpool_open_func(void *arg)
 	if (rn->rn_labelpaths) {
 		const char *path = NULL;
 		const char *devid = NULL;
-		const char *env = NULL;
 		rdsk_node_t *slice;
 		avl_index_t where;
-		int timeout;
 		int error;
 
 		if (label_paths(rn->rn_hdl, rn->rn_config, &path, &devid))
 			return;
 
-		env = getenv("ZPOOL_IMPORT_UDEV_TIMEOUT_MS");
-		if ((env == NULL) || sscanf(env, "%d", &timeout) != 1 ||
-		    timeout < 0) {
-			timeout = DISK_LABEL_WAIT;
-		}
-
 		/*
 		 * Allow devlinks to stabilize so all paths are available.
 		 */
-		zpool_label_disk_wait(rn->rn_name, timeout);
+		zpool_disk_wait(rn->rn_name);
 
 		if (path != NULL) {
 			slice = zutil_alloc(hdl, sizeof (rdsk_node_t));
@@ -682,6 +674,20 @@ zpool_label_disk_wait(const char *path, int timeout_ms)
 #endif /* HAVE_LIBUDEV */
 }
 
+/*
+ * Simplified version of zpool_label_disk_wait() where we wait for a device
+ * to appear using the default timeouts.
+ */
+int
+zpool_disk_wait(const char *path)
+{
+	int timeout;
+	timeout = zpool_getenv_int("ZPOOL_IMPORT_UDEV_TIMEOUT_MS",
+	    DISK_LABEL_WAIT);
+
+	return (zpool_label_disk_wait(path, timeout));
+}
+
 /*
  * Encode the persistent devices strings
  * used for the vdev disk label
@@ -766,20 +772,37 @@ no_dev:
  * Rescan the enclosure sysfs path for turning on enclosure LEDs and store it
  * in the nvlist * (if applicable).  Like:
  *    vdev_enc_sysfs_path: '/sys/class/enclosure/11:0:1:0/SLOT 4'
+ *
+ * If an old path was in the nvlist, and the rescan can not find a new path,
+ * then keep the old path, since the disk may have been removed.
+ *
+ * path: The vdev path (value from ZPOOL_CONFIG_PATH)
+ * key: The nvlist_t name (like ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH)
  */
-static void
+void
-update_vdev_config_dev_sysfs_path(nvlist_t *nv, const char *path)
+update_vdev_config_dev_sysfs_path(nvlist_t *nv, const char *path,
+    const char *key)
 {
 	char *upath, *spath;
+	const char *oldpath = NULL;
+
+	(void) nvlist_lookup_string(nv, key, &oldpath);
 
 	/* Add enclosure sysfs path (if disk is in an enclosure). */
 	upath = zfs_get_underlying_path(path);
 	spath = zfs_get_enclosure_sysfs_path(upath);
 
 	if (spath) {
-		nvlist_add_string(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH, spath);
+		(void) nvlist_add_string(nv, key, spath);
 	} else {
-		nvlist_remove_all(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
+		/*
+		 * We couldn't dynamically scan the disk's enclosure sysfs path.
+		 * This could be because the disk went away.  If there's an old
+		 * enclosure sysfs path in the nvlist, then keep using it.
+		 */
+		if (!oldpath) {
+			(void) nvlist_remove_all(nv, key);
+		}
 	}
 
 	free(upath);
@@ -799,7 +822,8 @@ sysfs_path_pool_vdev_iter_f(void *hdl_data, nvlist_t *nv, void *data)
 		return (1);
 
 	/* Rescan our enclosure sysfs path for this vdev */
-	update_vdev_config_dev_sysfs_path(nv, path);
+	update_vdev_config_dev_sysfs_path(nv, path,
+	    ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
 	return (0);
 }
 
@@ -888,7 +912,8 @@ update_vdev_config_dev_strs(nvlist_t *nv)
 			(void) nvlist_add_string(nv, ZPOOL_CONFIG_PHYS_PATH,
 			    vds.vds_devphys);
 		}
-		update_vdev_config_dev_sysfs_path(nv, path);
+		update_vdev_config_dev_sysfs_path(nv, path,
+		    ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
 	} else {
 		/* Clear out any stale entries. */
 		(void) nvlist_remove_all(nv, ZPOOL_CONFIG_DEVID);

lib/libzutil/zutil_import.c

@@ -1056,10 +1056,21 @@ zpool_read_label(int fd, nvlist_t **config, int *num_labels)
 				case EINVAL:
 					break;
 				case EINPROGRESS:
-					// This shouldn't be possible to
+					/*
-					// encounter, die if we do.
+					 * This shouldn't be possible to
+					 * encounter, die if we do.
+					 */
 					ASSERT(B_FALSE);
 					zfs_fallthrough;
+				case EREMOTEIO:
+					/*
+					 * May be returned by an NVMe device
+					 * which is visible in /dev/ but due
+					 * to a low-level format change, or
+					 * other error, needs to be rescanned.
+					 * Try the slow method.
+					 */
+					zfs_fallthrough;
 				case EOPNOTSUPP:
 				case ENOSYS:
 					do_slow = B_TRUE;
@@ -1210,13 +1221,26 @@ label_paths(libpc_handle_t *hdl, nvlist_t *label, const char **path,
 	nvlist_t *nvroot;
 	uint64_t pool_guid;
 	uint64_t vdev_guid;
+	uint64_t state;
 
 	*path = NULL;
 	*devid = NULL;
+	if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &vdev_guid) != 0)
+		return (ENOENT);
+
+	/*
+	 * In case of spare or l2cache, we directly return path/devid from the
+	 * label.
+	 */
+	if (!(nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_STATE, &state)) &&
+	    (state == POOL_STATE_SPARE || state == POOL_STATE_L2CACHE)) {
+		(void) nvlist_lookup_string(label, ZPOOL_CONFIG_PATH, path);
+		(void) nvlist_lookup_string(label, ZPOOL_CONFIG_DEVID, devid);
+		return (0);
+	}
 
 	if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_VDEV_TREE, &nvroot) ||
-	    nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_GUID, &pool_guid) ||
+	    nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_GUID, &pool_guid))
-	    nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &vdev_guid))
 		return (ENOENT);
 
 	return (label_paths_impl(hdl, nvroot, pool_guid, vdev_guid, path,
@@ -1898,6 +1922,104 @@ zpool_find_config(libpc_handle_t *hdl, const char *target, nvlist_t **configp,
 	return (0);
 }
 
+/* Return if a vdev is a leaf vdev.  Note: draid spares are leaf vdevs. */
+static boolean_t
+vdev_is_leaf(nvlist_t *nv)
+{
+	uint_t children = 0;
+	nvlist_t **child;
+
+	(void) nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
+	    &child, &children);
+
+	return (children == 0);
+}
+
+/* Return if a vdev is a leaf vdev and a real device (disk or file) */
+static boolean_t
+vdev_is_real_leaf(nvlist_t *nv)
+{
+	const char *type = NULL;
+	if (!vdev_is_leaf(nv))
+		return (B_FALSE);
+
+	(void) nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type);
+	if ((strcmp(type, VDEV_TYPE_DISK) == 0) ||
+	    (strcmp(type, VDEV_TYPE_FILE) == 0)) {
+		return (B_TRUE);
+	}
+
+	return (B_FALSE);
+}
+
+/*
+ * This function is called by our FOR_EACH_VDEV() macros.
+ *
+ * state:   State machine status (stored inside of a (nvlist_t *))
+ * nv:	     The current vdev nvlist_t we are iterating over.
+ * last_nv: The previous vdev nvlist_t we returned to the user in
+ *          the last iteration of FOR_EACH_VDEV().  We use it
+ *          to find the next vdev nvlist_t we should return.
+ * real_leaves_only: Only return leaf vdevs.
+ *
+ * Returns 1 if we found the next vdev nvlist_t for this iteration.  0 if
+ * we're still searching for it.
+ */
+static int
+__for_each_vdev_macro_helper_func(void *state, nvlist_t *nv, void *last_nv,
+    boolean_t real_leaves_only)
+{
+	enum {FIRST_NV = 0, NEXT_IS_MATCH = 1, STOP_LOOKING = 2};
+
+	/* The very first entry in the NV list is a special case */
+	if (*((nvlist_t **)state) == (nvlist_t *)FIRST_NV) {
+		if (real_leaves_only && !vdev_is_real_leaf(nv))
+			return (0);
+
+		*((nvlist_t **)last_nv) = nv;
+		*((nvlist_t **)state) = (nvlist_t *)STOP_LOOKING;
+		return (1);
+	}
+
+	/*
+	 * We came across our last_nv, meaning the next one is the one we
+	 * want
+	 */
+	if (nv == *((nvlist_t **)last_nv)) {
+		/* Next iteration of this function will return the nvlist_t */
+		*((nvlist_t **)state) = (nvlist_t *)NEXT_IS_MATCH;
+		return (0);
+	}
+
+	/*
+	 * We marked NEXT_IS_MATCH on the previous iteration, so this is the one
+	 * we want.
+	 */
+	if (*(nvlist_t **)state == (nvlist_t *)NEXT_IS_MATCH) {
+		if (real_leaves_only && !vdev_is_real_leaf(nv))
+			return (0);
+
+		*((nvlist_t **)last_nv) = nv;
+		*((nvlist_t **)state) = (nvlist_t *)STOP_LOOKING;
+		return (1);
+	}
+
+	return (0);
+}
+
+int
+for_each_vdev_macro_helper_func(void *state, nvlist_t *nv, void *last_nv)
+{
+	return (__for_each_vdev_macro_helper_func(state, nv, last_nv, B_FALSE));
+}
+
+int
+for_each_real_leaf_vdev_macro_helper_func(void *state, nvlist_t *nv,
+    void *last_nv)
+{
+	return (__for_each_vdev_macro_helper_func(state, nv, last_nv, B_TRUE));
+}
+
 /*
  * Internal function for iterating over the vdevs.
  *

lib/libzutil/zutil_pool.c

@@ -28,6 +28,7 @@
 #include <string.h>
 #include <sys/nvpair.h>
 #include <sys/fs/zfs.h>
+#include <math.h>
 
 #include <libzutil.h>
 
@@ -144,3 +145,33 @@ zpool_history_unpack(char *buf, uint64_t bytes_read, uint64_t *leftover,
 	*leftover = bytes_read;
 	return (0);
 }
+
+/*
+ * Floating point sleep().  Allows you to pass in a floating point value for
+ * seconds.
+ */
+void
+fsleep(float sec)
+{
+	struct timespec req;
+	req.tv_sec = floor(sec);
+	req.tv_nsec = (sec - (float)req.tv_sec) * NANOSEC;
+	nanosleep(&req, NULL);
+}
+
+/*
+ * Get environment variable 'env' and return it as an integer.
+ * If 'env' is not set, then return 'default_val' instead.
+ */
+int
+zpool_getenv_int(const char *env, int default_val)
+{
+	char *str;
+	int val;
+	str = getenv(env);
+	if ((str == NULL) || sscanf(str, "%d", &val) != 1 ||
+	    val < 0) {
+		val = default_val;
+	}
+	return (val);
+}

man/man8/zpool-clear.8

@@ -36,6 +36,7 @@
 .Sh SYNOPSIS
 .Nm zpool
 .Cm clear
+.Op Fl -power
 .Ar pool
 .Oo Ar device Oc Ns …
 .
@@ -52,6 +53,16 @@ Pools with
 enabled which have been suspended cannot be resumed.
 While the pool was suspended, it may have been imported on
 another host, and resuming I/O could result in pool damage.
+.Bl -tag -width Ds
+.It Fl -power
+Power on the devices's slot in the storage enclosure and wait for the device
+to show up before attempting to clear errors.
+This is done on all the devices specified.
+Alternatively, you can set the
+.Sy ZPOOL_AUTO_POWER_ON_SLOT
+environment variable to always enable this behavior.
+Note: This flag currently works on Linux only.
+.El
 .
 .Sh SEE ALSO
 .Xr zdb 8 ,

man/man8/zpool-iostat.8

@@ -146,7 +146,7 @@ Specify
 .Sy u
 for a printed representation of the internal representation of time.
 See
-.Xr time 2 .
+.Xr time 1 .
 Specify
 .Sy d
 for standard date format.

man/man8/zpool-list.8

@@ -95,7 +95,7 @@ Specify
 .Sy u
 for a printed representation of the internal representation of time.
 See
-.Xr time 2 .
+.Xr time 1 .
 Specify
 .Sy d
 for standard date format.

man/man8/zpool-offline.8

@@ -36,12 +36,13 @@
 .Sh SYNOPSIS
 .Nm zpool
 .Cm offline
-.Op Fl ft
+.Op Fl Sy -power Ns | Ns Op Fl Sy ft
 .Ar pool
 .Ar device Ns …
 .Nm zpool
 .Cm online
-.Op Fl e
+.Op Fl Sy -power
+.Op Fl Sy e
 .Ar pool
 .Ar device Ns …
 .
@@ -50,7 +51,7 @@
 .It Xo
 .Nm zpool
 .Cm offline
-.Op Fl ft
+.Op Fl Sy -power Ns | Ns Op Fl Sy ft
 .Ar pool
 .Ar device Ns …
 .Xc
@@ -60,6 +61,9 @@ While the
 is offline, no attempt is made to read or write to the device.
 This command is not applicable to spares.
 .Bl -tag -width Ds
+.It Fl -power
+Power off the device's slot in the storage enclosure.
+This flag currently works on Linux only
 .It Fl f
 Force fault.
 Instead of offlining the disk, put it into a faulted state.
@@ -73,6 +77,7 @@ Upon reboot, the specified physical device reverts to its previous state.
 .It Xo
 .Nm zpool
 .Cm online
+.Op Fl -power
 .Op Fl e
 .Ar pool
 .Ar device Ns …
@@ -80,6 +85,13 @@ Upon reboot, the specified physical device reverts to its previous state.
 Brings the specified physical device online.
 This command is not applicable to spares.
 .Bl -tag -width Ds
+.It Fl -power
+Power on the device's slot in the storage enclosure and wait for the device
+to show up before attempting to online it.
+Alternatively, you can set the
+.Sy ZPOOL_AUTO_POWER_ON_SLOT
+environment variable to always enable this behavior.
+This flag currently works on Linux only
 .It Fl e
 Expand the device to use all available space.
 If the device is part of a mirror or raidz then all devices must be expanded

man/man8/zpool-status.8

@@ -57,6 +57,8 @@ and the estimated time to completion.
 Both of these are only approximate, because the amount of data in the pool and
 the other workloads on the system can change.
 .Bl -tag -width Ds
+.It Fl -power
+Display vdev enclosure slot power status (on or off).
 .It Fl c Op Ar SCRIPT1 Ns Oo , Ns Ar SCRIPT2 Oc Ns …
 Run a script (or scripts) on each vdev and include the output as a new column
 in the
@@ -110,7 +112,7 @@ Specify
 .Sy u
 for a printed representation of the internal representation of time.
 See
-.Xr time 2 .
+.Xr time 1 .
 Specify
 .Sy d
 for standard date format.

man/man8/zpool-wait.8

@@ -97,7 +97,7 @@ Specify
 .Sy u
 for a printed representation of the internal representation of time.
 See
-.Xr time 2 .
+.Xr time 1 .
 Specify
 .Sy d
 for standard date format.

man/man8/zpool.8

@@ -444,7 +444,7 @@ rpool       14.6G  54.9G      4     55   250K  2.69M
 .Ed
 .
 .Sh ENVIRONMENT VARIABLES
-.Bl -tag -compact -width "ZPOOL_IMPORT_UDEV_TIMEOUT_MS"
+.Bl -tag -compact -width "ZPOOL_STATUS_NON_NATIVE_ASHIFT_IGNORE"
 .It Sy ZFS_ABORT
 Cause
 .Nm
@@ -456,6 +456,23 @@ Use ANSI color in
 and
 .Nm zpool Cm iostat
 output.
+.It Sy ZPOOL_AUTO_POWER_ON_SLOT
+Automatically attempt to turn on the drives enclosure slot power to a drive when
+running the
+.Nm zpool Cm online
+or
+.Nm zpool Cm clear
+commands.
+This has the same effect as passing the
+.Fl -power
+option to those commands.
+.It Sy ZPOOL_POWER_ON_SLOT_TIMEOUT_MS
+The maximum time in milliseconds to wait for a slot power sysfs value
+to return the correct value after writing it.
+For example, after writing "on" to the sysfs enclosure slot power_control file,
+it can take some time for the enclosure to power down the slot and return
+"on" if you read back the 'power_control' value.
+Defaults to 30 seconds (30000ms) if not set.
 .It Sy ZPOOL_IMPORT_PATH
 The search path for devices or files to use with the pool.
 This is a colon-separated list of directories in which

module/os/freebsd/zfs/zfs_file_os.c

@@ -53,26 +53,65 @@ int
 zfs_file_open(const char *path, int flags, int mode, zfs_file_t **fpp)
 {
 	struct thread *td;
-	int rc, fd;
+	struct vnode *vp;
+	struct file *fp;
+	struct nameidata nd;
+	int error;
 
 	td = curthread;
 	pwd_ensure_dirs();
-	/* 12.x doesn't take a const char * */
+
-	rc = kern_openat(td, AT_FDCWD, __DECONST(char *, path),
+	KASSERT((flags & (O_EXEC | O_PATH)) == 0,
-	    UIO_SYSSPACE, flags, mode);
+	    ("invalid flags: 0x%x", flags));
-	if (rc)
+	KASSERT((flags & O_ACCMODE) != O_ACCMODE,
-		return (SET_ERROR(rc));
+	    ("invalid flags: 0x%x", flags));
-	fd = td->td_retval[0];
+	flags = FFLAGS(flags);
-	td->td_retval[0] = 0;
+
-	if (fget(curthread, fd, &cap_no_rights, fpp))
+	error = falloc_noinstall(td, &fp);
-		kern_close(td, fd);
+	if (error != 0) {
+		return (error);
+	}
+	fp->f_flag = flags & FMASK;
+
+#if __FreeBSD_version >= 1400043
+	NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, path);
+#else
+	NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, path, td);
+#endif
+	error = vn_open(&nd, &flags, mode, fp);
+	if (error != 0) {
+		falloc_abort(td, fp);
+		return (SET_ERROR(error));
+	}
+	NDFREE_PNBUF(&nd);
+	vp = nd.ni_vp;
+	fp->f_vnode = vp;
+	if (fp->f_ops == &badfileops) {
+		finit_vnode(fp, flags, NULL, &vnops);
+	}
+	VOP_UNLOCK(vp);
+	if (vp->v_type != VREG) {
+		zfs_file_close(fp);
+		return (SET_ERROR(EACCES));
+	}
+
+	if (flags & O_TRUNC) {
+		error = fo_truncate(fp, 0, td->td_ucred, td);
+		if (error != 0) {
+			zfs_file_close(fp);
+			return (SET_ERROR(error));
+		}
+	}
+
+	*fpp = fp;
+
 	return (0);
 }
 
 void
 zfs_file_close(zfs_file_t *fp)
 {
-	fo_close(fp, curthread);
+	fdrop(fp, curthread);
 }
 
 static int
@@ -263,7 +302,7 @@ zfs_file_get(int fd)
 void
 zfs_file_put(zfs_file_t *fp)
 {
-	fdrop(fp, curthread);
+	zfs_file_close(fp);
 }
 
 loff_t

module/os/linux/spl/spl-condvar.c

@@ -20,7 +20,7 @@
  *  You should have received a copy of the GNU General Public License along
  *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
  *
- *  Solaris Porting Layer (SPL) Credential Implementation.
+ *  Solaris Porting Layer (SPL) Condition Variables Implementation.
  */
 
 #include <sys/condvar.h>

module/os/linux/spl/spl-kmem-cache.c

@@ -28,6 +28,7 @@
 #include <sys/timer.h>
 #include <sys/vmem.h>
 #include <sys/wait.h>
+#include <sys/string.h>
 #include <linux/slab.h>
 #include <linux/swap.h>
 #include <linux/prefetch.h>
@@ -91,7 +92,8 @@ MODULE_PARM_DESC(spl_kmem_cache_max_size, "Maximum size of slab in MB");
  * of 16K was determined to be optimal for architectures using 4K pages and
  * to also work well on architecutres using larger 64K page sizes.
  */
-static unsigned int spl_kmem_cache_slab_limit = 16384;
+static unsigned int spl_kmem_cache_slab_limit =
+    SPL_MAX_KMEM_ORDER_NR_PAGES * PAGE_SIZE;
 module_param(spl_kmem_cache_slab_limit, uint, 0644);
 MODULE_PARM_DESC(spl_kmem_cache_slab_limit,
 	"Objects less than N bytes use the Linux slab");
@@ -783,7 +785,7 @@ spl_kmem_cache_create(const char *name, size_t size, size_t align,
 	} else {
 		unsigned long slabflags = 0;
 
-		if (size > (SPL_MAX_KMEM_ORDER_NR_PAGES * PAGE_SIZE))
+		if (size > spl_kmem_cache_slab_limit)
 			goto out;
 
 #if defined(SLAB_USERCOPY)

module/os/linux/spl/spl-kstat.c

@@ -32,6 +32,7 @@
 #include <sys/vmem.h>
 #include <sys/cmn_err.h>
 #include <sys/sysmacros.h>
+#include <sys/string.h>
 
 static kmutex_t kstat_module_lock;
 static struct list_head kstat_module_list;

module/os/linux/spl/spl-thread.c

@@ -26,6 +26,7 @@
 #include <sys/thread.h>
 #include <sys/kmem.h>
 #include <sys/tsd.h>
+#include <sys/string.h>
 
 /*
  * Thread interfaces

module/os/linux/spl/spl-zone.c

@@ -30,6 +30,7 @@
 #include <linux/file.h>
 #include <linux/magic.h>
 #include <sys/zone.h>
+#include <sys/string.h>
 
 #if defined(CONFIG_USER_NS)
 #include <linux/statfs.h>

module/os/linux/zfs/abd_os.c

@@ -60,8 +60,16 @@
 #ifdef _KERNEL
 #include <linux/kmap_compat.h>
 #include <linux/scatterlist.h>
+#endif
+
+#ifdef _KERNEL
+#if defined(MAX_ORDER)
+#define	ABD_MAX_ORDER	(MAX_ORDER)
+#elif defined(MAX_PAGE_ORDER)
+#define	ABD_MAX_ORDER	(MAX_PAGE_ORDER)
+#endif
 #else
-#define	MAX_ORDER	1
+#define	ABD_MAX_ORDER	(1)
 #endif
 
 typedef struct abd_stats {
@@ -71,7 +79,7 @@ typedef struct abd_stats {
 	kstat_named_t abdstat_scatter_cnt;
 	kstat_named_t abdstat_scatter_data_size;
 	kstat_named_t abdstat_scatter_chunk_waste;
-	kstat_named_t abdstat_scatter_orders[MAX_ORDER];
+	kstat_named_t abdstat_scatter_orders[ABD_MAX_ORDER];
 	kstat_named_t abdstat_scatter_page_multi_chunk;
 	kstat_named_t abdstat_scatter_page_multi_zone;
 	kstat_named_t abdstat_scatter_page_alloc_retry;
@@ -139,7 +147,7 @@ static struct {
 	wmsum_t abdstat_scatter_cnt;
 	wmsum_t abdstat_scatter_data_size;
 	wmsum_t abdstat_scatter_chunk_waste;
-	wmsum_t abdstat_scatter_orders[MAX_ORDER];
+	wmsum_t abdstat_scatter_orders[ABD_MAX_ORDER];
 	wmsum_t abdstat_scatter_page_multi_chunk;
 	wmsum_t abdstat_scatter_page_multi_zone;
 	wmsum_t abdstat_scatter_page_alloc_retry;
@@ -222,7 +230,7 @@ abd_free_struct_impl(abd_t *abd)
 }
 
 #ifdef _KERNEL
-static unsigned zfs_abd_scatter_max_order = MAX_ORDER - 1;
+static unsigned zfs_abd_scatter_max_order = ABD_MAX_ORDER - 1;
 
 /*
  * Mark zfs data pages so they can be excluded from kernel crash dumps
@@ -272,7 +280,8 @@ abd_alloc_chunks(abd_t *abd, size_t size)
 	struct page *page, *tmp_page = NULL;
 	gfp_t gfp = __GFP_NOWARN | GFP_NOIO;
 	gfp_t gfp_comp = (gfp | __GFP_NORETRY | __GFP_COMP) & ~__GFP_RECLAIM;
-	unsigned int max_order = MIN(zfs_abd_scatter_max_order, MAX_ORDER - 1);
+	unsigned int max_order = MIN(zfs_abd_scatter_max_order,
+	    ABD_MAX_ORDER - 1);
 	unsigned int nr_pages = abd_chunkcnt_for_bytes(size);
 	unsigned int chunks = 0, zones = 0;
 	size_t remaining_size;
@@ -729,7 +738,7 @@ abd_kstats_update(kstat_t *ksp, int rw)
 	    wmsum_value(&abd_sums.abdstat_scatter_data_size);
 	as->abdstat_scatter_chunk_waste.value.ui64 =
 	    wmsum_value(&abd_sums.abdstat_scatter_chunk_waste);
-	for (int i = 0; i < MAX_ORDER; i++) {
+	for (int i = 0; i < ABD_MAX_ORDER; i++) {
 		as->abdstat_scatter_orders[i].value.ui64 =
 		    wmsum_value(&abd_sums.abdstat_scatter_orders[i]);
 	}
@@ -758,7 +767,7 @@ abd_init(void)
 	wmsum_init(&abd_sums.abdstat_scatter_cnt, 0);
 	wmsum_init(&abd_sums.abdstat_scatter_data_size, 0);
 	wmsum_init(&abd_sums.abdstat_scatter_chunk_waste, 0);
-	for (i = 0; i < MAX_ORDER; i++)
+	for (i = 0; i < ABD_MAX_ORDER; i++)
 		wmsum_init(&abd_sums.abdstat_scatter_orders[i], 0);
 	wmsum_init(&abd_sums.abdstat_scatter_page_multi_chunk, 0);
 	wmsum_init(&abd_sums.abdstat_scatter_page_multi_zone, 0);
@@ -768,7 +777,7 @@ abd_init(void)
 	abd_ksp = kstat_create("zfs", 0, "abdstats", "misc", KSTAT_TYPE_NAMED,
 	    sizeof (abd_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
 	if (abd_ksp != NULL) {
-		for (i = 0; i < MAX_ORDER; i++) {
+		for (i = 0; i < ABD_MAX_ORDER; i++) {
 			snprintf(abd_stats.abdstat_scatter_orders[i].name,
 			    KSTAT_STRLEN, "scatter_order_%d", i);
 			abd_stats.abdstat_scatter_orders[i].data_type =
@@ -798,7 +807,7 @@ abd_fini(void)
 	wmsum_fini(&abd_sums.abdstat_scatter_cnt);
 	wmsum_fini(&abd_sums.abdstat_scatter_data_size);
 	wmsum_fini(&abd_sums.abdstat_scatter_chunk_waste);
-	for (int i = 0; i < MAX_ORDER; i++)
+	for (int i = 0; i < ABD_MAX_ORDER; i++)
 		wmsum_fini(&abd_sums.abdstat_scatter_orders[i]);
 	wmsum_fini(&abd_sums.abdstat_scatter_page_multi_chunk);
 	wmsum_fini(&abd_sums.abdstat_scatter_page_multi_zone);

module/os/linux/zfs/vdev_disk.c

@@ -41,8 +41,28 @@
 #include <linux/blk-cgroup.h>
 #endif
 
+/*
+ * Linux 6.8.x uses a bdev_handle as an instance/refcount for an underlying
+ * block_device. Since it carries the block_device inside, its convenient to
+ * just use the handle as a proxy. For pre-6.8, we just emulate this with
+ * a cast, since we don't need any of the other fields inside the handle.
+ */
+#ifdef HAVE_BDEV_OPEN_BY_PATH
+typedef struct bdev_handle zfs_bdev_handle_t;
+#define	BDH_BDEV(bdh)		((bdh)->bdev)
+#define	BDH_IS_ERR(bdh)		(IS_ERR(bdh))
+#define	BDH_PTR_ERR(bdh)	(PTR_ERR(bdh))
+#define	BDH_ERR_PTR(err)	(ERR_PTR(err))
+#else
+typedef void zfs_bdev_handle_t;
+#define	BDH_BDEV(bdh)		((struct block_device *)bdh)
+#define	BDH_IS_ERR(bdh)		(IS_ERR(BDH_BDEV(bdh)))
+#define	BDH_PTR_ERR(bdh)	(PTR_ERR(BDH_BDEV(bdh)))
+#define	BDH_ERR_PTR(err)	(ERR_PTR(err))
+#endif
+
 typedef struct vdev_disk {
-	struct block_device		*vd_bdev;
+	zfs_bdev_handle_t		*vd_bdh;
 	krwlock_t			vd_lock;
 } vdev_disk_t;
 
@@ -209,29 +229,23 @@ static void
 vdev_disk_kobj_evt_post(vdev_t *v)
 {
 	vdev_disk_t *vd = v->vdev_tsd;
-	if (vd && vd->vd_bdev) {
+	if (vd && vd->vd_bdh) {
-		spl_signal_kobj_evt(vd->vd_bdev);
+		spl_signal_kobj_evt(BDH_BDEV(vd->vd_bdh));
 	} else {
 		vdev_dbgmsg(v, "vdev_disk_t is NULL for VDEV:%s\n",
 		    v->vdev_path);
 	}
 }
 
-#if !defined(HAVE_BLKDEV_GET_BY_PATH_4ARG)
+static zfs_bdev_handle_t *
-/*
+vdev_blkdev_get_by_path(const char *path, spa_mode_t mode, void *holder)
- * Define a dummy struct blk_holder_ops for kernel versions
- * prior to 6.5.
- */
-struct blk_holder_ops {};
-#endif
-
-static struct block_device *
-vdev_blkdev_get_by_path(const char *path, spa_mode_t mode, void *holder,
-    const struct blk_holder_ops *hops)
 {
-#ifdef HAVE_BLKDEV_GET_BY_PATH_4ARG
+#if defined(HAVE_BDEV_OPEN_BY_PATH)
+	return (bdev_open_by_path(path,
+	    vdev_bdev_mode(mode, B_TRUE), holder, NULL));
+#elif defined(HAVE_BLKDEV_GET_BY_PATH_4ARG)
 	return (blkdev_get_by_path(path,
-	    vdev_bdev_mode(mode, B_TRUE), holder, hops));
+	    vdev_bdev_mode(mode, B_TRUE), holder, NULL));
 #else
 	return (blkdev_get_by_path(path,
 	    vdev_bdev_mode(mode, B_TRUE), holder));
@@ -239,12 +253,15 @@ vdev_blkdev_get_by_path(const char *path, spa_mode_t mode, void *holder,
 }
 
 static void
-vdev_blkdev_put(struct block_device *bdev, spa_mode_t mode, void *holder)
+vdev_blkdev_put(zfs_bdev_handle_t *bdh, spa_mode_t mode, void *holder)
 {
-#ifdef HAVE_BLKDEV_PUT_HOLDER
+#if defined(HAVE_BDEV_RELEASE)
-	return (blkdev_put(bdev, holder));
+	return (bdev_release(bdh));
+#elif defined(HAVE_BLKDEV_PUT_HOLDER)
+	return (blkdev_put(BDH_BDEV(bdh), holder));
 #else
-	return (blkdev_put(bdev, vdev_bdev_mode(mode, B_TRUE)));
+	return (blkdev_put(BDH_BDEV(bdh),
+	    vdev_bdev_mode(mode, B_TRUE)));
 #endif
 }
 
@@ -252,7 +269,7 @@ static int
 vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
     uint64_t *logical_ashift, uint64_t *physical_ashift)
 {
-	struct block_device *bdev;
+	zfs_bdev_handle_t *bdh;
 #ifdef HAVE_BLK_MODE_T
 	blk_mode_t mode = vdev_bdev_mode(spa_mode(v->vdev_spa), B_FALSE);
 #else
@@ -282,10 +299,11 @@ vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
 		boolean_t reread_part = B_FALSE;
 
 		rw_enter(&vd->vd_lock, RW_WRITER);
-		bdev = vd->vd_bdev;
+		bdh = vd->vd_bdh;
-		vd->vd_bdev = NULL;
+		vd->vd_bdh = NULL;
 
-		if (bdev) {
+		if (bdh) {
+			struct block_device *bdev = BDH_BDEV(bdh);
 			if (v->vdev_expanding && bdev != bdev_whole(bdev)) {
 				vdev_bdevname(bdev_whole(bdev), disk_name + 5);
 				/*
@@ -307,15 +325,16 @@ vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
 					reread_part = B_TRUE;
 			}
 
-			vdev_blkdev_put(bdev, mode, zfs_vdev_holder);
+			vdev_blkdev_put(bdh, mode, zfs_vdev_holder);
 		}
 
 		if (reread_part) {
-			bdev = vdev_blkdev_get_by_path(disk_name, mode,
+			bdh = vdev_blkdev_get_by_path(disk_name, mode,
-			    zfs_vdev_holder, NULL);
+			    zfs_vdev_holder);
-			if (!IS_ERR(bdev)) {
+			if (!BDH_IS_ERR(bdh)) {
-				int error = vdev_bdev_reread_part(bdev);
+				int error =
-				vdev_blkdev_put(bdev, mode, zfs_vdev_holder);
+				    vdev_bdev_reread_part(BDH_BDEV(bdh));
+				vdev_blkdev_put(bdh, mode, zfs_vdev_holder);
 				if (error == 0) {
 					timeout = MSEC2NSEC(
 					    zfs_vdev_open_timeout_ms * 2);
@@ -358,11 +377,11 @@ vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
 	 * subsequent attempts are expected to eventually succeed.
 	 */
 	hrtime_t start = gethrtime();
-	bdev = ERR_PTR(-ENXIO);
+	bdh = BDH_ERR_PTR(-ENXIO);
-	while (IS_ERR(bdev) && ((gethrtime() - start) < timeout)) {
+	while (BDH_IS_ERR(bdh) && ((gethrtime() - start) < timeout)) {
-		bdev = vdev_blkdev_get_by_path(v->vdev_path, mode,
+		bdh = vdev_blkdev_get_by_path(v->vdev_path, mode,
-		    zfs_vdev_holder, NULL);
+		    zfs_vdev_holder);
-		if (unlikely(PTR_ERR(bdev) == -ENOENT)) {
+		if (unlikely(BDH_PTR_ERR(bdh) == -ENOENT)) {
 			/*
 			 * There is no point of waiting since device is removed
 			 * explicitly
@@ -371,52 +390,54 @@ vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
 				break;
 
 			schedule_timeout(MSEC_TO_TICK(10));
-		} else if (unlikely(PTR_ERR(bdev) == -ERESTARTSYS)) {
+		} else if (unlikely(BDH_PTR_ERR(bdh) == -ERESTARTSYS)) {
 			timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms * 10);
 			continue;
-		} else if (IS_ERR(bdev)) {
+		} else if (BDH_IS_ERR(bdh)) {
 			break;
 		}
 	}
 
-	if (IS_ERR(bdev)) {
+	if (BDH_IS_ERR(bdh)) {
-		int error = -PTR_ERR(bdev);
+		int error = -BDH_PTR_ERR(bdh);
 		vdev_dbgmsg(v, "open error=%d timeout=%llu/%llu", error,
 		    (u_longlong_t)(gethrtime() - start),
 		    (u_longlong_t)timeout);
-		vd->vd_bdev = NULL;
+		vd->vd_bdh = NULL;
 		v->vdev_tsd = vd;
 		rw_exit(&vd->vd_lock);
 		return (SET_ERROR(error));
 	} else {
-		vd->vd_bdev = bdev;
+		vd->vd_bdh = bdh;
 		v->vdev_tsd = vd;
 		rw_exit(&vd->vd_lock);
 	}
 
+	struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
+
 	/*  Determine the physical block size */
-	int physical_block_size = bdev_physical_block_size(vd->vd_bdev);
+	int physical_block_size = bdev_physical_block_size(bdev);
 
 	/*  Determine the logical block size */
-	int logical_block_size = bdev_logical_block_size(vd->vd_bdev);
+	int logical_block_size = bdev_logical_block_size(bdev);
 
 	/* Clear the nowritecache bit, causes vdev_reopen() to try again. */
 	v->vdev_nowritecache = B_FALSE;
 
 	/* Set when device reports it supports TRIM. */
-	v->vdev_has_trim = bdev_discard_supported(vd->vd_bdev);
+	v->vdev_has_trim = bdev_discard_supported(bdev);
 
 	/* Set when device reports it supports secure TRIM. */
-	v->vdev_has_securetrim = bdev_secure_discard_supported(vd->vd_bdev);
+	v->vdev_has_securetrim = bdev_secure_discard_supported(bdev);
 
 	/* Inform the ZIO pipeline that we are non-rotational */
-	v->vdev_nonrot = blk_queue_nonrot(bdev_get_queue(vd->vd_bdev));
+	v->vdev_nonrot = blk_queue_nonrot(bdev_get_queue(bdev));
 
 	/* Physical volume size in bytes for the partition */
-	*psize = bdev_capacity(vd->vd_bdev);
+	*psize = bdev_capacity(bdev);
 
 	/* Physical volume size in bytes including possible expansion space */
-	*max_psize = bdev_max_capacity(vd->vd_bdev, v->vdev_wholedisk);
+	*max_psize = bdev_max_capacity(bdev, v->vdev_wholedisk);
 
 	/* Based on the minimum sector size set the block size */
 	*physical_ashift = highbit64(MAX(physical_block_size,
@@ -436,8 +457,8 @@ vdev_disk_close(vdev_t *v)
 	if (v->vdev_reopening || vd == NULL)
 		return;
 
-	if (vd->vd_bdev != NULL) {
+	if (vd->vd_bdh != NULL) {
-		vdev_blkdev_put(vd->vd_bdev, spa_mode(v->vdev_spa),
+		vdev_blkdev_put(vd->vd_bdh, spa_mode(v->vdev_spa),
 		    zfs_vdev_holder);
 	}
 
@@ -849,10 +870,10 @@ vdev_disk_io_trim(zio_t *zio)
 
 #if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
 	if (zio->io_trim_flags & ZIO_TRIM_SECURE) {
-		return (-blkdev_issue_secure_erase(vd->vd_bdev,
+		return (-blkdev_issue_secure_erase(BDH_BDEV(vd->vd_bdh),
 		    zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS));
 	} else {
-		return (-blkdev_issue_discard(vd->vd_bdev,
+		return (-blkdev_issue_discard(BDH_BDEV(vd->vd_bdh),
 		    zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS));
 	}
 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD)
@@ -861,7 +882,7 @@ vdev_disk_io_trim(zio_t *zio)
 	if (zio->io_trim_flags & ZIO_TRIM_SECURE)
 		trim_flags |= BLKDEV_DISCARD_SECURE;
 #endif
-	return (-blkdev_issue_discard(vd->vd_bdev,
+	return (-blkdev_issue_discard(BDH_BDEV(vd->vd_bdh),
 	    zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, trim_flags));
 #else
 #error "Unsupported kernel"
@@ -891,7 +912,7 @@ vdev_disk_io_start(zio_t *zio)
 	 * If the vdev is closed, it's likely due to a failed reopen and is
 	 * in the UNAVAIL state.  Nothing to be done here but return failure.
 	 */
-	if (vd->vd_bdev == NULL) {
+	if (vd->vd_bdh == NULL) {
 		rw_exit(&vd->vd_lock);
 		zio->io_error = ENXIO;
 		zio_interrupt(zio);
@@ -919,7 +940,7 @@ vdev_disk_io_start(zio_t *zio)
 				break;
 			}
 
-			error = vdev_disk_io_flush(vd->vd_bdev, zio);
+			error = vdev_disk_io_flush(BDH_BDEV(vd->vd_bdh), zio);
 			if (error == 0) {
 				rw_exit(&vd->vd_lock);
 				return;
@@ -958,7 +979,7 @@ vdev_disk_io_start(zio_t *zio)
 	}
 
 	zio->io_target_timestamp = zio_handle_io_delay(zio);
-	error = __vdev_disk_physio(vd->vd_bdev, zio,
+	error = __vdev_disk_physio(BDH_BDEV(vd->vd_bdh), zio,
 	    zio->io_size, zio->io_offset, rw, 0);
 	rw_exit(&vd->vd_lock);
 
@@ -981,8 +1002,8 @@ vdev_disk_io_done(zio_t *zio)
 		vdev_t *v = zio->io_vd;
 		vdev_disk_t *vd = v->vdev_tsd;
 
-		if (!zfs_check_disk_status(vd->vd_bdev)) {
+		if (!zfs_check_disk_status(BDH_BDEV(vd->vd_bdh))) {
-			invalidate_bdev(vd->vd_bdev);
+			invalidate_bdev(BDH_BDEV(vd->vd_bdh));
 			v->vdev_remove_wanted = B_TRUE;
 			spa_async_request(zio->io_spa, SPA_ASYNC_REMOVE);
 		}

module/os/linux/zfs/zfs_vnops_os.c

@@ -1885,7 +1885,7 @@ zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zidmap_t *mnt_ns)
 {
 	struct inode	*ip;
 	zfsvfs_t	*zfsvfs = ZTOZSB(zp);
-	objset_t	*os = zfsvfs->z_os;
+	objset_t	*os;
 	zilog_t		*zilog;
 	dmu_tx_t	*tx;
 	vattr_t		oldva;
@@ -1917,6 +1917,7 @@ zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zidmap_t *mnt_ns)
 	if ((err = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
 		return (err);
 	ip = ZTOI(zp);
+	os = zfsvfs->z_os;
 
 	/*
 	 * If this is a xvattr_t, then get a pointer to the structure of

module/zfs/metaslab.c

@@ -5061,7 +5061,6 @@ metaslab_group_alloc(metaslab_group_t *mg, zio_alloc_list_t *zal,
     int allocator, boolean_t try_hard)
 {
 	uint64_t offset;
-	ASSERT(mg->mg_initialized);
 
 	offset = metaslab_group_alloc_normal(mg, zal, asize, txg, want_unique,
 	    dva, d, allocator, try_hard);
@@ -5212,8 +5211,6 @@ top:
 			goto next;
 		}
 
-		ASSERT(mg->mg_initialized);
-
 		/*
 		 * Avoid writing single-copy data to an unhealthy,
 		 * non-redundant vdev, unless we've already tried all

module/zfs/vdev.c

@@ -2484,23 +2484,37 @@ vdev_validate(vdev_t *vd)
 	return (0);
 }
 
+static void
+vdev_update_path(const char *prefix, char *svd, char **dvd, uint64_t guid)
+{
+	if (svd != NULL && *dvd != NULL) {
+		if (strcmp(svd, *dvd) != 0) {
+			zfs_dbgmsg("vdev_copy_path: vdev %llu: %s changed "
+			    "from '%s' to '%s'", (u_longlong_t)guid, prefix,
+			    *dvd, svd);
+			spa_strfree(*dvd);
+			*dvd = spa_strdup(svd);
+		}
+	} else if (svd != NULL) {
+		*dvd = spa_strdup(svd);
+		zfs_dbgmsg("vdev_copy_path: vdev %llu: path set to '%s'",
+		    (u_longlong_t)guid, *dvd);
+	}
+}
+
 static void
 vdev_copy_path_impl(vdev_t *svd, vdev_t *dvd)
 {
 	char *old, *new;
-	if (svd->vdev_path != NULL && dvd->vdev_path != NULL) {
+
-		if (strcmp(svd->vdev_path, dvd->vdev_path) != 0) {
+	vdev_update_path("vdev_path", svd->vdev_path, &dvd->vdev_path,
-			zfs_dbgmsg("vdev_copy_path: vdev %llu: path changed "
+	    dvd->vdev_guid);
-			    "from '%s' to '%s'", (u_longlong_t)dvd->vdev_guid,
+
-			    dvd->vdev_path, svd->vdev_path);
+	vdev_update_path("vdev_devid", svd->vdev_devid, &dvd->vdev_devid,
-			spa_strfree(dvd->vdev_path);
+	    dvd->vdev_guid);
-			dvd->vdev_path = spa_strdup(svd->vdev_path);
+
-		}
+	vdev_update_path("vdev_physpath", svd->vdev_physpath,
-	} else if (svd->vdev_path != NULL) {
+	    &dvd->vdev_physpath, dvd->vdev_guid);
-		dvd->vdev_path = spa_strdup(svd->vdev_path);
-		zfs_dbgmsg("vdev_copy_path: vdev %llu: path set to '%s'",
-		    (u_longlong_t)dvd->vdev_guid, dvd->vdev_path);
-	}
 
 	/*
 	 * Our enclosure sysfs path may have changed between imports

module/zfs/vdev_label.c

@@ -1023,6 +1023,10 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason)
 	int error;
 	uint64_t spare_guid = 0, l2cache_guid = 0;
 	int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL;
+	boolean_t reason_spare = (reason == VDEV_LABEL_SPARE || (reason ==
+	    VDEV_LABEL_REMOVE && vd->vdev_isspare));
+	boolean_t reason_l2cache = (reason == VDEV_LABEL_L2CACHE || (reason ==
+	    VDEV_LABEL_REMOVE && vd->vdev_isl2cache));
 
 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
 
@@ -1108,34 +1112,20 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason)
 	 * really part of an active pool just yet.  The labels will
 	 * be written again with a meaningful txg by spa_sync().
 	 */
-	if (reason == VDEV_LABEL_SPARE ||
+	if (reason_spare || reason_l2cache) {
-	    (reason == VDEV_LABEL_REMOVE && vd->vdev_isspare)) {
 		/*
-		 * For inactive hot spares, we generate a special label that
+		 * For inactive hot spares and level 2 ARC devices, we generate
-		 * identifies as a mutually shared hot spare.  We write the
+		 * a special label that identifies as a mutually shared hot
-		 * label if we are adding a hot spare, or if we are removing an
+		 * spare or l2cache device. We write the label in case of
-		 * active hot spare (in which case we want to revert the
+		 * addition or removal of hot spare or l2cache vdev (in which
-		 * labels).
+		 * case we want to revert the labels).
 		 */
 		VERIFY(nvlist_alloc(&label, NV_UNIQUE_NAME, KM_SLEEP) == 0);
 
 		VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_VERSION,
 		    spa_version(spa)) == 0);
 		VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_POOL_STATE,
-		    POOL_STATE_SPARE) == 0);
+		    reason_spare ? POOL_STATE_SPARE : POOL_STATE_L2CACHE) == 0);
-		VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_GUID,
-		    vd->vdev_guid) == 0);
-	} else if (reason == VDEV_LABEL_L2CACHE ||
-	    (reason == VDEV_LABEL_REMOVE && vd->vdev_isl2cache)) {
-		/*
-		 * For level 2 ARC devices, add a special label.
-		 */
-		VERIFY(nvlist_alloc(&label, NV_UNIQUE_NAME, KM_SLEEP) == 0);
-
-		VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_VERSION,
-		    spa_version(spa)) == 0);
-		VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_POOL_STATE,
-		    POOL_STATE_L2CACHE) == 0);
 		VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_GUID,
 		    vd->vdev_guid) == 0);
 
@@ -1146,8 +1136,34 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason)
 		 * spa->spa_l2cache->sav_config (populated in
 		 * spa_ld_open_aux_vdevs()).
 		 */
-		VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_ASHIFT,
+		if (reason_l2cache) {
-		    vd->vdev_ashift) == 0);
+			VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_ASHIFT,
+			    vd->vdev_ashift) == 0);
+		}
+
+		/*
+		 * Add path information to help find it during pool import
+		 */
+		if (vd->vdev_path != NULL) {
+			VERIFY(nvlist_add_string(label, ZPOOL_CONFIG_PATH,
+			    vd->vdev_path) == 0);
+		}
+		if (vd->vdev_devid != NULL) {
+			VERIFY(nvlist_add_string(label, ZPOOL_CONFIG_DEVID,
+			    vd->vdev_devid) == 0);
+		}
+		if (vd->vdev_physpath != NULL) {
+			VERIFY(nvlist_add_string(label, ZPOOL_CONFIG_PHYS_PATH,
+			    vd->vdev_physpath) == 0);
+		}
+
+		/*
+		 * When spare or l2cache (aux) vdev is added during pool
+		 * creation, spa->spa_uberblock is not written until this
+		 * point. Write it on next config sync.
+		 */
+		if (uberblock_verify(&spa->spa_uberblock))
+			spa->spa_aux_sync_uber = B_TRUE;
 	} else {
 		uint64_t txg = 0ULL;
 
@@ -1749,6 +1765,16 @@ vdev_uberblock_sync_list(vdev_t **svd, int svdcount, uberblock_t *ub, int flags)
 	for (int v = 0; v < svdcount; v++)
 		vdev_uberblock_sync(zio, &good_writes, ub, svd[v], flags);
 
+	if (spa->spa_aux_sync_uber) {
+		for (int v = 0; v < spa->spa_spares.sav_count; v++) {
+			vdev_uberblock_sync(zio, &good_writes, ub,
+			    spa->spa_spares.sav_vdevs[v], flags);
+		}
+		for (int v = 0; v < spa->spa_l2cache.sav_count; v++) {
+			vdev_uberblock_sync(zio, &good_writes, ub,
+			    spa->spa_l2cache.sav_vdevs[v], flags);
+		}
+	}
 	(void) zio_wait(zio);
 
 	/*
@@ -1763,6 +1789,19 @@ vdev_uberblock_sync_list(vdev_t **svd, int svdcount, uberblock_t *ub, int flags)
 			zio_flush(zio, svd[v]);
 		}
 	}
+	if (spa->spa_aux_sync_uber) {
+		spa->spa_aux_sync_uber = B_FALSE;
+		for (int v = 0; v < spa->spa_spares.sav_count; v++) {
+			if (vdev_writeable(spa->spa_spares.sav_vdevs[v])) {
+				zio_flush(zio, spa->spa_spares.sav_vdevs[v]);
+			}
+		}
+		for (int v = 0; v < spa->spa_l2cache.sav_count; v++) {
+			if (vdev_writeable(spa->spa_l2cache.sav_vdevs[v])) {
+				zio_flush(zio, spa->spa_l2cache.sav_vdevs[v]);
+			}
+		}
+	}
 
 	(void) zio_wait(zio);
 

module/zfs/vdev_trim.c

@@ -194,7 +194,8 @@ vdev_autotrim_wait_kick(vdev_t *vd, int num_of_kick)
 	for (int i = 0; i < num_of_kick; i++) {
 		if (vd->vdev_autotrim_exit_wanted)
 			break;
-		cv_wait(&vd->vdev_autotrim_kick_cv, &vd->vdev_autotrim_lock);
+		cv_wait_idle(&vd->vdev_autotrim_kick_cv,
+		    &vd->vdev_autotrim_lock);
 	}
 	boolean_t exit_wanted = vd->vdev_autotrim_exit_wanted;
 	mutex_exit(&vd->vdev_autotrim_lock);

module/zfs/zfs_vnops.c

@@ -801,11 +801,11 @@ zfs_setsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr)
 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
 	int error;
 	boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
-	zilog_t	*zilog = zfsvfs->z_log;
+	zilog_t	*zilog;
 
 	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
 		return (error);
-
+	zilog = zfsvfs->z_log;
 	error = zfs_setacl(zp, vsecp, skipaclchk, cr);
 
 	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)

rpm/generic/zfs-kmod.spec.in

@@ -150,6 +150,30 @@ for kernel_version in %{?kernel_versions}; do
 done
 
 
+# Module signing (modsign)
+#
+# This must be run _after_ find-debuginfo.sh runs, otherwise that will strip
+# the signature off of the modules.
+# (Based on Fedora's kernel.spec workaround)
+%define __modsign_install_post \
+    sign_pem="%{ksrc}/certs/signing_key.pem"; \
+    sign_x509="%{ksrc}/certs/signing_key.x509"; \
+    if [ -f "${sign_x509}" ]\
+    then \
+        echo "Signing kernel modules ..."; \
+        for kmod in $(find ${RPM_BUILD_ROOT}%{kmodinstdir_prefix}/*/extra/ -name \*.ko); do \
+            %{ksrc}/scripts/sign-file sha256 ${sign_pem} ${sign_x509} ${kmod}; \
+        done \
+    fi \
+%{nil}
+
+# hack to ensure signing happens after find-debuginfo.sh runs
+%define __spec_install_post \
+    %{?__debug_package:%{__debug_install_post}}\
+    %{__arch_install_post}\
+    %{__os_install_post}\
+    %{__modsign_install_post}
+
 %install
 rm -rf ${RPM_BUILD_ROOT}
 

rpm/redhat/zfs-kmod.spec.in

@@ -72,6 +72,30 @@ fi
         %{?kernel_llvm}
 make %{?_smp_mflags}
 
+# Module signing (modsign)
+#
+# This must be run _after_ find-debuginfo.sh runs, otherwise that will strip
+# the signature off of the modules.
+# (Based on Fedora's kernel.spec workaround)
+%define __modsign_install_post \
+        sign_pem="%{ksrc}/certs/signing_key.pem"; \
+        sign_x509="%{ksrc}/certs/signing_key.x509"; \
+        if [ -f "${sign_x509}" ]\
+        then \
+            echo "Signing kernel modules ..."; \
+            for kmod in $(find %{buildroot}/lib/modules/%{kverrel}/extra/ -name \*.ko); do \
+                    %{ksrc}/scripts/sign-file sha256 ${sign_pem} ${sign_x509} ${kmod}; \
+            done \
+        fi \
+%{nil}
+
+# hack to ensure signing happens after find-debuginfo.sh runs
+%define __spec_install_post \
+        %{?__debug_package:%{__debug_install_post}}\
+        %{__arch_install_post}\
+        %{__os_install_post}\
+        %{__modsign_install_post}
+
 %install
 make install \
         DESTDIR=${RPM_BUILD_ROOT} \

tests/Makefile.am

@@ -16,6 +16,7 @@ dist_scripts_test_runner_include_DATA = \
 
 scripts_runfilesdir = $(datadir)/$(PACKAGE)/runfiles
 dist_scripts_runfiles_DATA = \
+	%D%/runfiles/bclone.run \
 	%D%/runfiles/common.run \
 	%D%/runfiles/freebsd.run \
 	%D%/runfiles/linux.run \