Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity

vdev_disk: ensure trim errors are returned immediately 

After 08fd5ccc3, the discard issuing code was organised such that if
requesting an async discard or secure erase failed before the IO was
issued (that is, calling __blkdev_issue_discard() returned an error),
the failed zio would never be executed, resulting in txg_sync hanging
forever waiting for IO to finish.

This commit fixes that by immediately executing a failed zio on error.
To handle the successful synchronous op case, we fake an async op by,
when not using an asynchronous submission method, queuing the successful
result zio as part of the discard handler.

Since it was hard to understand the differences between discard and
secure erase, and sync and async, across different kernel versions, I've
commented and reorganised the code a bit to try and make everything more
contained and linear.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Signed-off-by: Rob Norris <rob.norris@klarasystems.com>
(cherry picked from commit ba9f587a77)
This commit is contained in:
Rob N 2024-04-09 04:50:24 +10:00 committed by Brian Behlendorf
parent 28520cad25
commit d0d9dccc61
2 changed files with 172 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
config/kernel-blkdev.m4
View File

@ -523,12 +523,29 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BDEVNAME], [
]) ])
dnl # dnl #
dnl # 5.19 API: blkdev_issue_secure_erase() dnl # TRIM support: discard and secure erase. We make use of asynchronous
dnl # 4.7 API: __blkdev_issue_discard(..., BLKDEV_DISCARD_SECURE) dnl # functions when available.
dnl # 3.10 API: blkdev_issue_discard(..., BLKDEV_DISCARD_SECURE)
dnl # dnl #
AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_SECURE_ERASE], [ dnl # 3.10:
ZFS_LINUX_TEST_SRC([blkdev_issue_secure_erase], [ dnl # sync discard: blkdev_issue_discard(..., 0)
dnl # sync erase: blkdev_issue_discard(..., BLKDEV_DISCARD_SECURE)
dnl # async discard: [not available]
dnl # async erase: [not available]
dnl #
dnl # 4.7:
dnl # sync discard: blkdev_issue_discard(..., 0)
dnl # sync erase: blkdev_issue_discard(..., BLKDEV_DISCARD_SECURE)
dnl # async discard: __blkdev_issue_discard(..., 0)
dnl # async erase: __blkdev_issue_discard(..., BLKDEV_DISCARD_SECURE)
dnl #
dnl # 5.19:
dnl # sync discard: blkdev_issue_discard(...)
dnl # sync erase: blkdev_issue_secure_erase(...)
dnl # async discard: __blkdev_issue_discard(...)
dnl # async erase: [not available]
dnl #
AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_DISCARD], [
ZFS_LINUX_TEST_SRC([blkdev_issue_discard_noflags], [
#include <linux/blkdev.h> #include <linux/blkdev.h>
],[ ],[
struct block_device *bdev = NULL; struct block_device *bdev = NULL;
@ -536,10 +553,33 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_SECURE_ERASE], [
sector_t nr_sects = 0; sector_t nr_sects = 0;
int error __attribute__ ((unused)); int error __attribute__ ((unused));
error = blkdev_issue_secure_erase(bdev, error = blkdev_issue_discard(bdev,
sector, nr_sects, GFP_KERNEL); sector, nr_sects, GFP_KERNEL);
]) ])
ZFS_LINUX_TEST_SRC([blkdev_issue_discard_flags], [
#include <linux/blkdev.h>
],[
struct block_device *bdev = NULL;
sector_t sector = 0;
sector_t nr_sects = 0;
unsigned long flags = 0;
int error __attribute__ ((unused));
error = blkdev_issue_discard(bdev,
sector, nr_sects, GFP_KERNEL, flags);
])
ZFS_LINUX_TEST_SRC([blkdev_issue_discard_async_noflags], [
#include <linux/blkdev.h>
],[
struct block_device *bdev = NULL;
sector_t sector = 0;
sector_t nr_sects = 0;
struct bio *biop = NULL;
int error __attribute__ ((unused));
error = __blkdev_issue_discard(bdev,
sector, nr_sects, GFP_KERNEL, &biop);
])
ZFS_LINUX_TEST_SRC([blkdev_issue_discard_async_flags], [ ZFS_LINUX_TEST_SRC([blkdev_issue_discard_async_flags], [
#include <linux/blkdev.h> #include <linux/blkdev.h>
],[ ],[
@ -553,22 +593,52 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_SECURE_ERASE], [
error = __blkdev_issue_discard(bdev, error = __blkdev_issue_discard(bdev,
sector, nr_sects, GFP_KERNEL, flags, &biop); sector, nr_sects, GFP_KERNEL, flags, &biop);
]) ])
ZFS_LINUX_TEST_SRC([blkdev_issue_secure_erase], [
ZFS_LINUX_TEST_SRC([blkdev_issue_discard_flags], [
#include <linux/blkdev.h> #include <linux/blkdev.h>
],[ ],[
struct block_device *bdev = NULL; struct block_device *bdev = NULL;
sector_t sector = 0; sector_t sector = 0;
sector_t nr_sects = 0; sector_t nr_sects = 0;
unsigned long flags = 0;
int error __attribute__ ((unused)); int error __attribute__ ((unused));
error = blkdev_issue_discard(bdev, error = blkdev_issue_secure_erase(bdev,
sector, nr_sects, GFP_KERNEL, flags); sector, nr_sects, GFP_KERNEL);
]) ])
]) ])
AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_ISSUE_SECURE_ERASE], [ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_ISSUE_DISCARD], [
AC_MSG_CHECKING([whether blkdev_issue_discard() is available])
ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_noflags], [
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD_NOFLAGS, 1,
[blkdev_issue_discard() is available])
],[
AC_MSG_RESULT(no)
])
AC_MSG_CHECKING([whether blkdev_issue_discard(flags) is available])
ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_flags], [
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS, 1,
[blkdev_issue_discard(flags) is available])
],[
AC_MSG_RESULT(no)
])
AC_MSG_CHECKING([whether __blkdev_issue_discard() is available])
ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_async_noflags], [
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_NOFLAGS, 1,
[__blkdev_issue_discard() is available])
],[
AC_MSG_RESULT(no)
])
AC_MSG_CHECKING([whether __blkdev_issue_discard(flags) is available])
ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_async_flags], [
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS, 1,
[__blkdev_issue_discard(flags) is available])
],[
AC_MSG_RESULT(no)
])
AC_MSG_CHECKING([whether blkdev_issue_secure_erase() is available]) AC_MSG_CHECKING([whether blkdev_issue_secure_erase() is available])
ZFS_LINUX_TEST_RESULT([blkdev_issue_secure_erase], [ ZFS_LINUX_TEST_RESULT([blkdev_issue_secure_erase], [
AC_MSG_RESULT(yes) AC_MSG_RESULT(yes)
@ -576,24 +646,6 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_ISSUE_SECURE_ERASE], [
[blkdev_issue_secure_erase() is available]) [blkdev_issue_secure_erase() is available])
],[ ],[
AC_MSG_RESULT(no) AC_MSG_RESULT(no)
AC_MSG_CHECKING([whether __blkdev_issue_discard() is available])
ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_async_flags], [
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC, 1,
[__blkdev_issue_discard() is available])
],[
AC_MSG_RESULT(no)
AC_MSG_CHECKING([whether blkdev_issue_discard() is available])
ZFS_LINUX_TEST_RESULT([blkdev_issue_discard_flags], [
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_BLKDEV_ISSUE_DISCARD, 1,
[blkdev_issue_discard() is available])
],[
ZFS_LINUX_TEST_ERROR([blkdev_issue_discard()])
])
])
]) ])
]) ])
@ -657,7 +709,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV], [
ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_CHECK_MEDIA_CHANGE ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_CHECK_MEDIA_CHANGE
ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_WHOLE ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_WHOLE
ZFS_AC_KERNEL_SRC_BLKDEV_BDEVNAME ZFS_AC_KERNEL_SRC_BLKDEV_BDEVNAME
ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_SECURE_ERASE ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_DISCARD
ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_KOBJ ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_KOBJ
ZFS_AC_KERNEL_SRC_BLKDEV_PART_TO_DEV ZFS_AC_KERNEL_SRC_BLKDEV_PART_TO_DEV
ZFS_AC_KERNEL_SRC_BLKDEV_DISK_CHECK_MEDIA_CHANGE ZFS_AC_KERNEL_SRC_BLKDEV_DISK_CHECK_MEDIA_CHANGE
@ -678,7 +730,7 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV], [
ZFS_AC_KERNEL_BLKDEV_BDEV_WHOLE ZFS_AC_KERNEL_BLKDEV_BDEV_WHOLE
ZFS_AC_KERNEL_BLKDEV_BDEVNAME ZFS_AC_KERNEL_BLKDEV_BDEVNAME
ZFS_AC_KERNEL_BLKDEV_GET_ERESTARTSYS ZFS_AC_KERNEL_BLKDEV_GET_ERESTARTSYS
ZFS_AC_KERNEL_BLKDEV_ISSUE_SECURE_ERASE ZFS_AC_KERNEL_BLKDEV_ISSUE_DISCARD
ZFS_AC_KERNEL_BLKDEV_BDEV_KOBJ ZFS_AC_KERNEL_BLKDEV_BDEV_KOBJ
ZFS_AC_KERNEL_BLKDEV_PART_TO_DEV ZFS_AC_KERNEL_BLKDEV_PART_TO_DEV
ZFS_AC_KERNEL_BLKDEV_DISK_CHECK_MEDIA_CHANGE ZFS_AC_KERNEL_BLKDEV_DISK_CHECK_MEDIA_CHANGE

135
module/os/linux/zfs/vdev_disk.c
View File

@ -1243,8 +1243,6 @@ vdev_disk_io_flush(struct block_device *bdev, zio_t *zio)
return (0); return (0);
} }
#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE) || \
defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC)
BIO_END_IO_PROTO(vdev_disk_discard_end_io, bio, error) BIO_END_IO_PROTO(vdev_disk_discard_end_io, bio, error)
{ {
zio_t *zio = bio->bi_private; zio_t *zio = bio->bi_private;
@ -1259,54 +1257,99 @@ BIO_END_IO_PROTO(vdev_disk_discard_end_io, bio, error)
zio_interrupt(zio); zio_interrupt(zio);
} }
/*
* Wrappers for the different secure erase and discard APIs. We use async
* when available; in this case, *biop is set to the last bio in the chain.
*/
static int static int
vdev_issue_discard_trim(zio_t *zio, unsigned long flags) vdev_bdev_issue_secure_erase(zfs_bdev_handle_t *bdh, sector_t sector,
sector_t nsect, struct bio **biop)
{ {
int ret; *biop = NULL;
struct bio *bio = NULL; int error;
#if defined(BLKDEV_DISCARD_SECURE) #if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
ret = - __blkdev_issue_discard( error = blkdev_issue_secure_erase(BDH_BDEV(bdh),
BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh), sector, nsect, GFP_NOFS);
zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, flags, &bio); #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
error = __blkdev_issue_discard(BDH_BDEV(bdh),
sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE, biop);
#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
error = blkdev_issue_discard(BDH_BDEV(bdh),
sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE);
#else #else
(void) flags; #error "unsupported kernel"
ret = - __blkdev_issue_discard(
BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, &bio);
#endif #endif
if (!ret && bio) {
return (error);
}
static int
vdev_bdev_issue_discard(zfs_bdev_handle_t *bdh, sector_t sector,
sector_t nsect, struct bio **biop)
{
*biop = NULL;
int error;
#if defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
error = __blkdev_issue_discard(BDH_BDEV(bdh),
sector, nsect, GFP_NOFS, 0, biop);
#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_NOFLAGS)
error = __blkdev_issue_discard(BDH_BDEV(bdh),
sector, nsect, GFP_NOFS, biop);
#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
error = blkdev_issue_discard(BDH_BDEV(bdh),
sector, nsect, GFP_NOFS, 0);
#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_NOFLAGS)
error = blkdev_issue_discard(BDH_BDEV(bdh),
sector, nsect, GFP_NOFS);
#else
#error "unsupported kernel"
#endif
return (error);
}
/*
* Entry point for TRIM ops. This calls the right wrapper for secure erase or
* discard, and then does the appropriate finishing work for error vs success
* and async vs sync.
*/
static int
vdev_disk_io_trim(zio_t *zio)
{
int error;
struct bio *bio;
zfs_bdev_handle_t *bdh = ((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh;
sector_t sector = zio->io_offset >> 9;
sector_t nsects = zio->io_size >> 9;
if (zio->io_trim_flags & ZIO_TRIM_SECURE)
error = vdev_bdev_issue_secure_erase(bdh, sector, nsects, &bio);
else
error = vdev_bdev_issue_discard(bdh, sector, nsects, &bio);
if (error != 0)
return (SET_ERROR(-error));
if (bio == NULL) {
/*
* This was a synchronous op that completed successfully, so
* return it to ZFS immediately.
*/
zio_interrupt(zio);
} else {
/*
* This was an asynchronous op; set up completion callback and
* issue it.
*/
bio->bi_private = zio; bio->bi_private = zio;
bio->bi_end_io = vdev_disk_discard_end_io; bio->bi_end_io = vdev_disk_discard_end_io;
vdev_submit_bio(bio); vdev_submit_bio(bio);
} }
return (ret);
}
#endif
static int return (0);
vdev_disk_io_trim(zio_t *zio)
{
unsigned long trim_flags = 0;
if (zio->io_trim_flags & ZIO_TRIM_SECURE) {
#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
return (-blkdev_issue_secure_erase(
BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS));
#elif defined(BLKDEV_DISCARD_SECURE)
trim_flags |= BLKDEV_DISCARD_SECURE;
#endif
}
#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE) || \
defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC)
return (vdev_issue_discard_trim(zio, trim_flags));
#elif defined(HAVE_BLKDEV_ISSUE_DISCARD)
return (-blkdev_issue_discard(
BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, trim_flags));
#else
#error "Unsupported kernel"
#endif
} }
int (*vdev_disk_io_rw_fn)(zio_t *zio) = NULL; int (*vdev_disk_io_rw_fn)(zio_t *zio) = NULL;
@ -1381,14 +1424,12 @@ vdev_disk_io_start(zio_t *zio)
return; return;
case ZIO_TYPE_TRIM: case ZIO_TYPE_TRIM:
zio->io_error = vdev_disk_io_trim(zio); error = vdev_disk_io_trim(zio);
rw_exit(&vd->vd_lock); rw_exit(&vd->vd_lock);
#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE) if (error) {
if (zio->io_trim_flags & ZIO_TRIM_SECURE) zio->io_error = error;
zio_interrupt(zio); zio_execute(zio);
#elif defined(HAVE_BLKDEV_ISSUE_DISCARD) }
zio_interrupt(zio);
#endif
return; return;
case ZIO_TYPE_READ: case ZIO_TYPE_READ: