diff --git a/config/kernel-generic_io_acct.m4 b/config/kernel-generic_io_acct.m4 index 423b3e5a35..e4ab503d5e 100644 --- a/config/kernel-generic_io_acct.m4 +++ b/config/kernel-generic_io_acct.m4 @@ -2,6 +2,16 @@ dnl # dnl # Check for generic io accounting interface. dnl # AC_DEFUN([ZFS_AC_KERNEL_SRC_GENERIC_IO_ACCT], [ + ZFS_LINUX_TEST_SRC([bio_io_acct], [ + #include + ], [ + struct bio *bio = NULL; + unsigned long start_time; + + start_time = bio_start_io_acct(bio); + bio_end_io_acct(bio, start_time); + ]) + ZFS_LINUX_TEST_SRC([generic_acct_3args], [ #include @@ -29,36 +39,49 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_GENERIC_IO_ACCT], [ AC_DEFUN([ZFS_AC_KERNEL_GENERIC_IO_ACCT], [ dnl # - dnl # 3.19 API addition + dnl # 5.7 API, dnl # - dnl # torvalds/linux@394ffa50 allows us to increment iostat - dnl # counters without generic_make_request(). + dnl # Added bio_start_io_acct() and bio_end_io_acct() helpers. dnl # - AC_MSG_CHECKING([whether generic IO accounting wants 3 args]) - ZFS_LINUX_TEST_RESULT_SYMBOL([generic_acct_3args], - [generic_start_io_acct], [block/bio.c], [ + AC_MSG_CHECKING([whether generic bio_*_io_acct() are available]) + ZFS_LINUX_TEST_RESULT([bio_io_acct], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_GENERIC_IO_ACCT_3ARG, 1, - [generic_start_io_acct()/generic_end_io_acct() available]) + AC_DEFINE(HAVE_BIO_IO_ACCT, 1, [bio_*_io_acct() available]) ], [ AC_MSG_RESULT(no) dnl # - dnl # Linux 4.14 API, + dnl # 4.14 API, dnl # dnl # generic_start_io_acct/generic_end_io_acct now require dnl # request_queue to be provided. No functional changes, dnl # but preparation for inflight accounting. dnl # - AC_MSG_CHECKING([whether generic IO accounting wants 4 args]) + AC_MSG_CHECKING([whether generic_*_io_acct wants 4 args]) ZFS_LINUX_TEST_RESULT_SYMBOL([generic_acct_4args], [generic_start_io_acct], [block/bio.c], [ AC_MSG_RESULT(yes) AC_DEFINE(HAVE_GENERIC_IO_ACCT_4ARG, 1, - [generic_start_io_acct()/generic_end_io_acct() ] - [4 arg available]) + [generic_*_io_acct() 4 arg available]) ], [ AC_MSG_RESULT(no) + + dnl # + dnl # 3.19 API addition + dnl # + dnl # torvalds/linux@394ffa50 allows us to increment + dnl # iostat counters without generic_make_request(). + dnl # + AC_MSG_CHECKING( + [whether generic_*_io_acct wants 3 args]) + ZFS_LINUX_TEST_RESULT_SYMBOL([generic_acct_3args], + [generic_start_io_acct], [block/bio.c], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_GENERIC_IO_ACCT_3ARG, 1, + [generic_*_io_acct() 3 arg available]) + ], [ + AC_MSG_RESULT(no) + ]) ]) ]) ]) diff --git a/include/os/linux/kernel/linux/blkdev_compat.h b/include/os/linux/kernel/linux/blkdev_compat.h index 4d84900bec..e1270cb25d 100644 --- a/include/os/linux/kernel/linux/blkdev_compat.h +++ b/include/os/linux/kernel/linux/blkdev_compat.h @@ -523,25 +523,38 @@ blk_queue_discard_secure(struct request_queue *q) */ #define VDEV_HOLDER ((void *)0x2401de7) -static inline void -blk_generic_start_io_acct(struct request_queue *q, int rw, - unsigned long sectors, struct hd_struct *part) +static inline unsigned long +blk_generic_start_io_acct(struct request_queue *q __attribute__((unused)), + struct gendisk *disk __attribute__((unused)), + int rw __attribute__((unused)), struct bio *bio) { -#if defined(HAVE_GENERIC_IO_ACCT_3ARG) - generic_start_io_acct(rw, sectors, part); +#if defined(HAVE_BIO_IO_ACCT) + return (bio_start_io_acct(bio)); +#elif defined(HAVE_GENERIC_IO_ACCT_3ARG) + unsigned long start_time = jiffies; + generic_start_io_acct(rw, bio_sectors(bio), &disk->part0); + return (start_time); #elif defined(HAVE_GENERIC_IO_ACCT_4ARG) - generic_start_io_acct(q, rw, sectors, part); + unsigned long start_time = jiffies; + generic_start_io_acct(q, rw, bio_sectors(bio), &disk->part0); + return (start_time); +#else + /* Unsupported */ + return (0); #endif } static inline void -blk_generic_end_io_acct(struct request_queue *q, int rw, - struct hd_struct *part, unsigned long start_time) +blk_generic_end_io_acct(struct request_queue *q __attribute__((unused)), + struct gendisk *disk __attribute__((unused)), + int rw __attribute__((unused)), struct bio *bio, unsigned long start_time) { -#if defined(HAVE_GENERIC_IO_ACCT_3ARG) - generic_end_io_acct(rw, part, start_time); +#if defined(HAVE_BIO_IO_ACCT) + bio_end_io_acct(bio, start_time); +#elif defined(HAVE_GENERIC_IO_ACCT_3ARG) + generic_end_io_acct(rw, &disk->part0, start_time); #elif defined(HAVE_GENERIC_IO_ACCT_4ARG) - generic_end_io_acct(q, rw, part, start_time); + generic_end_io_acct(q, rw, &disk->part0, start_time); #endif } diff --git a/module/os/linux/zfs/zvol_os.c b/module/os/linux/zfs/zvol_os.c index 29ad673684..9a9a721ceb 100644 --- a/module/os/linux/zfs/zvol_os.c +++ b/module/os/linux/zfs/zvol_os.c @@ -106,10 +106,14 @@ zvol_write(void *arg) return; } + struct request_queue *q = zv->zv_zso->zvo_queue; + struct gendisk *disk = zv->zv_zso->zvo_disk; ssize_t start_resid = uio.uio_resid; - unsigned long start_jif = jiffies; - blk_generic_start_io_acct(zv->zv_zso->zvo_queue, WRITE, - bio_sectors(bio), &zv->zv_zso->zvo_disk->part0); + unsigned long start_time; + + boolean_t acct = blk_queue_io_stat(q); + if (acct) + start_time = blk_generic_start_io_acct(q, disk, WRITE, bio); boolean_t sync = bio_is_fua(bio) || zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS; @@ -153,8 +157,10 @@ zvol_write(void *arg) zil_commit(zv->zv_zilog, ZVOL_OBJ); rw_exit(&zv->zv_suspend_lock); - blk_generic_end_io_acct(zv->zv_zso->zvo_queue, - WRITE, &zv->zv_zso->zvo_disk->part0, start_jif); + + if (acct) + blk_generic_end_io_acct(q, disk, WRITE, bio, start_time); + BIO_END_IO(bio, -error); kmem_free(zvr, sizeof (zv_request_t)); } @@ -171,15 +177,18 @@ zvol_discard(void *arg) boolean_t sync; int error = 0; dmu_tx_t *tx; - unsigned long start_jif; ASSERT3P(zv, !=, NULL); ASSERT3U(zv->zv_open_count, >, 0); ASSERT3P(zv->zv_zilog, !=, NULL); - start_jif = jiffies; - blk_generic_start_io_acct(zv->zv_zso->zvo_queue, WRITE, - bio_sectors(bio), &zv->zv_zso->zvo_disk->part0); + struct request_queue *q = zv->zv_zso->zvo_queue; + struct gendisk *disk = zv->zv_zso->zvo_disk; + unsigned long start_time; + + boolean_t acct = blk_queue_io_stat(q); + if (acct) + start_time = blk_generic_start_io_acct(q, disk, WRITE, bio); sync = bio_is_fua(bio) || zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS; @@ -224,8 +233,10 @@ zvol_discard(void *arg) unlock: rw_exit(&zv->zv_suspend_lock); - blk_generic_end_io_acct(zv->zv_zso->zvo_queue, WRITE, - &zv->zv_zso->zvo_disk->part0, start_jif); + + if (acct) + blk_generic_end_io_acct(q, disk, WRITE, bio, start_time); + BIO_END_IO(bio, -error); kmem_free(zvr, sizeof (zv_request_t)); } @@ -244,10 +255,14 @@ zvol_read(void *arg) ASSERT3P(zv, !=, NULL); ASSERT3U(zv->zv_open_count, >, 0); + struct request_queue *q = zv->zv_zso->zvo_queue; + struct gendisk *disk = zv->zv_zso->zvo_disk; ssize_t start_resid = uio.uio_resid; - unsigned long start_jif = jiffies; - blk_generic_start_io_acct(zv->zv_zso->zvo_queue, READ, bio_sectors(bio), - &zv->zv_zso->zvo_disk->part0); + unsigned long start_time; + + boolean_t acct = blk_queue_io_stat(q); + if (acct) + start_time = blk_generic_start_io_acct(q, disk, READ, bio); zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock, uio.uio_loffset, uio.uio_resid, RL_READER); @@ -275,8 +290,10 @@ zvol_read(void *arg) task_io_account_read(nread); rw_exit(&zv->zv_suspend_lock); - blk_generic_end_io_acct(zv->zv_zso->zvo_queue, READ, - &zv->zv_zso->zvo_disk->part0, start_jif); + + if (acct) + blk_generic_end_io_acct(q, disk, READ, bio, start_time); + BIO_END_IO(bio, -error); kmem_free(zvr, sizeof (zv_request_t)); }