Connect dataset_kstats for FreeBSD

Expand the FreeBSD spl for kstats to support all current types

Move the dataset_kstats_t back to zvol_state_t from zfs_state_os_t
now that it is common once again

```
kstat.zfs/mypool.dataset.objset-0x10b.nunlinked: 0
kstat.zfs/mypool.dataset.objset-0x10b.nunlinks: 0
kstat.zfs/mypool.dataset.objset-0x10b.nread: 150528
kstat.zfs/mypool.dataset.objset-0x10b.reads: 48
kstat.zfs/mypool.dataset.objset-0x10b.nwritten: 134217728
kstat.zfs/mypool.dataset.objset-0x10b.writes: 1024
kstat.zfs/mypool.dataset.objset-0x10b.dataset_name: mypool/datasetname
```

Reviewed-by: Ryan Moeller <ryan@iXsystems.com>
Reviewed by: Sean Eric Fagan <sef@ixsystems.com>
Reviewed-by: Serapheim Dimitropoulos <serapheim@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Allan Jude <allan@klarasystems.com>
Closes #10386

include/os/freebsd/zfs/sys/zfs_vfsops.h

@@ -27,6 +27,7 @@
 #ifndef	_SYS_FS_ZFS_VFSOPS_H
 #define	_SYS_FS_ZFS_VFSOPS_H
 
+#include <sys/dataset_kstats.h>
 #include <sys/list.h>
 #include <sys/vfs.h>
 #include <sys/zil.h>
@@ -82,6 +83,7 @@ struct zfsvfs {
 	uint8_t		z_xattr;	/* xattr type in use */
 	uint64_t	z_version;	/* ZPL version */
 	uint64_t	z_shares_dir;	/* hidden shares dir */
+	dataset_kstats_t	z_kstat;	/* fs kstats */
 	kmutex_t	z_lock;
 	uint64_t	z_userquota_obj;
 	uint64_t	z_groupquota_obj;

include/sys/zvol_impl.h

@@ -49,6 +49,7 @@ typedef struct zvol_state {
 	zilog_t			*zv_zilog;	/* ZIL handle */
 	zfs_rangelock_t		zv_rangelock;	/* for range locking */
 	dnode_t			*zv_dn;		/* dnode hold */
+	dataset_kstats_t	zv_kstat;	/* zvol kstats */
 	list_node_t		zv_next;	/* next zvol_state_t linkage */
 	uint64_t		zv_hash;	/* name hash */
 	struct hlist_node	zv_hlink;	/* hash link */

module/os/freebsd/spl/spl_kstat.c

@@ -162,13 +162,38 @@ __kstat_create(const char *module, int instance, const char *name,
 static int
 kstat_sysctl(SYSCTL_HANDLER_ARGS)
 {
-	kstat_named_t *ksent = arg1;
+	kstat_t *ksp = arg1;
+	kstat_named_t *ksent = ksp->ks_data;
 	uint64_t val;
 
+	/* Select the correct element */
+	ksent += arg2;
+	/* Update the aggsums before reading */
+	(void) ksp->ks_update(ksp, KSTAT_READ);
 	val = ksent->value.ui64;
+
 	return (sysctl_handle_64(oidp, &val, 0, req));
 }
 
+static int
+kstat_sysctl_string(SYSCTL_HANDLER_ARGS)
+{
+	kstat_t *ksp = arg1;
+	kstat_named_t *ksent = ksp->ks_data;
+	char *val;
+	uint32_t len = 0;
+
+	/* Select the correct element */
+	ksent += arg2;
+	/* Update the aggsums before reading */
+	(void) ksp->ks_update(ksp, KSTAT_READ);
+	val = KSTAT_NAMED_STR_PTR(ksent);
+	len = KSTAT_NAMED_STR_BUFLEN(ksent);
+	val[len-1] = '\0';
+
+	return (sysctl_handle_string(oidp, val, len, req));
+}
+
 void
 kstat_install(kstat_t *ksp)
 {
@@ -195,37 +220,57 @@ kstat_install(kstat_t *ksp)
 			namelast = ksent->name;
 		}
 		switch (typelast) {
+			case KSTAT_DATA_CHAR:
+				/* Not Implemented */
+				break;
 			case KSTAT_DATA_INT32:
 				SYSCTL_ADD_PROC(&ksp->ks_sysctl_ctx,
 				    SYSCTL_CHILDREN(ksp->ks_sysctl_root),
 				    OID_AUTO, namelast,
-				    CTLTYPE_S32 | CTLFLAG_RD, ksent,
+				    CTLTYPE_S32 | CTLFLAG_RD, ksp, i,
-				    sizeof (*ksent), kstat_sysctl, "I",
+				    kstat_sysctl, "I", namelast);
-				    namelast);
 				break;
 			case KSTAT_DATA_UINT32:
 				SYSCTL_ADD_PROC(&ksp->ks_sysctl_ctx,
 				    SYSCTL_CHILDREN(ksp->ks_sysctl_root),
 				    OID_AUTO, namelast,
-				    CTLTYPE_U32 | CTLFLAG_RD, ksent,
+				    CTLTYPE_U32 | CTLFLAG_RD, ksp, i,
-				    sizeof (*ksent), kstat_sysctl, "IU",
+				    kstat_sysctl, "IU", namelast);
-				    namelast);
 				break;
 			case KSTAT_DATA_INT64:
 				SYSCTL_ADD_PROC(&ksp->ks_sysctl_ctx,
 				    SYSCTL_CHILDREN(ksp->ks_sysctl_root),
 				    OID_AUTO, namelast,
-				    CTLTYPE_S64 | CTLFLAG_RD, ksent,
+				    CTLTYPE_S64 | CTLFLAG_RD, ksp, i,
-				    sizeof (*ksent), kstat_sysctl, "Q",
+				    kstat_sysctl, "Q", namelast);
-				    namelast);
 				break;
 			case KSTAT_DATA_UINT64:
 				SYSCTL_ADD_PROC(&ksp->ks_sysctl_ctx,
 				    SYSCTL_CHILDREN(ksp->ks_sysctl_root),
 				    OID_AUTO, namelast,
-				    CTLTYPE_U64 | CTLFLAG_RD, ksent,
+				    CTLTYPE_U64 | CTLFLAG_RD, ksp, i,
-				    sizeof (*ksent), kstat_sysctl, "QU",
+				    kstat_sysctl, "QU", namelast);
-				    namelast);
+				break;
+			case KSTAT_DATA_LONG:
+				SYSCTL_ADD_PROC(&ksp->ks_sysctl_ctx,
+				    SYSCTL_CHILDREN(ksp->ks_sysctl_root),
+				    OID_AUTO, namelast,
+				    CTLTYPE_LONG | CTLFLAG_RD, ksp, i,
+				    kstat_sysctl, "L", namelast);
+				break;
+			case KSTAT_DATA_ULONG:
+				SYSCTL_ADD_PROC(&ksp->ks_sysctl_ctx,
+				    SYSCTL_CHILDREN(ksp->ks_sysctl_root),
+				    OID_AUTO, namelast,
+				    CTLTYPE_ULONG | CTLFLAG_RD, ksp, i,
+				    kstat_sysctl, "LU", namelast);
+				break;
+			case KSTAT_DATA_STRING:
+				SYSCTL_ADD_PROC(&ksp->ks_sysctl_ctx,
+				    SYSCTL_CHILDREN(ksp->ks_sysctl_root),
+				    OID_AUTO, namelast,
+				    CTLTYPE_STRING | CTLFLAG_RD, ksp, i,
+				    kstat_sysctl_string, "A", namelast);
 				break;
 			default:
 				panic("unsupported type: %d", typelast);

module/os/freebsd/zfs/zfs_dir.c

@@ -276,6 +276,8 @@ zfs_unlinked_add(znode_t *zp, dmu_tx_t *tx)
 
 	VERIFY3U(0, ==,
 	    zap_add_int(zfsvfs->z_os, zfsvfs->z_unlinkedobj, zp->z_id, tx));
+
+	dataset_kstats_update_nunlinks_kstat(&zfsvfs->z_kstat, 1);
 }
 
 /*
@@ -532,6 +534,8 @@ zfs_rmnode(znode_t *zp)
 
 	mutex_exit(&os->os_dsl_dataset->ds_dir->dd_activity_lock);
 
+	dataset_kstats_update_nunlinked_kstat(&zfsvfs->z_kstat, 1);
+
 	zfs_znode_delete(zp, tx);
 
 	dmu_tx_commit(tx);

module/os/freebsd/zfs/zfs_vfsops.c

@@ -1014,6 +1014,9 @@ zfsvfs_setup(zfsvfs_t *zfsvfs, boolean_t mounting)
 	if (mounting) {
 		boolean_t readonly;
 
+		ASSERT3P(zfsvfs->z_kstat.dk_kstats, ==, NULL);
+		dataset_kstats_create(&zfsvfs->z_kstat, zfsvfs->z_os);
+
 		/*
 		 * During replay we remove the read only flag to
 		 * allow replays to succeed.
@@ -1023,6 +1026,16 @@ zfsvfs_setup(zfsvfs_t *zfsvfs, boolean_t mounting)
 			zfsvfs->z_vfs->vfs_flag &= ~VFS_RDONLY;
 		} else {
 			dsl_dir_t *dd;
+			zap_stats_t zs;
+
+			if (zap_get_stats(zfsvfs->z_os, zfsvfs->z_unlinkedobj,
+			    &zs) == 0) {
+				dataset_kstats_update_nunlinks_kstat(
+				    &zfsvfs->z_kstat, zs.zs_num_entries);
+				dprintf_ds(zfsvfs->z_os->os_dsl_dataset,
+				    "num_entries in unlinked set: %llu",
+				    zs.zs_num_entries);
+			}
 
 			zfs_unlinked_drain(zfsvfs);
 			dd = zfsvfs->z_os->os_dsl_dataset->ds_dir;
@@ -1112,6 +1125,7 @@ zfsvfs_free(zfsvfs_t *zfsvfs)
 	rw_destroy(&zfsvfs->z_fuid_lock);
 	for (i = 0; i != ZFS_OBJ_MTX_SZ; i++)
 		mutex_destroy(&zfsvfs->z_hold_mtx[i]);
+	dataset_kstats_destroy(&zfsvfs->z_kstat);
 	kmem_free(zfsvfs, sizeof (zfsvfs_t));
 }
 

module/os/freebsd/zfs/zfs_vnops.c

@@ -736,8 +736,9 @@ zfs_read(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr)
 {
 	znode_t		*zp = VTOZ(vp);
 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
-	ssize_t		n, nbytes;
+	ssize_t		n, nbytes, start_resid;
 	int		error = 0;
+	int64_t		nread;
 	zfs_locked_range_t		*lr;
 
 	ZFS_ENTER(zfsvfs);
@@ -794,6 +795,7 @@ zfs_read(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr)
 
 	ASSERT(uio->uio_loffset < zp->z_size);
 	n = MIN(uio->uio_resid, zp->z_size - uio->uio_loffset);
+	start_resid = n;
 
 	while (n > 0) {
 		nbytes = MIN(n, zfs_read_chunk_size -
@@ -816,6 +818,10 @@ zfs_read(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr)
 
 		n -= nbytes;
 	}
+
+	nread = start_resid - n;
+	dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, nread);
+
 out:
 	zfs_rangelock_exit(lr);
 
@@ -872,6 +878,7 @@ zfs_write(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr)
 	sa_bulk_attr_t	bulk[4];
 	uint64_t	mtime[2], ctime[2];
 	uint64_t	uid, gid, projid;
+	int64_t		nwritten;
 
 	/*
 	 * Fasttrack empty write
@@ -1214,6 +1221,9 @@ zfs_write(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr)
 	    zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
 		zil_commit(zilog, zp->z_id);
 
+	nwritten = start_resid - uio->uio_resid;
+	dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, nwritten);
+
 	ZFS_EXIT(zfsvfs);
 	return (0);
 }

module/os/freebsd/zfs/zvol_os.c

@@ -85,6 +85,7 @@
 #include <sys/vdev_raidz.h>
 #include <sys/zvol.h>
 #include <sys/zil_impl.h>
+#include <sys/dataset_kstats.h>
 #include <sys/dbuf.h>
 #include <sys/dmu_tx.h>
 #include <sys/zfeature.h>
@@ -672,6 +673,23 @@ unlock:
 	if (bp->bio_completed < bp->bio_length && off > volsize)
 		error = EINVAL;
 
+	switch (bp->bio_cmd) {
+	case BIO_FLUSH:
+		break;
+	case BIO_READ:
+		dataset_kstats_update_read_kstats(&zv->zv_kstat,
+		    bp->bio_completed);
+		break;
+	case BIO_WRITE:
+		dataset_kstats_update_write_kstats(&zv->zv_kstat,
+		    bp->bio_completed);
+		break;
+	case BIO_DELETE:
+		break;
+	default:
+		break;
+	}
+
 	if (sync) {
 sync:
 		zil_commit(zv->zv_zilog, ZVOL_OBJ);
@@ -1193,6 +1211,7 @@ zvol_free(zvol_state_t *zv)
 	}
 
 	mutex_destroy(&zv->zv_state_lock);
+	dataset_kstats_destroy(&zv->zv_kstat);
 	kmem_free(zv->zv_zso, sizeof (struct zvol_state_os));
 	kmem_free(zv, sizeof (zvol_state_t));
 	zvol_minors--;
@@ -1310,6 +1329,8 @@ zvol_create_minor_impl(const char *name)
 		else
 			zil_replay(os, zv, zvol_replay_vector);
 	}
+	ASSERT3P(zv->zv_kstat.dk_kstats, ==, NULL);
+	dataset_kstats_create(&zv->zv_kstat, zv->zv_objset);
 
 	/* XXX do prefetch */
 

module/os/linux/zfs/zvol_os.c

@@ -50,7 +50,6 @@ unsigned int zvol_threads = 32;
 struct zvol_state_os {
 	struct gendisk		*zvo_disk;	/* generic disk */
 	struct request_queue	*zvo_queue;	/* request queue */
-	dataset_kstats_t	zvo_kstat;	/* zvol kstats */
 	dev_t			zvo_dev;	/* device id */
 };
 
@@ -163,7 +162,7 @@ zvol_write(void *arg)
 	zfs_rangelock_exit(lr);
 
 	int64_t nwritten = start_resid - uio.uio_resid;
-	dataset_kstats_update_write_kstats(&zv->zv_zso->zvo_kstat, nwritten);
+	dataset_kstats_update_write_kstats(&zv->zv_kstat, nwritten);
 	task_io_account_write(nwritten);
 
 	if (sync)
@@ -286,7 +285,7 @@ zvol_read(void *arg)
 	zfs_rangelock_exit(lr);
 
 	int64_t nread = start_resid - uio.uio_resid;
-	dataset_kstats_update_read_kstats(&zv->zv_zso->zvo_kstat, nread);
+	dataset_kstats_update_read_kstats(&zv->zv_kstat, nread);
 	task_io_account_read(nread);
 
 	rw_exit(&zv->zv_suspend_lock);
@@ -864,7 +863,7 @@ zvol_free(zvol_state_t *zv)
 	    MINOR(zv->zv_zso->zvo_dev) >> ZVOL_MINOR_BITS);
 
 	mutex_destroy(&zv->zv_state_lock);
-	dataset_kstats_destroy(&zv->zv_zso->zvo_kstat);
+	dataset_kstats_destroy(&zv->zv_kstat);
 
 	kmem_free(zv->zv_zso, sizeof (struct zvol_state_os));
 	kmem_free(zv, sizeof (zvol_state_t));
@@ -963,8 +962,8 @@ zvol_os_create_minor(const char *name)
 		else
 			zil_replay(os, zv, zvol_replay_vector);
 	}
-	ASSERT3P(zv->zv_zso->zvo_kstat.dk_kstats, ==, NULL);
+	ASSERT3P(zv->zv_kstat.dk_kstats, ==, NULL);
-	dataset_kstats_create(&zv->zv_zso->zvo_kstat, zv->zv_objset);
+	dataset_kstats_create(&zv->zv_kstat, zv->zv_objset);
 
 	/*
 	 * When udev detects the addition of the device it will immediately