From 4568b5cfba923f5c27b32c61459d9f870772a50f Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Mon, 12 Apr 2021 19:59:22 +0200 Subject: [PATCH] FreeBSD: add support for lockless symlink lookup Reviewed-by: Ryan Moeller Signed-off-by: Mateusz Guzik Closes #11883 --- include/os/freebsd/zfs/sys/zfs_znode_impl.h | 1 + module/os/freebsd/zfs/zfs_vnops_os.c | 86 ++++++++++++++++++++- module/os/freebsd/zfs/zfs_znode.c | 15 ++++ 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/include/os/freebsd/zfs/sys/zfs_znode_impl.h b/include/os/freebsd/zfs/sys/zfs_znode_impl.h index d7cdb360c2..e90008c70a 100644 --- a/include/os/freebsd/zfs/sys/zfs_znode_impl.h +++ b/include/os/freebsd/zfs/sys/zfs_znode_impl.h @@ -54,6 +54,7 @@ extern "C" { #define ZNODE_OS_FIELDS \ struct zfsvfs *z_zfsvfs; \ vnode_t *z_vnode; \ + char *z_cached_symlink; \ uint64_t z_uid; \ uint64_t z_gid; \ uint64_t z_gen; \ diff --git a/module/os/freebsd/zfs/zfs_vnops_os.c b/module/os/freebsd/zfs/zfs_vnops_os.c index 5588d787af..90fab3b278 100644 --- a/module/os/freebsd/zfs/zfs_vnops_os.c +++ b/module/os/freebsd/zfs/zfs_vnops_os.c @@ -4466,6 +4466,28 @@ zfs_freebsd_fplookup_vexec(struct vop_fplookup_vexec_args *v) } #endif +#if __FreeBSD_version >= 1300139 +static int +zfs_freebsd_fplookup_symlink(struct vop_fplookup_symlink_args *v) +{ + vnode_t *vp; + znode_t *zp; + char *target; + + vp = v->a_vp; + zp = VTOZ_SMR(vp); + if (__predict_false(zp == NULL)) { + return (EAGAIN); + } + + target = atomic_load_consume_ptr(&zp->z_cached_symlink); + if (target == NULL) { + return (EAGAIN); + } + return (cache_symlink_resolve(v->a_fpl, target, strlen(target))); +} +#endif + #ifndef _SYS_SYSPROTO_H_ struct vop_access_args { struct vnode *a_vp; @@ -4953,6 +4975,10 @@ zfs_freebsd_symlink(struct vop_symlink_args *ap) struct componentname *cnp = ap->a_cnp; vattr_t *vap = ap->a_vap; znode_t *zp = NULL; +#if __FreeBSD_version >= 1300139 + char *symlink; + size_t symlink_len; +#endif int rc; ASSERT(cnp->cn_flags & SAVENAME); @@ -4963,8 +4989,21 @@ zfs_freebsd_symlink(struct vop_symlink_args *ap) rc = zfs_symlink(VTOZ(ap->a_dvp), cnp->cn_nameptr, vap, ap->a_target, &zp, cnp->cn_cred, 0 /* flags */); - if (rc == 0) + if (rc == 0) { *ap->a_vpp = ZTOV(zp); + ASSERT_VOP_ELOCKED(ZTOV(zp), __func__); +#if __FreeBSD_version >= 1300139 + MPASS(zp->z_cached_symlink == NULL); + symlink_len = strlen(ap->a_target); + symlink = cache_symlink_alloc(symlink_len + 1, M_WAITOK); + if (symlink != NULL) { + memcpy(symlink, ap->a_target, symlink_len); + symlink[symlink_len] = '\0'; + atomic_store_rel_ptr((uintptr_t *)&zp->z_cached_symlink, + (uintptr_t)symlink); + } +#endif + } return (rc); } @@ -4980,8 +5019,42 @@ static int zfs_freebsd_readlink(struct vop_readlink_args *ap) { zfs_uio_t uio; + int error; +#if __FreeBSD_version >= 1300139 + znode_t *zp = VTOZ(ap->a_vp); + char *symlink, *base; + size_t symlink_len; + bool trycache; +#endif + zfs_uio_init(&uio, ap->a_uio); - return (zfs_readlink(ap->a_vp, &uio, ap->a_cred, NULL)); +#if __FreeBSD_version >= 1300139 + trycache = false; + if (zfs_uio_segflg(&uio) == UIO_SYSSPACE && + zfs_uio_iovcnt(&uio) == 1) { + base = zfs_uio_iovbase(&uio, 0); + symlink_len = zfs_uio_iovlen(&uio, 0); + trycache = true; + } +#endif + error = zfs_readlink(ap->a_vp, &uio, ap->a_cred, NULL); +#if __FreeBSD_version >= 1300139 + if (atomic_load_ptr(&zp->z_cached_symlink) != NULL || + error != 0 || !trycache) { + return (error); + } + symlink_len -= zfs_uio_resid(&uio); + symlink = cache_symlink_alloc(symlink_len + 1, M_WAITOK); + if (symlink != NULL) { + memcpy(symlink, base, symlink_len); + symlink[symlink_len] = '\0'; + if (!atomic_cmpset_rel_ptr((uintptr_t *)&zp->z_cached_symlink, + (uintptr_t)NULL, (uintptr_t)symlink)) { + cache_symlink_free(symlink, symlink_len + 1); + } + } +#endif + return (error); } #ifndef _SYS_SYSPROTO_H_ @@ -5743,6 +5816,9 @@ struct vop_vector zfs_vnodeops = { .vop_reclaim = zfs_freebsd_reclaim, #if __FreeBSD_version >= 1300102 .vop_fplookup_vexec = zfs_freebsd_fplookup_vexec, +#endif +#if __FreeBSD_version >= 1300139 + .vop_fplookup_symlink = zfs_freebsd_fplookup_symlink, #endif .vop_access = zfs_freebsd_access, .vop_allocate = VOP_EINVAL, @@ -5792,6 +5868,9 @@ struct vop_vector zfs_fifoops = { .vop_fsync = zfs_freebsd_fsync, #if __FreeBSD_version >= 1300102 .vop_fplookup_vexec = zfs_freebsd_fplookup_vexec, +#endif +#if __FreeBSD_version >= 1300139 + .vop_fplookup_symlink = zfs_freebsd_fplookup_symlink, #endif .vop_access = zfs_freebsd_access, .vop_getattr = zfs_freebsd_getattr, @@ -5815,6 +5894,9 @@ struct vop_vector zfs_shareops = { .vop_default = &default_vnodeops, #if __FreeBSD_version >= 1300121 .vop_fplookup_vexec = VOP_EAGAIN, +#endif +#if __FreeBSD_version >= 1300139 + .vop_fplookup_symlink = VOP_EAGAIN, #endif .vop_access = zfs_freebsd_access, .vop_inactive = zfs_freebsd_inactive, diff --git a/module/os/freebsd/zfs/zfs_znode.c b/module/os/freebsd/zfs/zfs_znode.c index 1ef38299a6..5dbfc8ff1e 100644 --- a/module/os/freebsd/zfs/zfs_znode.c +++ b/module/os/freebsd/zfs/zfs_znode.c @@ -444,6 +444,9 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz, zp->z_blksz = blksz; zp->z_seq = 0x7A4653; zp->z_sync_cnt = 0; +#if __FreeBSD_version >= 1300139 + atomic_store_ptr(&zp->z_cached_symlink, NULL); +#endif vp = ZTOV(zp); @@ -1237,6 +1240,9 @@ void zfs_znode_free(znode_t *zp) { zfsvfs_t *zfsvfs = zp->z_zfsvfs; +#if __FreeBSD_version >= 1300139 + char *symlink; +#endif ASSERT(zp->z_sa_hdl == NULL); zp->z_vnode = NULL; @@ -1246,6 +1252,15 @@ zfs_znode_free(znode_t *zp) zfsvfs->z_nr_znodes--; mutex_exit(&zfsvfs->z_znodes_lock); +#if __FreeBSD_version >= 1300139 + symlink = atomic_load_ptr(&zp->z_cached_symlink); + if (symlink != NULL) { + atomic_store_rel_ptr((uintptr_t *)&zp->z_cached_symlink, + (uintptr_t)NULL); + cache_symlink_free(symlink, strlen(symlink) + 1); + } +#endif + if (zp->z_acl_cached) { zfs_acl_free(zp->z_acl_cached); zp->z_acl_cached = NULL;