diff --git a/module/os/freebsd/zfs/zfs_vnops_os.c b/module/os/freebsd/zfs/zfs_vnops_os.c
index a18ac21dc7..4672b9c2b8 100644
--- a/module/os/freebsd/zfs/zfs_vnops_os.c
+++ b/module/os/freebsd/zfs/zfs_vnops_os.c
@@ -5695,7 +5695,7 @@ zfs_setextattr(struct vop_setextattr_args *ap)
 	}
 	if (error) {
 		error = zfs_setextattr_dir(ap, attrname);
-		if (error == 0)
+		if (error == 0 && zp->z_is_sa)
 			/*
 			 * Successfully put into dir, we need to clear the one
 			 * in SA if present.