c/mirror_zfs
1
0
Fork 0
mirror of https://git.proxmox.com/git/mirror_zfs.git synced 2024-12-26 03:09:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

FreeBSD: Add support for lockless lookup

Browse Source 
Authored-by: mjg <mjg@FreeBSD.org>
Reviewed-by: Ryan Moeller <ryan@ixsystems.com>
Signed-off-by: Matt Macy <mmacy@FreeBSD.org>
Closes #10657
This commit is contained in:
Matthew Macy 2020-08-05 10:19:51 -07:00 committed by GitHub
parent 22dcf89181
commit 1b376d176e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 176 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
include/os/freebsd/spl/sys/ccompat.h
View File

@ -55,6 +55,21 @@
#else
#define getnewvnode_reserve_() getnewvnode_reserve(1)
#endif
#if __FreeBSD_version < 1300102
#define ASSERT_VOP_IN_SEQC(zp)
#define MNTK_FPLOOKUP 0
#define vn_seqc_write_begin(vp)
#define vn_seqc_write_end(vp)
#ifndef VFS_SMR_DECLARE
#define VFS_SMR_DECLARE
#endif
#ifndef VFS_SMR_ZONE_SET
#define VFS_SMR_ZONE_SET(zone)
#endif
#endif
struct hlist_node {
struct hlist_node *next, **pprev;
};

1
include/os/freebsd/zfs/sys/zfs_znode_impl.h
View File

@ -99,6 +99,7 @@ extern minor_t zfsdev_minor_alloc(void);
#define ZTOV(ZP) ((ZP)->z_vnode)
#define ZTOI(ZP) ((ZP)->z_vnode)
#define VTOZ(VP) ((struct znode *)(VP)->v_data)
#define VTOZ_SMR(VP) ((znode_t *)vn_load_v_data_smr(VP))
#define ITOZ(VP) ((struct znode *)(VP)->v_data)
#define zhold(zp) vhold(ZTOV((zp)))
#define zrele(zp) vrele(ZTOV((zp)))

3
module/os/freebsd/spl/spl_vfs.c
View File

@ -161,6 +161,7 @@ mount_snapshot(kthread_t *td, vnode_t **vpp, const char *fstype, char *fspath,
vput(vp);
return (error);
}
vn_seqc_write_begin(vp);
VOP_UNLOCK1(vp);
/*
@ -213,6 +214,7 @@ mount_snapshot(kthread_t *td, vnode_t **vpp, const char *fstype, char *fspath,
VI_LOCK(vp);
vp->v_iflag &= ~VI_MOUNT;
VI_UNLOCK(vp);
vn_seqc_write_end(vp);
vput(vp);
vfs_unbusy(mp);
vfs_freeopts(mp->mnt_optnew);
@ -248,6 +250,7 @@ mount_snapshot(kthread_t *td, vnode_t **vpp, const char *fstype, char *fspath,
vfs_event_signal(NULL, VQ_MOUNT, 0);
if (VFS_ROOT(mp, LK_EXCLUSIVE, &mvp))
panic("mount: lost mount");
vn_seqc_write_end(vp);
VOP_UNLOCK1(vp);
#if __FreeBSD_version >= 1300048
vfs_op_exit(mp);

3
module/os/freebsd/zfs/zfs_acl.c
View File

@ -1144,6 +1144,7 @@ zfs_acl_chown_setattr(znode_t *zp)
if (zp->z_zfsvfs->z_replay == B_FALSE)
ASSERT_VOP_ELOCKED(ZTOV(zp), __func__);
ASSERT(MUTEX_HELD(&zp->z_acl_lock));
ASSERT_VOP_IN_SEQC(ZTOV(zp));
if ((error = zfs_acl_node_read(zp, B_TRUE, &aclp, B_FALSE)) == 0)
zp->z_mode = zfs_mode_compute(zp->z_mode, aclp,
@ -1171,6 +1172,8 @@ zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t *cr, dmu_tx_t *tx)
int count = 0;
zfs_acl_phys_t acl_phys;
ASSERT_VOP_IN_SEQC(ZTOV(zp));
mode = zp->z_mode;
mode = zfs_mode_compute(mode, aclp, &zp->z_pflags,

3
module/os/freebsd/zfs/zfs_vfsops.c
View File

@ -1192,6 +1192,9 @@ zfs_domount(vfs_t *vfsp, char *osname)
vfsp->mnt_kern_flag |= MNTK_NOMSYNC;
vfsp->mnt_kern_flag |= MNTK_VMSETSIZE_BUG;
#if defined(_KERNEL) && !defined(KMEM_DEBUG)
vfsp->mnt_kern_flag |= MNTK_FPLOOKUP;
#endif
/*
* The fsid is 64 bits, composed of an 8-bit fs type, which
* separates our fsid from any other filesystem types, and a

58
module/os/freebsd/zfs/zfs_vnops.c
View File

@ -39,6 +39,9 @@
#include <sys/endian.h>
#include <sys/vm.h>
#include <sys/vnode.h>
#if __FreeBSD_version >= 1300102
#include <sys/smr.h>
#endif
#include <sys/dirent.h>
#include <sys/file.h>
#include <sys/stat.h>
@ -91,6 +94,8 @@
#define VN_OPEN_INVFS 0x0
#endif
VFS_SMR_DECLARE;
#if __FreeBSD_version >= 1300047
#define vm_page_wire_lock(pp)
#define vm_page_wire_unlock(pp)
@ -3932,6 +3937,7 @@ zfs_rename_(vnode_t *sdvp, vnode_t **svpp, struct componentname *scnp,
char *snm = scnp->cn_nameptr;
char *tnm = tcnp->cn_nameptr;
int error = 0;
bool want_seqc_end __maybe_unused = false;
/* Reject renames across filesystems. */
if ((*svpp)->v_mount != tdvp->v_mount ||
@ -4075,6 +4081,15 @@ zfs_rename_(vnode_t *sdvp, vnode_t **svpp, struct componentname *scnp,
}
}
vn_seqc_write_begin(*svpp);
vn_seqc_write_begin(sdvp);
if (*tvpp != NULL)
vn_seqc_write_begin(*tvpp);
if (tdvp != *tvpp)
vn_seqc_write_begin(tdvp);
#if __FreeBSD_version >= 1300102
want_seqc_end = true;
#endif
vnevent_rename_src(*svpp, sdvp, scnp->cn_nameptr, ct);
if (tzp)
vnevent_rename_dest(*tvpp, tdvp, tnm, ct);
@ -4161,10 +4176,20 @@ zfs_rename_(vnode_t *sdvp, vnode_t **svpp, struct componentname *scnp,
unlockout: /* all 4 vnodes are locked, ZFS_ENTER called */
ZFS_EXIT(zfsvfs);
if (want_seqc_end) {
vn_seqc_write_end(*svpp);
vn_seqc_write_end(sdvp);
if (*tvpp != NULL)
vn_seqc_write_end(*tvpp);
if (tdvp != *tvpp)
vn_seqc_write_end(tdvp);
want_seqc_end = false;
}
VOP_UNLOCK1(*svpp);
VOP_UNLOCK1(sdvp);
out: /* original two vnodes are locked */
MPASS(!want_seqc_end);
if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
zil_commit(zilog, 0);
@ -5196,6 +5221,33 @@ zfs_freebsd_write(struct vop_write_args *ap)
ap->a_cred));
}
#if __FreeBSD_version >= 1300102
/*
* VOP_FPLOOKUP_VEXEC routines are subject to special circumstances, see
* the comment above cache_fplookup for details.
*/
static int
zfs_freebsd_fplookup_vexec(struct vop_fplookup_vexec_args *v)
{
vnode_t *vp;
znode_t *zp;
uint64_t pflags;
vp = v->a_vp;
zp = VTOZ_SMR(vp);
if (__predict_false(zp == NULL))
return (EAGAIN);
pflags = atomic_load_64(&zp->z_pflags);
if (pflags & ZFS_AV_QUARANTINED)
return (EAGAIN);
if (pflags & ZFS_XATTR)
return (EAGAIN);
if ((pflags & ZFS_NO_EXECS_DENIED) == 0)
return (EAGAIN);
return (0);
}
#endif
#ifndef _SYS_SYSPROTO_H_
struct vop_access_args {
struct vnode *a_vp;
@ -6483,6 +6535,9 @@ struct vop_vector zfs_vnodeops = {
.vop_need_inactive = zfs_freebsd_need_inactive,
#endif
.vop_reclaim = zfs_freebsd_reclaim,
#if __FreeBSD_version >= 1300102
.vop_fplookup_vexec = zfs_freebsd_fplookup_vexec,
#endif
.vop_access = zfs_freebsd_access,
.vop_allocate = VOP_EINVAL,
.vop_lookup = zfs_cache_lookup,
@ -6537,6 +6592,9 @@ VFS_VOP_VECTOR_REGISTER(zfs_vnodeops);
struct vop_vector zfs_fifoops = {
.vop_default = &fifo_specops,
.vop_fsync = zfs_freebsd_fsync,
#if __FreeBSD_version >= 1300102
.vop_fplookup_vexec = zfs_freebsd_fplookup_vexec,
#endif
.vop_access = zfs_freebsd_access,
.vop_getattr = zfs_freebsd_getattr,
.vop_inactive = zfs_freebsd_inactive,

105
module/os/freebsd/zfs/zfs_znode.c
View File

@ -97,7 +97,13 @@ SYSCTL_INT(_debug_sizeof, OID_AUTO, znode, CTLFLAG_RD,
*/
krwlock_t zfsvfs_lock;
#if defined(_KERNEL) && !defined(KMEM_DEBUG) && \
__FreeBSD_version >= 1300102
#define _ZFS_USE_SMR
static uma_zone_t znode_uma_zone;
#else
static kmem_cache_t *znode_cache = NULL;
#endif
extern struct vop_vector zfs_vnodeops;
extern struct vop_vector zfs_fifoops;
@ -169,6 +175,53 @@ zfs_znode_cache_destructor(void *buf, void *arg)
ASSERT(zp->z_acl_cached == NULL);
}
#ifdef _ZFS_USE_SMR
VFS_SMR_DECLARE;
static int
zfs_znode_cache_constructor_smr(void *mem, int size __unused, void *private,
int flags)
{
return (zfs_znode_cache_constructor(mem, private, flags));
}
static void
zfs_znode_cache_destructor_smr(void *mem, int size __unused, void *private)
{
zfs_znode_cache_destructor(mem, private);
}
void
zfs_znode_init(void)
{
/*
* Initialize zcache
*/
rw_init(&zfsvfs_lock, NULL, RW_DEFAULT, NULL);
ASSERT(znode_uma_zone == NULL);
znode_uma_zone = uma_zcreate("zfs_znode_cache",
sizeof (znode_t), zfs_znode_cache_constructor_smr,
zfs_znode_cache_destructor_smr, NULL, NULL, 0, 0);
VFS_SMR_ZONE_SET(znode_uma_zone);
}
static znode_t *
zfs_znode_alloc_kmem(int flags)
{
return (uma_zalloc_smr(znode_uma_zone, flags));
}
static void
zfs_znode_free_kmem(znode_t *zp)
{
uma_zfree_smr(znode_uma_zone, zp);
}
#else
void
zfs_znode_init(void)
{
@ -180,18 +233,40 @@ zfs_znode_init(void)
znode_cache = kmem_cache_create("zfs_znode_cache",
sizeof (znode_t), 0, zfs_znode_cache_constructor,
zfs_znode_cache_destructor, NULL, NULL, NULL, 0);
// kmem_cache_set_move(znode_cache, zfs_znode_move);
}
static znode_t *
zfs_znode_alloc_kmem(int flags)
{
return (kmem_cache_alloc(znode_cache, flags));
}
static void
zfs_znode_free_kmem(znode_t *zp)
{
kmem_cache_free(znode_cache, zp);
}
#endif
void
zfs_znode_fini(void)
{
/*
* Cleanup zcache
*/
if (znode_cache)
#ifdef _ZFS_USE_SMR
if (znode_uma_zone) {
uma_zdestroy(znode_uma_zone);
znode_uma_zone = NULL;
}
#else
if (znode_cache) {
kmem_cache_destroy(znode_cache);
znode_cache = NULL;
znode_cache = NULL;
}
#endif
rw_destroy(&zfsvfs_lock);
}
@ -211,7 +286,7 @@ zfs_create_share_dir(zfsvfs_t *zfsvfs, dmu_tx_t *tx)
vattr.va_uid = crgetuid(kcred);
vattr.va_gid = crgetgid(kcred);
sharezp = kmem_cache_alloc(znode_cache, KM_SLEEP);
sharezp = zfs_znode_alloc_kmem(KM_SLEEP);
ASSERT(!POINTER_IS_VALID(sharezp->z_zfsvfs));
sharezp->z_moved = 0;
sharezp->z_unlinked = 0;
@ -230,7 +305,7 @@ zfs_create_share_dir(zfsvfs_t *zfsvfs, dmu_tx_t *tx)
zfs_acl_ids_free(&acl_ids);
sa_handle_destroy(sharezp->z_sa_hdl);
kmem_cache_free(znode_cache, sharezp);
zfs_znode_free_kmem(sharezp);
return (error);
}
@ -349,7 +424,12 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz,
int count = 0;
int error;
zp = kmem_cache_alloc(znode_cache, KM_SLEEP);
zp = zfs_znode_alloc_kmem(KM_SLEEP);
#ifndef _ZFS_USE_SMR
KASSERT((zfsvfs->z_parent->z_vfs->mnt_kern_flag & MNTK_FPLOOKUP) == 0,
("%s: fast path lookup enabled without smr", __func__));
#endif
#if __FreeBSD_version >= 1300076
KASSERT(curthread->td_vp_reserved != NULL,
@ -360,7 +440,7 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz,
#endif
error = getnewvnode("zfs", zfsvfs->z_parent->z_vfs, &zfs_vnodeops, &vp);
if (error != 0) {
kmem_cache_free(znode_cache, zp);
zfs_znode_free_kmem(zp);
return (NULL);
}
zp->z_vnode = vp;
@ -416,7 +496,7 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz,
sa_handle_destroy(zp->z_sa_hdl);
zfs_vnode_forget(vp);
zp->z_vnode = NULL;
kmem_cache_free(znode_cache, zp);
zfs_znode_free_kmem(zp);
return (NULL);
}
@ -763,6 +843,8 @@ zfs_xvattr_set(znode_t *zp, xvattr_t *xvap, dmu_tx_t *tx)
xoap = xva_getxoptattr(xvap);
ASSERT(xoap);
ASSERT_VOP_IN_SEQC(ZTOV(zp));
if (XVA_ISSET_REQ(xvap, XAT_CREATETIME)) {
uint64_t times[2];
ZFS_TIME_ENCODE(&xoap->xoa_createtime, times);
@ -1191,8 +1273,7 @@ zfs_znode_free(znode_t *zp)
zp->z_acl_cached = NULL;
}
kmem_cache_free(znode_cache, zp);
zfs_znode_free_kmem(zp);
}
void
@ -1628,7 +1709,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
zfsvfs = kmem_zalloc(sizeof (zfsvfs_t), KM_SLEEP);
rootzp = kmem_cache_alloc(znode_cache, KM_SLEEP);
rootzp = zfs_znode_alloc_kmem(KM_SLEEP);
ASSERT(!POINTER_IS_VALID(rootzp->z_zfsvfs));
rootzp->z_moved = 0;
rootzp->z_unlinked = 0;
@ -1672,7 +1753,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
POINTER_INVALIDATE(&rootzp->z_zfsvfs);
sa_handle_destroy(rootzp->z_sa_hdl);
kmem_cache_free(znode_cache, rootzp);
zfs_znode_free_kmem(rootzp);
/*
* Create shares directory