/*      $NetBSD: vfs_mount.c,v 1.111 2026/03/22 13:03:26 yamt Exp $     */

/*-
 * Copyright (c) 1997-2020 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
 * NASA Ames Research Center, by Charles M. Hannum, and by Andrew Doran.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Copyright (c) 1989, 1993
 *      The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *      @(#)vfs_subr.c  8.13 (Berkeley) 4/18/94
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: vfs_mount.c,v 1.111 2026/03/22 13:03:26 yamt Exp $");

#include "veriexec.h"

#include <sys/param.h>
#include <sys/kernel.h>

#include <sys/atomic.h>
#include <sys/buf.h>
#include <sys/conf.h>
#include <sys/device.h>
#include <sys/extattr.h>
#include <sys/fcntl.h>
#include <sys/filedesc.h>
#include <sys/fstrans.h>
#include <sys/kauth.h>
#include <sys/kmem.h>
#include <sys/module.h>
#include <sys/mount.h>
#include <sys/namei.h>
#include <sys/sdt.h>
#include <sys/syscallargs.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/verified_exec.h>
#include <sys/vfs_syscalls.h>
#include <sys/vnode_impl.h>

#include <miscfs/deadfs/deadfs.h>
#include <miscfs/genfs/genfs.h>
#include <miscfs/specfs/specdev.h>

#include <uvm/uvm_swap.h>

enum mountlist_type {
        ME_MOUNT,
        ME_MARKER
};
struct mountlist_entry {
        TAILQ_ENTRY(mountlist_entry) me_list;   /* Mount list. */
        struct mount *me_mount;                 /* Actual mount if ME_MOUNT,
                                                   current mount else. */
        enum mountlist_type me_type;            /* Mount or marker. */
};
struct mount_iterator {
        struct mountlist_entry mi_entry;
};

static struct vnode *vfs_vnode_iterator_next1(struct vnode_iterator *,
    bool (*)(void *, struct vnode *), void *, bool);

/* Root filesystem. */
vnode_t *                       rootvnode;

/* Mounted filesystem list. */
static TAILQ_HEAD(mountlist, mountlist_entry) mountlist;
static kmutex_t                 mountlist_lock __cacheline_aligned;
int vnode_offset_next_by_lru    /* XXX: ugly hack for pstat.c */
    = offsetof(vnode_impl_t, vi_lrulist.tqe_next);

kmutex_t                        vfs_list_lock __cacheline_aligned;

static specificdata_domain_t    mount_specificdata_domain;
static kmutex_t                 mntid_lock;

static kmutex_t                 mountgen_lock __cacheline_aligned;
static uint64_t                 mountgen;

void
vfs_mount_sysinit(void)
{

        TAILQ_INIT(&mountlist);
        mutex_init(&mountlist_lock, MUTEX_DEFAULT, IPL_NONE);
        mutex_init(&vfs_list_lock, MUTEX_DEFAULT, IPL_NONE);

        mount_specificdata_domain = specificdata_domain_create();
        mutex_init(&mntid_lock, MUTEX_DEFAULT, IPL_NONE);
        mutex_init(&mountgen_lock, MUTEX_DEFAULT, IPL_NONE);
        mountgen = 0;
}

struct mount *
vfs_mountalloc(struct vfsops *vfsops, vnode_t *vp)
{
        struct mount *mp;
        int error __diagused;

        mp = kmem_zalloc(sizeof(*mp), KM_SLEEP);
        mp->mnt_op = vfsops;
        mp->mnt_refcnt = 1;
        TAILQ_INIT(&mp->mnt_vnodelist);
        mp->mnt_renamelock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
        mp->mnt_vnodelock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
        mp->mnt_updating = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
        mp->mnt_vnodecovered = vp;
        mount_initspecific(mp);

        error = fstrans_mount(mp);
        KASSERT(error == 0);

        mutex_enter(&mountgen_lock);
        mp->mnt_gen = mountgen++;
        mutex_exit(&mountgen_lock);

        return mp;
}

/*
 * vfs_rootmountalloc: lookup a filesystem type, and if found allocate and
 * initialize a mount structure for it.
 *
 * Devname is usually updated by mount(8) after booting.
 */
int
vfs_rootmountalloc(const char *fstypename, const char *devname,
    struct mount **mpp)
{
        struct vfsops *vfsp = NULL;
        struct mount *mp;
        int error __diagused;

        mutex_enter(&vfs_list_lock);
        LIST_FOREACH(vfsp, &vfs_list, vfs_list)
                if (!strncmp(vfsp->vfs_name, fstypename,
                    sizeof(mp->mnt_stat.f_fstypename)))
                        break;
        if (vfsp == NULL) {
                mutex_exit(&vfs_list_lock);
                return SET_ERROR(ENODEV);
        }
        vfsp->vfs_refcount++;
        mutex_exit(&vfs_list_lock);

        if ((mp = vfs_mountalloc(vfsp, NULL)) == NULL)
                return SET_ERROR(ENOMEM);
        error = vfs_busy(mp);
        KASSERT(error == 0);
        mp->mnt_flag = MNT_RDONLY;
        (void)strlcpy(mp->mnt_stat.f_fstypename, vfsp->vfs_name,
            sizeof(mp->mnt_stat.f_fstypename));
        mp->mnt_stat.f_mntonname[0] = '/';
        mp->mnt_stat.f_mntonname[1] = '\0';
        mp->mnt_stat.f_mntfromname[sizeof(mp->mnt_stat.f_mntfromname) - 1] =
            '\0';
        (void)copystr(devname, mp->mnt_stat.f_mntfromname,
            sizeof(mp->mnt_stat.f_mntfromname) - 1, 0);
        *mpp = mp;
        return 0;
}

/*
 * vfs_getnewfsid: get a new unique fsid.
 */
void
vfs_getnewfsid(struct mount *mp)
{
        static u_short xxxfs_mntid;
        struct mountlist_entry *me;
        fsid_t tfsid;
        int mtype;

        mutex_enter(&mntid_lock);
        if (xxxfs_mntid == 0)
                ++xxxfs_mntid;
        mtype = makefstype(mp->mnt_op->vfs_name);
        tfsid.__fsid_val[0] = makedev(mtype & 0xff, xxxfs_mntid);
        tfsid.__fsid_val[1] = mtype;
        /* Always increment to not return the same fsid to parallel mounts. */
        xxxfs_mntid++;

        /*
         * Directly walk mountlist to prevent deadlock through
         * mountlist_iterator_next() -> vfs_busy().
         */
        mutex_enter(&mountlist_lock);
        for (me = TAILQ_FIRST(&mountlist); me != TAILQ_END(&mountlist); ) {
                if (me->me_type == ME_MOUNT &&
                    me->me_mount->mnt_stat.f_fsidx.__fsid_val[0] ==
                    tfsid.__fsid_val[0] &&
                    me->me_mount->mnt_stat.f_fsidx.__fsid_val[1] ==
                    tfsid.__fsid_val[1]) {
                        tfsid.__fsid_val[0]++;
                        xxxfs_mntid++;
                        me = TAILQ_FIRST(&mountlist);
                } else {
                        me = TAILQ_NEXT(me, me_list);
                }
        }
        mutex_exit(&mountlist_lock);

        mp->mnt_stat.f_fsidx.__fsid_val[0] = tfsid.__fsid_val[0];
        mp->mnt_stat.f_fsidx.__fsid_val[1] = tfsid.__fsid_val[1];
        mp->mnt_stat.f_fsid = mp->mnt_stat.f_fsidx.__fsid_val[0];
        mutex_exit(&mntid_lock);
}

/*
 * Lookup a mount point by filesystem identifier.
 *
 * XXX Needs to add a reference to the mount point.
 */
struct mount *
vfs_getvfs(fsid_t *fsid)
{
        mount_iterator_t *iter;
        struct mount *mp;

        mountlist_iterator_init(&iter);
        while ((mp = mountlist_iterator_next(iter)) != NULL) {
                if (mp->mnt_stat.f_fsidx.__fsid_val[0] == fsid->__fsid_val[0] &&
                    mp->mnt_stat.f_fsidx.__fsid_val[1] == fsid->__fsid_val[1]) {
                        mountlist_iterator_destroy(iter);
                        return mp;
                }
        }
        mountlist_iterator_destroy(iter);
        return NULL;
}

/*
 * Take a reference to a mount structure.
 */
void
vfs_ref(struct mount *mp)
{

        KASSERT(mp->mnt_refcnt > 0 || mutex_owned(&mountlist_lock));

        atomic_inc_uint(&mp->mnt_refcnt);
}

/*
 * Drop a reference to a mount structure, freeing if the last reference.
 */
void
vfs_rele(struct mount *mp)
{

        membar_release();
        if (__predict_true((int)atomic_dec_uint_nv(&mp->mnt_refcnt) > 0)) {
                return;
        }
        membar_acquire();

        /*
         * Nothing else has visibility of the mount: we can now
         * free the data structures.
         */
        KASSERT(mp->mnt_refcnt == 0);
        specificdata_fini(mount_specificdata_domain, &mp->mnt_specdataref);
        mutex_obj_free(mp->mnt_updating);
        mutex_obj_free(mp->mnt_renamelock);
        mutex_obj_free(mp->mnt_vnodelock);
        if (mp->mnt_op != NULL) {
                vfs_delref(mp->mnt_op);
        }
        fstrans_unmount(mp);
        /*
         * Final free of mp gets done from fstrans_mount_dtor().
         *
         * Prevents this memory to be reused as a mount before
         * fstrans releases all references to it.
         */
}

/*
 * Mark a mount point as busy, and gain a new reference to it.  Used to
 * prevent the file system from being unmounted during critical sections.
 *
 * vfs_busy can be called multiple times and by multiple threads
 * and must be accompanied by the same number of vfs_unbusy calls.
 *
 * => The caller must hold a pre-existing reference to the mount.
 * => Will fail if the file system is being unmounted, or is unmounted.
 */
static inline int
_vfs_busy(struct mount *mp, bool wait)
{

        KASSERT(mp->mnt_refcnt > 0);

        if (wait) {
                fstrans_start(mp);
        } else {
                if (fstrans_start_nowait(mp))
                        return SET_ERROR(EBUSY);
        }
        if (__predict_false((mp->mnt_iflag & IMNT_GONE) != 0)) {
                fstrans_done(mp);
                return SET_ERROR(ENOENT);
        }
        vfs_ref(mp);
        return 0;
}

int
vfs_busy(struct mount *mp)
{

        return _vfs_busy(mp, true);
}

int
vfs_trybusy(struct mount *mp)
{

        return _vfs_busy(mp, false);
}

/*
 * Unbusy a busy filesystem.
 *
 * Every successful vfs_busy() call must be undone by a vfs_unbusy() call.
 */
void
vfs_unbusy(struct mount *mp)
{

        KASSERT(mp->mnt_refcnt > 0);

        fstrans_done(mp);
        vfs_rele(mp);
}

/*
 * Change a file systems lower mount.
 * Both the current and the new lower mount may be NULL.  The caller
 * guarantees exclusive access to the mount and holds a pre-existing
 * reference to the new lower mount.
 */
int
vfs_set_lowermount(struct mount *mp, struct mount *lowermp)
{
        struct mount *oldlowermp;
        int error;

#ifdef DEBUG
        /*
         * Limit the depth of file system stack so kernel sanitizers
         * may stress mount/unmount without exhausting the kernel stack.
         */
        int depth;
        struct mount *mp2;

        for (depth = 0, mp2 = lowermp; mp2; depth++, mp2 = mp2->mnt_lower) {
                if (depth == 23)
                        return SET_ERROR(EINVAL);
        }
#endif

        if (lowermp) {
                if (lowermp == dead_rootmount)
                        return SET_ERROR(ENOENT);
                error = vfs_busy(lowermp);
                if (error)
                        return error;
                vfs_ref(lowermp);
        }

        oldlowermp = mp->mnt_lower;
        mp->mnt_lower = lowermp;

        if (lowermp)
                vfs_unbusy(lowermp);

        if (oldlowermp)
                vfs_rele(oldlowermp);

        return 0;
}

struct vnode_iterator {
        vnode_impl_t vi_vnode;
};

void
vfs_vnode_iterator_init(struct mount *mp, struct vnode_iterator **vnip)
{
        vnode_t *vp;
        vnode_impl_t *vip;

        vp = vnalloc_marker(mp);
        vip = VNODE_TO_VIMPL(vp);

        mutex_enter(mp->mnt_vnodelock);
        TAILQ_INSERT_HEAD(&mp->mnt_vnodelist, vip, vi_mntvnodes);
        vp->v_usecount = 1;
        mutex_exit(mp->mnt_vnodelock);

        *vnip = (struct vnode_iterator *)vip;
}

void
vfs_vnode_iterator_destroy(struct vnode_iterator *vni)
{
        vnode_impl_t *mvip = &vni->vi_vnode;
        vnode_t *mvp = VIMPL_TO_VNODE(mvip);
        kmutex_t *lock;

        KASSERT(vnis_marker(mvp));
        if (vrefcnt(mvp) != 0) {
                lock = mvp->v_mount->mnt_vnodelock;
                mutex_enter(lock);
                TAILQ_REMOVE(&mvp->v_mount->mnt_vnodelist, mvip, vi_mntvnodes);
                mvp->v_usecount = 0;
                mutex_exit(lock);
        }
        vnfree_marker(mvp);
}

static struct vnode *
vfs_vnode_iterator_next1(struct vnode_iterator *vni,
    bool (*f)(void *, struct vnode *), void *cl, bool do_wait)
{
        vnode_impl_t *mvip = &vni->vi_vnode;
        struct mount *mp = VIMPL_TO_VNODE(mvip)->v_mount;
        vnode_t *vp;
        vnode_impl_t *vip;
        kmutex_t *lock;
        int error;

        KASSERT(vnis_marker(VIMPL_TO_VNODE(mvip)));

        lock = mp->mnt_vnodelock;
        do {
                mutex_enter(lock);
                vip = TAILQ_NEXT(mvip, vi_mntvnodes);
                TAILQ_REMOVE(&mp->mnt_vnodelist, mvip, vi_mntvnodes);
                VIMPL_TO_VNODE(mvip)->v_usecount = 0;
again:
                if (vip == NULL) {
                        mutex_exit(lock);
                        return NULL;
                }
                vp = VIMPL_TO_VNODE(vip);
                KASSERT(vp != NULL);
                mutex_enter(vp->v_interlock);
                if (vnis_marker(vp) ||
                    vdead_check(vp, (do_wait ? 0 : VDEAD_NOWAIT)) ||
                    (f && !(*f)(cl, vp))) {
                        mutex_exit(vp->v_interlock);
                        vip = TAILQ_NEXT(vip, vi_mntvnodes);
                        goto again;
                }

                TAILQ_INSERT_AFTER(&mp->mnt_vnodelist, vip, mvip,
                    vi_mntvnodes);
                VIMPL_TO_VNODE(mvip)->v_usecount = 1;
                mutex_exit(lock);
                error = vcache_vget(vp);
                KASSERT(error == 0 || error == ENOENT);
        } while (error != 0);

        return vp;
}

struct vnode *
vfs_vnode_iterator_next(struct vnode_iterator *vni,
    bool (*f)(void *, struct vnode *), void *cl)
{

        return vfs_vnode_iterator_next1(vni, f, cl, false);
}

/*
 * Move a vnode from one mount queue to another.
 */
void
vfs_insmntque(vnode_t *vp, struct mount *mp)
{
        vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
        struct mount *omp;
        kmutex_t *lock;

        KASSERT(mp == NULL || (mp->mnt_iflag & IMNT_GONE) == 0 ||
            vp->v_tag == VT_VFS);

        /*
         * Delete from old mount point vnode list, if on one.
         */
        if ((omp = vp->v_mount) != NULL) {
                lock = omp->mnt_vnodelock;
                mutex_enter(lock);
                TAILQ_REMOVE(&vp->v_mount->mnt_vnodelist, vip, vi_mntvnodes);
                mutex_exit(lock);
        }

        /*
         * Insert into list of vnodes for the new mount point, if
         * available.  The caller must take a reference on the mount
         * structure and donate to the vnode.
         */
        if ((vp->v_mount = mp) != NULL) {
                lock = mp->mnt_vnodelock;
                mutex_enter(lock);
                TAILQ_INSERT_TAIL(&mp->mnt_vnodelist, vip, vi_mntvnodes);
                mutex_exit(lock);
        }

        if (omp != NULL) {
                /* Release reference to old mount. */
                vfs_rele(omp);
        }
}

/*
 * Remove any vnodes in the vnode table belonging to mount point mp.
 *
 * If FORCECLOSE is not specified, there should not be any active ones,
 * return error if any are found (nb: this is a user error, not a
 * system error). If FORCECLOSE is specified, detach any active vnodes
 * that are found.
 *
 * If WRITECLOSE is set, only flush out regular file vnodes open for
 * writing.
 *
 * SKIPSYSTEM causes any vnodes marked VV_SYSTEM to be skipped.
 */
#ifdef DEBUG
int busyprt = 0;        /* print out busy vnodes */
struct ctldebug debug1 = { "busyprt", &busyprt };
#endif

static vnode_t *
vflushnext(struct vnode_iterator *marker, int *when)
{
        if (getticks() > *when) {
                yield();
                *when = getticks() + hz / 10;
        }
        preempt_point();
        return vfs_vnode_iterator_next1(marker, NULL, NULL, true);
}

/*
 * Flush one vnode.  Referenced on entry, unreferenced on return.
 */
static int
vflush_one(vnode_t *vp, vnode_t *skipvp, int flags)
{
        int error;
        struct vattr vattr;

        if (vp == skipvp ||
            ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM))) {
                vrele(vp);
                return 0;
        }
        /*
         * If WRITECLOSE is set, only flush out regular file
         * vnodes open for writing or open and unlinked.
         */
        if ((flags & WRITECLOSE)) {
                if (vp->v_type != VREG) {
                        vrele(vp);
                        return 0;
                }
                error = vn_lock(vp, LK_EXCLUSIVE);
                if (error) {
                        KASSERT(error == ENOENT);
                        vrele(vp);
                        return 0;
                }
                error = VOP_FSYNC(vp, curlwp->l_cred, FSYNC_WAIT, 0, 0);
                if (error == 0)
                        error = VOP_GETATTR(vp, &vattr, curlwp->l_cred);
                VOP_UNLOCK(vp);
                if (error) {
                        vrele(vp);
                        return error;
                }
                if (vp->v_writecount == 0 && vattr.va_nlink > 0) {
                        vrele(vp);
                        return 0;
                }
        }
        /*
         * First try to recycle the vnode.
         */
        if (vrecycle(vp))
                return 0;
        /*
         * If FORCECLOSE is set, forcibly close the vnode.
         * For block or character devices, revert to an
         * anonymous device.  For all other files, just
         * kill them.
         */
        if (flags & FORCECLOSE) {
                if (vrefcnt(vp) > 1 &&
                    (vp->v_type == VBLK || vp->v_type == VCHR))
                        vcache_make_anon(vp);
                else
                        vgone(vp);
                return 0;
        }
        vrele(vp);
        return SET_ERROR(EBUSY);
}

int
vflush(struct mount *mp, vnode_t *skipvp, int flags)
{
        vnode_t *vp;
        struct vnode_iterator *marker;
        int busy, error, when, retries = 2;

        do {
                busy = error = when = 0;

                /*
                 * First, flush out any vnode references from the
                 * deferred vrele list.
                 */
                vrele_flush(mp);

                vfs_vnode_iterator_init(mp, &marker);

                while ((vp = vflushnext(marker, &when)) != NULL) {
                        error = vflush_one(vp, skipvp, flags);
                        if (error == EBUSY) {
                                error = 0;
                                busy++;
#ifdef DEBUG
                                if (busyprt && retries == 0)
                                        vprint("vflush: busy vnode", vp);
#endif
                        } else if (error != 0) {
                                break;
                        }
                }

                vfs_vnode_iterator_destroy(marker);
        } while (error == 0 && busy > 0 && retries-- > 0);

        if (error)
                return error;
        if (busy)
                return SET_ERROR(EBUSY);
        return 0;
}

/*
 * Mount a file system.
 */

/*
 * Scan all active processes to see if any of them have a current or root
 * directory onto which the new filesystem has just been  mounted. If so,
 * replace them with the new mount point.
 */
static void
mount_checkdirs(vnode_t *olddp)
{
        vnode_t *newdp, *rele1, *rele2;
        struct cwdinfo *cwdi;
        struct proc *p;
        bool retry;

        if (vrefcnt(olddp) == 1) {
                return;
        }
        if (VFS_ROOT(olddp->v_mountedhere, LK_EXCLUSIVE, &newdp))
                panic("mount: lost mount");

        do {
                retry = false;
                mutex_enter(&proc_lock);
                PROCLIST_FOREACH(p, &allproc) {
                        if ((cwdi = p->p_cwdi) == NULL)
                                continue;
                        /*
                         * Cannot change to the old directory any more,
                         * so even if we see a stale value it is not a
                         * problem.
                         */
                        if (cwdi->cwdi_cdir != olddp &&
                            cwdi->cwdi_rdir != olddp)
                                continue;
                        retry = true;
                        rele1 = NULL;
                        rele2 = NULL;
                        atomic_inc_uint(&cwdi->cwdi_refcnt);
                        mutex_exit(&proc_lock);
                        rw_enter(&cwdi->cwdi_lock, RW_WRITER);
                        if (cwdi->cwdi_cdir == olddp) {
                                rele1 = cwdi->cwdi_cdir;
                                vref(newdp);
                                cwdi->cwdi_cdir = newdp;
                        }
                        if (cwdi->cwdi_rdir == olddp) {
                                rele2 = cwdi->cwdi_rdir;
                                vref(newdp);
                                cwdi->cwdi_rdir = newdp;
                        }
                        rw_exit(&cwdi->cwdi_lock);
                        cwdfree(cwdi);
                        if (rele1 != NULL)
                                vrele(rele1);
                        if (rele2 != NULL)
                                vrele(rele2);
                        mutex_enter(&proc_lock);
                        break;
                }
                mutex_exit(&proc_lock);
        } while (retry);

        if (rootvnode == olddp) {
                vrele(rootvnode);
                vref(newdp);
                rootvnode = newdp;
        }
        vput(newdp);
}

/*
 * Start extended attributes
 */
static int
start_extattr(struct mount *mp)
{
        int error;

        error = VFS_EXTATTRCTL(mp, EXTATTR_CMD_START, NULL, 0, NULL);
        if (error)
                printf("%s: failed to start extattr: error = %d\n",
                       mp->mnt_stat.f_mntonname, error);

        return error;
}

int
mount_domount(struct lwp *l, vnode_t **vpp, struct vfsops *vfsops,
    const char *path, int flags, void *data, size_t *data_len)
{
        vnode_t *vp = *vpp;
        struct mount *mp;
        struct pathbuf *pb;
        struct nameidata nd;
        int error, error2;

        error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_MOUNT,
            KAUTH_REQ_SYSTEM_MOUNT_NEW, vp, KAUTH_ARG(flags), data);
        if (error) {
                vfs_delref(vfsops);
                return error;
        }

        /* Cannot make a non-dir a mount-point (from here anyway). */
        if (vp->v_type != VDIR) {
                vfs_delref(vfsops);
                return SET_ERROR(ENOTDIR);
        }

        if (flags & MNT_EXPORTED) {
                vfs_delref(vfsops);
                return SET_ERROR(EINVAL);
        }

        if ((mp = vfs_mountalloc(vfsops, vp)) == NULL) {
                vfs_delref(vfsops);
                return SET_ERROR(ENOMEM);
        }

        mp->mnt_stat.f_owner = kauth_cred_geteuid(l->l_cred);

        /*
         * The underlying file system may refuse the mount for
         * various reasons.  Allow the user to force it to happen.
         *
         * Set the mount level flags.
         */
        mp->mnt_flag = flags & (MNT_BASIC_FLAGS | MNT_FORCE | MNT_IGNORE);

        error = VFS_MOUNT(mp, path, data, data_len);
        mp->mnt_flag &= ~MNT_OP_FLAGS;

        if (error != 0) {
                vfs_rele(mp);
                return error;
        }

        /* Suspend new file system before taking mnt_updating. */
        do {
                error2 = vfs_suspend(mp, 0);
        } while (error2 == EINTR || error2 == ERESTART);
        KASSERT(error2 == 0 || error2 == EOPNOTSUPP);
        mutex_enter(mp->mnt_updating);

        /*
         * Validate and prepare the mount point.
         */
        error = pathbuf_copyin(path, &pb);
        if (error != 0) {
                goto err_mounted;
        }
        NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | TRYEMULROOT, pb);
        error = namei(&nd);
        pathbuf_destroy(pb);
        if (error != 0) {
                goto err_mounted;
        }
        if (nd.ni_vp != vp) {
                vput(nd.ni_vp);
                error = SET_ERROR(EINVAL);
                goto err_mounted;
        }
        if (vp->v_mountedhere != NULL) {
                vput(nd.ni_vp);
                error = SET_ERROR(EBUSY);
                goto err_mounted;
        }
        error = vinvalbuf(vp, V_SAVE, l->l_cred, l, 0, 0);
        if (error != 0) {
                vput(nd.ni_vp);
                goto err_mounted;
        }

        /*
         * Put the new filesystem on the mount list after root.
         */
        cache_purge(vp);
        mp->mnt_iflag &= ~IMNT_WANTRDWR;

        mountlist_append(mp);
        if ((mp->mnt_flag & (MNT_RDONLY | MNT_ASYNC)) == 0)
                vfs_syncer_add_to_worklist(mp);
        vp->v_mountedhere = mp;
        vput(nd.ni_vp);

        mount_checkdirs(vp);
        mutex_exit(mp->mnt_updating);
        if (error2 == 0)
                vfs_resume(mp);

        /* Hold an additional reference to the mount across VFS_START(). */
        vfs_ref(mp);
        (void) VFS_STATVFS(mp, &mp->mnt_stat);
        error = VFS_START(mp, 0);
        if (error) {
                vrele(vp);
        } else if (flags & MNT_EXTATTR) {
                if (start_extattr(mp) != 0)
                        mp->mnt_flag &= ~MNT_EXTATTR;
        }
        /* Drop reference held for VFS_START(). */
        vfs_rele(mp);
        *vpp = NULL;
        return error;

err_mounted:
        if (VFS_UNMOUNT(mp, MNT_FORCE) != 0)
                panic("Unmounting fresh file system failed");
        mutex_exit(mp->mnt_updating);
        if (error2 == 0)
                vfs_resume(mp);
        vfs_set_lowermount(mp, NULL);
        vfs_rele(mp);

        return error;
}

/*
 * Do the actual file system unmount.  File system is assumed to have
 * been locked by the caller.
 *
 * => Caller hold reference to the mount, explicitly for dounmount().
 */
int
dounmount(struct mount *mp, int flags, struct lwp *l)
{
        struct vnode *coveredvp, *vp;
        struct vnode_impl *vip;
        int error, async, used_syncer, used_extattr;
        const bool was_suspended = fstrans_is_owner(mp);

#if NVERIEXEC > 0
        error = veriexec_unmountchk(mp);
        if (error)
                return (error);
#endif /* NVERIEXEC > 0 */

        if (!was_suspended) {
                error = vfs_suspend(mp, 0);
                if (error) {
                        return error;
                }
        }

        KASSERT((mp->mnt_iflag & IMNT_GONE) == 0);

        used_syncer = (mp->mnt_iflag & IMNT_ONWORKLIST) != 0;
        used_extattr = mp->mnt_flag & MNT_EXTATTR;

        mp->mnt_iflag |= IMNT_UNMOUNT;
        mutex_enter(mp->mnt_updating);
        /*
         * Temporarily clear the MNT_ASYNC flags so that bwrite() doesn't
         * convert the sync writes to delayed writes.
         */
        async = mp->mnt_flag & MNT_ASYNC;
        mp->mnt_flag &= ~MNT_ASYNC;
        cache_purgevfs(mp);     /* remove cache entries for this file sys */
        if (used_syncer)
                vfs_syncer_remove_from_worklist(mp);
        error = 0;
        if (((mp->mnt_flag & MNT_RDONLY) == 0) && ((flags & MNT_FORCE) == 0)) {
                error = VFS_SYNC(mp, MNT_WAIT, l->l_cred);
        }
        if (error == 0 || (flags & MNT_FORCE)) {
                error = VFS_UNMOUNT(mp, flags);
        }
        if (error) {
                mp->mnt_iflag &= ~IMNT_UNMOUNT;
                mp->mnt_flag |= async;
                if ((mp->mnt_flag & (MNT_RDONLY | MNT_ASYNC)) == 0)
                        vfs_syncer_add_to_worklist(mp);
                mutex_exit(mp->mnt_updating);
                if (!was_suspended)
                        vfs_resume(mp);
                if (used_extattr) {
                        if (start_extattr(mp) != 0)
                                mp->mnt_flag &= ~MNT_EXTATTR;
                        else
                                mp->mnt_flag |= MNT_EXTATTR;
                }
                return (error);
        }
        mutex_exit(mp->mnt_updating);

        /*
         * mark filesystem as gone to prevent further umounts
         * after mnt_umounting lock is gone, this also prevents
         * vfs_busy() from succeeding.
         */
        mp->mnt_iflag |= IMNT_GONE;
        if ((coveredvp = mp->mnt_vnodecovered) != NULLVP) {
                coveredvp->v_mountedhere = NULL;
        }
        if (!was_suspended)
                vfs_resume(mp);

        mountlist_remove(mp);

        if ((vip = TAILQ_FIRST(&mp->mnt_vnodelist)) != NULL) {
                vp = VIMPL_TO_VNODE(vip);
                vprint("dangling", vp);
                panic("unmount: dangling vnode");
        }
        vfs_hooks_unmount(mp);

        vfs_set_lowermount(mp, NULL);
        vfs_rele(mp);   /* reference from mount() */
        if (coveredvp != NULLVP) {
                vrele(coveredvp);
        }
        return (0);
}

/*
 * Unmount all file systems.
 * We traverse the list in reverse order under the assumption that doing so
 * will avoid needing to worry about dependencies.
 */
bool
vfs_unmountall(struct lwp *l)
{

        printf("unmounting file systems...\n");
        return vfs_unmountall1(l, true, true);
}

static void
vfs_unmount_print(struct mount *mp, const char *pfx)
{

        aprint_verbose("%sunmounted %s on %s type %s\n", pfx,
            mp->mnt_stat.f_mntfromname, mp->mnt_stat.f_mntonname,
            mp->mnt_stat.f_fstypename);
}

/*
 * Return the mount with the highest generation less than "gen".
 */
static struct mount *
vfs_unmount_next(uint64_t gen)
{
        mount_iterator_t *iter;
        struct mount *mp, *nmp;

        nmp = NULL;

        mountlist_iterator_init(&iter);
        while ((mp = mountlist_iterator_next(iter)) != NULL) {
                if ((nmp == NULL || mp->mnt_gen > nmp->mnt_gen) &&
                    mp->mnt_gen < gen) {
                        if (nmp != NULL)
                                vfs_rele(nmp);
                        nmp = mp;
                        vfs_ref(nmp);
                }
        }
        mountlist_iterator_destroy(iter);

        return nmp;
}

bool
vfs_unmount_forceone(struct lwp *l)
{
        struct mount *mp;
        int error;

        mp = vfs_unmount_next(mountgen);
        if (mp == NULL) {
                return false;
        }

#ifdef DEBUG
        printf("forcefully unmounting %s (%s)...\n",
            mp->mnt_stat.f_mntonname, mp->mnt_stat.f_mntfromname);
#endif
        if ((error = dounmount(mp, MNT_FORCE, l)) == 0) {
                vfs_unmount_print(mp, "forcefully ");
                return true;
        } else {
                vfs_rele(mp);
        }

#ifdef DEBUG
        printf("forceful unmount of %s failed with error %d\n",
            mp->mnt_stat.f_mntonname, error);
#endif

        return false;
}

bool
vfs_unmountall1(struct lwp *l, bool force, bool verbose)
{
        struct mount *mp;
        mount_iterator_t *iter;
        bool any_error = false, progress = false;
        uint64_t gen;
        int error;

        gen = mountgen;
        for (;;) {
                mp = vfs_unmount_next(gen);
                if (mp == NULL)
                        break;
                gen = mp->mnt_gen;

#ifdef DEBUG
                printf("unmounting %p %s (%s)...\n",
                    (void *)mp, mp->mnt_stat.f_mntonname,
                    mp->mnt_stat.f_mntfromname);
#endif
                if ((error = dounmount(mp, force ? MNT_FORCE : 0, l)) == 0) {
                        vfs_unmount_print(mp, "");
                        progress = true;
                } else {
                        vfs_rele(mp);
                        if (verbose) {
                                printf("unmount of %s failed with error %d\n",
                                    mp->mnt_stat.f_mntonname, error);
                        }
                        any_error = true;
                }
        }
        if (verbose) {
                printf("unmounting done\n");
        }
        if (any_error && verbose) {
                printf("WARNING: some file systems would not unmount\n");
        }
        /* If the mountlist is empty it is time to remove swap. */
        mountlist_iterator_init(&iter);
        if (mountlist_iterator_next(iter) == NULL) {
                uvm_swap_shutdown(l);
        }
        mountlist_iterator_destroy(iter);

        return progress;
}

void
vfs_sync_all(struct lwp *l)
{
        printf("syncing disks... ");

        /* remove user processes from run queue */
        suspendsched();
        (void)spl0();

        /* avoid coming back this way again if we panic. */
        doing_shutdown = 1;

        do_sys_sync(l);

        /* Wait for sync to finish. */
        if (vfs_syncwait() != 0) {
#if defined(DDB) && defined(DEBUG_HALT_BUSY)
                Debugger();
#endif
                printf("giving up\n");
                return;
        } else
                printf("done\n");
}

/*
 * Sync and unmount file systems before shutting down.
 */
void
vfs_shutdown(void)
{
        lwp_t *l = curlwp;

        vfs_sync_all(l);

        /*
         * If we have panicked - do not make the situation potentially
         * worse by unmounting the file systems.
         */
        if (panicstr != NULL) {
                return;
        }

        /* Unmount file systems. */
        vfs_unmountall(l);
}

/*
 * Print a list of supported file system types (used by vfs_mountroot)
 */
static void
vfs_print_fstypes(void)
{
        struct vfsops *v;
        int cnt = 0;

        mutex_enter(&vfs_list_lock);
        LIST_FOREACH(v, &vfs_list, vfs_list)
                ++cnt;
        mutex_exit(&vfs_list_lock);

        if (cnt == 0) {
                printf("WARNING: No file system modules have been loaded.\n");
                return;
        }

        printf("Supported file systems:");
        mutex_enter(&vfs_list_lock);
        LIST_FOREACH(v, &vfs_list, vfs_list) {
                printf(" %s", v->vfs_name);
        }
        mutex_exit(&vfs_list_lock);
        printf("\n");
}

/*
 * Mount the root file system.  If the operator didn't specify a
 * file system to use, try all possible file systems until one
 * succeeds.
 */
int
vfs_mountroot(void)
{
        struct vfsops *v;
        int error = ENODEV;

        if (root_device == NULL)
                panic("vfs_mountroot: root device unknown");

        switch (device_class(root_device)) {
        case DV_IFNET:
                if (rootdev != NODEV)
                        panic("vfs_mountroot: rootdev set for DV_IFNET "
                            "(0x%llx -> %llu,%llu)",
                            (unsigned long long)rootdev,
                            (unsigned long long)major(rootdev),
                            (unsigned long long)minor(rootdev));
                break;

        case DV_DISK:
                if (rootdev == NODEV)
                        panic("vfs_mountroot: rootdev not set for DV_DISK");
                if (bdevvp(rootdev, &rootvp))
                        panic("vfs_mountroot: can't get vnode for rootdev");
                vn_lock(rootvp, LK_EXCLUSIVE | LK_RETRY);
                error = VOP_OPEN(rootvp, FREAD, FSCRED);
                VOP_UNLOCK(rootvp);
                if (error) {
                        printf("vfs_mountroot: can't open root device\n");
                        return (error);
                }
                break;

        case DV_VIRTUAL:
                break;

        default:
                printf("%s: inappropriate for root file system\n",
                    device_xname(root_device));
                return SET_ERROR(ENODEV);
        }

        /*
         * If user specified a root fs type, use it.  Make sure the
         * specified type exists and has a mount_root()
         */
        if (strcmp(rootfstype, ROOT_FSTYPE_ANY) != 0) {
                v = vfs_getopsbyname(rootfstype);
                error = SET_ERROR(EFTYPE);
                if (v != NULL) {
                        if (v->vfs_mountroot != NULL) {
                                error = (v->vfs_mountroot)();
                        }
                        v->vfs_refcount--;
                }
                goto done;
        }

        /*
         * Try each file system currently configured into the kernel.
         */
        mutex_enter(&vfs_list_lock);
        LIST_FOREACH(v, &vfs_list, vfs_list) {
                if (v->vfs_mountroot == NULL)
                        continue;
#ifdef DEBUG
                aprint_normal("mountroot: trying %s...\n", v->vfs_name);
#endif
                v->vfs_refcount++;
                mutex_exit(&vfs_list_lock);
                error = (*v->vfs_mountroot)();
                mutex_enter(&vfs_list_lock);
                v->vfs_refcount--;
                if (!error) {
                        aprint_normal("root file system type: %s\n",
                            v->vfs_name);
                        break;
                }
        }
        mutex_exit(&vfs_list_lock);

        if (v == NULL) {
                vfs_print_fstypes();
                printf("no file system for %s", device_xname(root_device));
                if (device_class(root_device) == DV_DISK)
                        printf(" (dev 0x%llx)", (unsigned long long)rootdev);
                printf("\n");
                error = SET_ERROR(EFTYPE);
        }

done:
        if (error && device_class(root_device) == DV_DISK) {
                vn_lock(rootvp, LK_EXCLUSIVE | LK_RETRY);
                VOP_CLOSE(rootvp, FREAD, FSCRED);
                VOP_UNLOCK(rootvp);
                vrele(rootvp);
        }
        if (error == 0) {
                mount_iterator_t *iter;
                struct mount *mp;

                mountlist_iterator_init(&iter);
                mp = mountlist_iterator_next(iter);
                KASSERT(mp != NULL);
                mountlist_iterator_destroy(iter);

                mp->mnt_flag |= MNT_ROOTFS;
                mp->mnt_op->vfs_refcount++;

                /*
                 * Get the vnode for '/'.  Set cwdi0.cwdi_cdir to
                 * reference it, and donate it the reference grabbed
                 * with VFS_ROOT().
                 */
                error = VFS_ROOT(mp, LK_NONE, &rootvnode);
                if (error)
                        panic("cannot find root vnode, error=%d", error);
                cwdi0.cwdi_cdir = rootvnode;
                cwdi0.cwdi_rdir = NULL;

                /*
                 * Now that root is mounted, we can fixup initproc's CWD
                 * info.  All other processes are kthreads, which merely
                 * share proc0's CWD info.
                 */
                initproc->p_cwdi->cwdi_cdir = rootvnode;
                vref(initproc->p_cwdi->cwdi_cdir);
                initproc->p_cwdi->cwdi_rdir = NULL;
                /*
                 * Enable loading of modules from the filesystem
                 */
                module_load_vfs_init();

        }
        return (error);
}

/*
 * mount_specific_key_create --
 *      Create a key for subsystem mount-specific data.
 */
int
mount_specific_key_create(specificdata_key_t *keyp, specificdata_dtor_t dtor)
{

        return specificdata_key_create(mount_specificdata_domain, keyp, dtor);
}

/*
 * mount_specific_key_delete --
 *      Delete a key for subsystem mount-specific data.
 */
void
mount_specific_key_delete(specificdata_key_t key)
{

        specificdata_key_delete(mount_specificdata_domain, key);
}

/*
 * mount_initspecific --
 *      Initialize a mount's specificdata container.
 */
void
mount_initspecific(struct mount *mp)
{
        int error __diagused;

        error = specificdata_init(mount_specificdata_domain,
            &mp->mnt_specdataref);
        KASSERT(error == 0);
}

/*
 * mount_finispecific --
 *      Finalize a mount's specificdata container.
 */
void
mount_finispecific(struct mount *mp)
{

        specificdata_fini(mount_specificdata_domain, &mp->mnt_specdataref);
}

/*
 * mount_getspecific --
 *      Return mount-specific data corresponding to the specified key.
 */
void *
mount_getspecific(struct mount *mp, specificdata_key_t key)
{

        return specificdata_getspecific(mount_specificdata_domain,
            &mp->mnt_specdataref, key);
}

/*
 * mount_setspecific --
 *      Set mount-specific data corresponding to the specified key.
 */
void
mount_setspecific(struct mount *mp, specificdata_key_t key, void *data)
{

        specificdata_setspecific(mount_specificdata_domain,
            &mp->mnt_specdataref, key, data);
}

/*
 * Check to see if a filesystem is mounted on a block device.
 */
int
vfs_mountedon(vnode_t *vp)
{
        vnode_t *vq;
        int error = 0;

        if (vp->v_type != VBLK)
                return SET_ERROR(ENOTBLK);
        if (spec_node_getmountedfs(vp) != NULL)
                return SET_ERROR(EBUSY);
        if (spec_node_lookup_by_dev(vp->v_type, vp->v_rdev, VDEAD_NOWAIT, &vq)
            == 0) {
                if (spec_node_getmountedfs(vq) != NULL)
                        error = SET_ERROR(EBUSY);
                vrele(vq);
        }

        return error;
}

/*
 * Check if a device pointed to by vp is mounted.
 *
 * Returns:
 *   EINVAL     if it's not a disk
 *   EBUSY      if it's a disk and mounted
 *   0          if it's a disk and not mounted
 */
int
rawdev_mounted(vnode_t *vp, vnode_t **bvpp)
{
        vnode_t *bvp;
        dev_t dev;
        int d_type;

        bvp = NULL;
        d_type = D_OTHER;

        if (iskmemvp(vp))
                return SET_ERROR(EINVAL);

        switch (vp->v_type) {
        case VCHR: {
                const struct cdevsw *cdev;

                dev = vp->v_rdev;
                cdev = cdevsw_lookup(dev);
                if (cdev != NULL) {
                        dev_t blkdev;

                        blkdev = devsw_chr2blk(dev);
                        if (blkdev != NODEV) {
                                if (vfinddev(blkdev, VBLK, &bvp) != 0) {
                                        d_type = (cdev->d_flag & D_TYPEMASK);
                                        /* XXX: what if bvp disappears? */
                                        vrele(bvp);
                                }
                        }
                }

                break;
        }

        case VBLK: {
                const struct bdevsw *bdev;

                dev = vp->v_rdev;
                bdev = bdevsw_lookup(dev);
                if (bdev != NULL)
                        d_type = (bdev->d_flag & D_TYPEMASK);

                bvp = vp;

                break;
        }

        default:
                break;
        }

        if (d_type != D_DISK)
                return SET_ERROR(EINVAL);

        if (bvpp != NULL)
                *bvpp = bvp;

        /*
         * XXX: This is bogus. We should be failing the request
         * XXX: not only if this specific slice is mounted, but
         * XXX: if it's on a disk with any other mounted slice.
         */
        if (vfs_mountedon(bvp))
                return SET_ERROR(EBUSY);

        return 0;
}

/*
 * Make a 'unique' number from a mount type name.
 */
long
makefstype(const char *type)
{
        long rv;

        for (rv = 0; *type; type++) {
                rv <<= 2;
                rv ^= *type;
        }
        return rv;
}

static struct mountlist_entry *
mountlist_alloc(enum mountlist_type type, struct mount *mp)
{
        struct mountlist_entry *me;

        me = kmem_zalloc(sizeof(*me), KM_SLEEP);
        me->me_mount = mp;
        me->me_type = type;

        return me;
}

static void
mountlist_free(struct mountlist_entry *me)
{

        kmem_free(me, sizeof(*me));
}

void
mountlist_iterator_init(mount_iterator_t **mip)
{
        struct mountlist_entry *me;

        me = mountlist_alloc(ME_MARKER, NULL);
        mutex_enter(&mountlist_lock);
        TAILQ_INSERT_HEAD(&mountlist, me, me_list);
        mutex_exit(&mountlist_lock);
        *mip = (mount_iterator_t *)me;
}

void
mountlist_iterator_destroy(mount_iterator_t *mi)
{
        struct mountlist_entry *marker = &mi->mi_entry;

        if (marker->me_mount != NULL)
                vfs_unbusy(marker->me_mount);

        mutex_enter(&mountlist_lock);
        TAILQ_REMOVE(&mountlist, marker, me_list);
        mutex_exit(&mountlist_lock);

        mountlist_free(marker);

}

/*
 * Return the next mount or NULL for this iterator.
 * Mark it busy on success.
 */
static inline struct mount *
_mountlist_iterator_next(mount_iterator_t *mi, bool wait)
{
        struct mountlist_entry *me, *marker = &mi->mi_entry;
        struct mount *mp;
        int error;

        if (marker->me_mount != NULL) {
                vfs_unbusy(marker->me_mount);
                marker->me_mount = NULL;
        }

        mutex_enter(&mountlist_lock);
        for (;;) {
                KASSERT(marker->me_type == ME_MARKER);

                me = TAILQ_NEXT(marker, me_list);
                if (me == NULL) {
                        /* End of list: keep marker and return. */
                        mutex_exit(&mountlist_lock);
                        return NULL;
                }
                TAILQ_REMOVE(&mountlist, marker, me_list);
                TAILQ_INSERT_AFTER(&mountlist, me, marker, me_list);

                /* Skip other markers. */
                if (me->me_type != ME_MOUNT)
                        continue;

                /* Take an initial reference for vfs_busy() below. */
                mp = me->me_mount;
                KASSERT(mp != NULL);
                vfs_ref(mp);
                mutex_exit(&mountlist_lock);

                /* Try to mark this mount busy and return on success. */
                if (wait)
                        error = vfs_busy(mp);
                else
                        error = vfs_trybusy(mp);
                if (error == 0) {
                        vfs_rele(mp);
                        marker->me_mount = mp;
                        return mp;
                }
                vfs_rele(mp);
                mutex_enter(&mountlist_lock);
        }
}

struct mount *
mountlist_iterator_next(mount_iterator_t *mi)
{

        return _mountlist_iterator_next(mi, true);
}

struct mount *
mountlist_iterator_trynext(mount_iterator_t *mi)
{

        return _mountlist_iterator_next(mi, false);
}

/*
 * Attach new mount to the end of the mount list.
 */
void
mountlist_append(struct mount *mp)
{
        struct mountlist_entry *me;

        me = mountlist_alloc(ME_MOUNT, mp);
        mutex_enter(&mountlist_lock);
        TAILQ_INSERT_TAIL(&mountlist, me, me_list);
        mutex_exit(&mountlist_lock);
}

/*
 * Remove mount from mount list.
 */
void
mountlist_remove(struct mount *mp)
{
        struct mountlist_entry *me;

        mutex_enter(&mountlist_lock);
        TAILQ_FOREACH(me, &mountlist, me_list)
                if (me->me_type == ME_MOUNT && me->me_mount == mp)
                        break;
        KASSERT(me != NULL);
        TAILQ_REMOVE(&mountlist, me, me_list);
        mutex_exit(&mountlist_lock);
        mountlist_free(me);
}

/*
 * Unlocked variant to traverse the mountlist.
 * To be used from DDB only.
 */
struct mount *
_mountlist_next(struct mount *mp)
{
        struct mountlist_entry *me;

        if (mp == NULL) {
                me = TAILQ_FIRST(&mountlist);
        } else {
                TAILQ_FOREACH(me, &mountlist, me_list)
                        if (me->me_type == ME_MOUNT && me->me_mount == mp)
                                break;
                if (me != NULL)
                        me = TAILQ_NEXT(me, me_list);
        }

        while (me != NULL && me->me_type != ME_MOUNT)
                me = TAILQ_NEXT(me, me_list);

        return (me ? me->me_mount : NULL);
}