/*      $NetBSD: subr_devsw.c,v 1.55 2026/01/04 03:15:58 riastradh Exp $        */

/*-
 * Copyright (c) 2001, 2002, 2007, 2008 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by MAEKAWA Masahide <gehenna@NetBSD.org>, and by Andrew Doran.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Overview
 *
 *      subr_devsw.c: registers device drivers by name and by major
 *      number, and provides wrapper methods for performing I/O and
 *      other tasks on device drivers, keying on the device number
 *      (dev_t).
 *
 *      When the system is built, the config(8) command generates
 *      static tables of device drivers built into the kernel image
 *      along with their associated methods.  These are recorded in
 *      the cdevsw0 and bdevsw0 tables.  Drivers can also be added to
 *      and removed from the system dynamically.
 *
 * Allocation
 *
 *      When the system initially boots only the statically allocated
 *      indexes (bdevsw0, cdevsw0) are used.  If these overflow due to
 *      allocation, we allocate a fixed block of memory to hold the new,
 *      expanded index.  This "fork" of the table is only ever performed
 *      once in order to guarantee that other threads may safely access
 *      the device tables:
 *
 *      o Once a thread has a "reference" to the table via an earlier
 *        open() call, we know that the entry in the table must exist
 *        and so it is safe to access it.
 *
 *      o Regardless of whether other threads see the old or new
 *        pointers, they will point to a correct device switch
 *        structure for the operation being performed.
 *
 *      XXX Currently, the wrapper methods such as cdev_read() verify
 *      that a device driver does in fact exist before calling the
 *      associated driver method.  This should be changed so that
 *      once the device is has been referenced by a vnode (opened),
 *      calling the other methods should be valid until that reference
 *      is dropped.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: subr_devsw.c,v 1.55 2026/01/04 03:15:58 riastradh Exp $");

#ifdef _KERNEL_OPT
#include "opt_dtrace.h"
#endif

#include <sys/param.h>
#include <sys/types.h>

#include <sys/atomic.h>
#include <sys/buf.h>
#include <sys/conf.h>
#include <sys/cpu.h>
#include <sys/device.h>
#include <sys/kmem.h>
#include <sys/localcount.h>
#include <sys/poll.h>
#include <sys/pserialize.h>
#include <sys/reboot.h>
#include <sys/sdt.h>
#include <sys/systm.h>
#include <sys/tty.h>
#include <sys/xcall.h>

#ifdef DEVSW_DEBUG
#define DPRINTF(x)      printf x
#else /* DEVSW_DEBUG */
#define DPRINTF(x)
#endif /* DEVSW_DEBUG */

#define MAXDEVSW        512     /* the maximum of major device number */
#define BDEVSW_SIZE     (sizeof(struct bdevsw *))
#define CDEVSW_SIZE     (sizeof(struct cdevsw *))
#define DEVSWCONV_SIZE  (sizeof(struct devsw_conv))

struct devswref {
        struct localcount       *dr_lc;
};

/* XXX bdevsw, cdevsw, max_bdevsws, and max_cdevsws should be volatile */
extern const struct bdevsw **bdevsw, *bdevsw0[];
extern const struct cdevsw **cdevsw, *cdevsw0[];
extern struct devsw_conv *devsw_conv, devsw_conv0[];
extern const int sys_bdevsws, sys_cdevsws;
extern int max_bdevsws, max_cdevsws, max_devsw_convs;

static struct devswref *cdevswref;
static struct devswref *bdevswref;
static kcondvar_t devsw_cv;

static int bdevsw_attach(const struct bdevsw *, devmajor_t *);
static int cdevsw_attach(const struct cdevsw *, devmajor_t *);
static void devsw_detach_locked(const struct bdevsw *, const struct cdevsw *);

kmutex_t device_lock;

void (*biodone_vfs)(buf_t *) = (void *)nullop;

/*
 * bdev probes
 */
SDT_PROBE_DEFINE6(sdt, bdev, open, acquire,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/,
    "int"/*unit*/,
    "device_t"/*dv*/);
SDT_PROBE_DEFINE4(sdt, bdev, open, entry,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/);
SDT_PROBE_DEFINE5(sdt, bdev, open, return,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/,
    "int"/*error*/);
SDT_PROBE_DEFINE6(sdt, bdev, open, release,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/,
    "int"/*unit*/,
    "device_t"/*dv*/);

SDT_PROBE_DEFINE4(sdt, bdev, cancel, entry,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/);
SDT_PROBE_DEFINE5(sdt, bdev, cancel, return,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/,
    "int"/*error*/);

SDT_PROBE_DEFINE4(sdt, bdev, close, entry,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/);
SDT_PROBE_DEFINE5(sdt, bdev, close, return,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/,
    "int"/*error*/);

SDT_PROBE_DEFINE3(sdt, bdev, strategy, entry,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "struct buf *"/*bp*/);
SDT_PROBE_DEFINE3(sdt, bdev, strategy, return,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "struct buf *"/*bp*/);

SDT_PROBE_DEFINE5(sdt, bdev, ioctl, entry,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "unsigned long"/*cmd*/,
    "void *"/*data*/,
    "int"/*flag*/);
SDT_PROBE_DEFINE6(sdt, bdev, ioctl, return,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "unsigned long"/*cmd*/,
    "void *"/*data*/,
    "int"/*flag*/,
    "int"/*error*/);

SDT_PROBE_DEFINE2(sdt, bdev, psize, entry,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/);
SDT_PROBE_DEFINE3(sdt, bdev, psize, return,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "int"/*psize*/);

SDT_PROBE_DEFINE4(sdt, bdev, discard, entry,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "off_t"/*pos*/,
    "off_t"/*len*/);
SDT_PROBE_DEFINE5(sdt, bdev, discard, return,
    "struct bdevsw *"/*bdevsw*/,
    "dev_t"/*dev*/,
    "off_t"/*pos*/,
    "off_t"/*len*/,
    "int"/*error*/);

/*
 * cdev probes
 */
SDT_PROBE_DEFINE6(sdt, cdev, open, acquire,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/,
    "int"/*unit*/,
    "device_t"/*dv*/);
SDT_PROBE_DEFINE4(sdt, cdev, open, entry,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/);
SDT_PROBE_DEFINE5(sdt, cdev, open, return,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/,
    "int"/*error*/);
SDT_PROBE_DEFINE6(sdt, cdev, open, release,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/,
    "int"/*unit*/,
    "device_t"/*dv*/);

SDT_PROBE_DEFINE4(sdt, cdev, cancel, entry,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/);
SDT_PROBE_DEFINE5(sdt, cdev, cancel, return,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/,
    "int"/*error*/);

SDT_PROBE_DEFINE4(sdt, cdev, close, entry,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/);
SDT_PROBE_DEFINE5(sdt, cdev, close, return,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "int"/*flag*/,
    "int"/*devtype*/,
    "int"/*error*/);

SDT_PROBE_DEFINE4(sdt, cdev, read, entry,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "struct uio *"/*uio*/,
    "int"/*flag*/);
SDT_PROBE_DEFINE5(sdt, cdev, read, return,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "struct uio *"/*uio*/,
    "int"/*flag*/,
    "int"/*error*/);

SDT_PROBE_DEFINE4(sdt, cdev, write, entry,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "struct uio *"/*uio*/,
    "int"/*flag*/);
SDT_PROBE_DEFINE5(sdt, cdev, write, return,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "struct uio *"/*uio*/,
    "int"/*flag*/,
    "int"/*error*/);

SDT_PROBE_DEFINE5(sdt, cdev, ioctl, entry,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "unsigned long"/*cmd*/,
    "void *"/*data*/,
    "int"/*flag*/);
SDT_PROBE_DEFINE6(sdt, cdev, ioctl, return,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "unsigned long"/*cmd*/,
    "void *"/*data*/,
    "int"/*flag*/,
    "int"/*error*/);

SDT_PROBE_DEFINE4(sdt, cdev, stop, entry,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "struct tty *"/*tp*/,
    "int"/*flag*/);
SDT_PROBE_DEFINE4(sdt, cdev, stop, return,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "struct tty *"/*tp*/,
    "int"/*flag*/);

SDT_PROBE_DEFINE3(sdt, cdev, poll, entry,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "int"/*events*/);
SDT_PROBE_DEFINE4(sdt, cdev, poll, return,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "int"/*events*/,
    "int"/*revents*/);

SDT_PROBE_DEFINE4(sdt, cdev, mmap, entry,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "off_t"/*off*/,
    "int"/*flag*/);
SDT_PROBE_DEFINE5(sdt, cdev, mmap, return,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "off_t"/*off*/,
    "int"/*flag*/,
    "paddr_t"/*mmapcookie*/);

SDT_PROBE_DEFINE3(sdt, cdev, kqfilter, entry,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "struct knote *"/*kn*/);
SDT_PROBE_DEFINE4(sdt, cdev, kqfilter, return,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "struct knote *"/*kn*/,
    "int"/*error*/);

SDT_PROBE_DEFINE4(sdt, cdev, discard, entry,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "off_t"/*pos*/,
    "off_t"/*len*/);
SDT_PROBE_DEFINE5(sdt, cdev, discard, return,
    "struct cdevsw *"/*cdevsw*/,
    "dev_t"/*dev*/,
    "off_t"/*pos*/,
    "off_t"/*len*/,
    "int"/*error*/);

void
devsw_init(void)
{

        KASSERT(sys_bdevsws < MAXDEVSW - 1);
        KASSERT(sys_cdevsws < MAXDEVSW - 1);
        mutex_init(&device_lock, MUTEX_DEFAULT, IPL_NONE);

        cv_init(&devsw_cv, "devsw");
}

int
devsw_attach(const char *devname,
             const struct bdevsw *bdev, devmajor_t *bmajor,
             const struct cdevsw *cdev, devmajor_t *cmajor)
{
        struct devsw_conv *conv;
        char *name;
        int error, i;

        if (devname == NULL || cdev == NULL)
                return SET_ERROR(EINVAL);

        mutex_enter(&device_lock);

        for (i = 0; i < max_devsw_convs; i++) {
                conv = &devsw_conv[i];
                if (conv->d_name == NULL || strcmp(devname, conv->d_name) != 0)
                        continue;

                if ((bdev != NULL) && (*bmajor < 0))
                        *bmajor = conv->d_bmajor;
                if (*cmajor < 0)
                        *cmajor = conv->d_cmajor;

                if (*bmajor != conv->d_bmajor || *cmajor != conv->d_cmajor) {
                        error = SET_ERROR(EINVAL);
                        goto out;
                }
                if ((*bmajor >= 0 && bdev == NULL) || *cmajor < 0) {
                        error = SET_ERROR(EINVAL);
                        goto out;
                }

                if ((*bmajor >= 0 && bdevsw[*bmajor] != NULL) ||
                    cdevsw[*cmajor] != NULL) {
                        error = SET_ERROR(EEXIST);
                        goto out;
                }
                break;
        }

        /*
         * XXX This should allocate what it needs up front so we never
         * need to flail around trying to unwind.
         */
        error = bdevsw_attach(bdev, bmajor);
        if (error != 0)
                goto out;
        error = cdevsw_attach(cdev, cmajor);
        if (error != 0) {
                devsw_detach_locked(bdev, NULL);
                goto out;
        }

        /*
         * If we already found a conv, we're done.  Otherwise, find an
         * empty slot or extend the table.
         */
        if (i < max_devsw_convs) {
                error = 0;
                goto out;
        }

        for (i = 0; i < max_devsw_convs; i++) {
                if (devsw_conv[i].d_name == NULL)
                        break;
        }
        if (i == max_devsw_convs) {
                struct devsw_conv *newptr;
                int old_convs, new_convs;

                old_convs = max_devsw_convs;
                new_convs = old_convs + 1;

                newptr = kmem_zalloc(new_convs * DEVSWCONV_SIZE, KM_NOSLEEP);
                if (newptr == NULL) {
                        devsw_detach_locked(bdev, cdev);
                        error = SET_ERROR(ENOMEM);
                        goto out;
                }
                newptr[old_convs].d_name = NULL;
                newptr[old_convs].d_bmajor = -1;
                newptr[old_convs].d_cmajor = -1;
                memcpy(newptr, devsw_conv, old_convs * DEVSWCONV_SIZE);
                if (devsw_conv != devsw_conv0)
                        kmem_free(devsw_conv, old_convs * DEVSWCONV_SIZE);
                devsw_conv = newptr;
                max_devsw_convs = new_convs;
        }

        name = kmem_strdupsize(devname, NULL, KM_NOSLEEP);
        if (name == NULL) {
                devsw_detach_locked(bdev, cdev);
                error = SET_ERROR(ENOMEM);
                goto out;
        }

        devsw_conv[i].d_name = name;
        devsw_conv[i].d_bmajor = *bmajor;
        devsw_conv[i].d_cmajor = *cmajor;
        error = 0;
out:
        mutex_exit(&device_lock);
        return error;
}

static int
bdevsw_attach(const struct bdevsw *devsw, devmajor_t *devmajor)
{
        const struct bdevsw **newbdevsw = NULL;
        struct devswref *newbdevswref = NULL;
        struct localcount *lc;
        devmajor_t bmajor;
        int i;

        KASSERT(mutex_owned(&device_lock));

        if (devsw == NULL)
                return 0;

        if (*devmajor < 0) {
                for (bmajor = sys_bdevsws; bmajor < max_bdevsws; bmajor++) {
                        if (bdevsw[bmajor] != NULL)
                                continue;
                        for (i = 0; i < max_devsw_convs; i++) {
                                if (devsw_conv[i].d_bmajor == bmajor)
                                        break;
                        }
                        if (i != max_devsw_convs)
                                continue;
                        break;
                }
                *devmajor = bmajor;
        }

        if (*devmajor >= MAXDEVSW) {
                printf("%s: block majors exhausted\n", __func__);
                return SET_ERROR(ENOMEM);
        }

        if (bdevswref == NULL) {
                newbdevswref = kmem_zalloc(MAXDEVSW * sizeof(newbdevswref[0]),
                    KM_NOSLEEP);
                if (newbdevswref == NULL)
                        return SET_ERROR(ENOMEM);
                atomic_store_release(&bdevswref, newbdevswref);
        }

        if (*devmajor >= max_bdevsws) {
                KASSERT(bdevsw == bdevsw0);
                newbdevsw = kmem_zalloc(MAXDEVSW * sizeof(newbdevsw[0]),
                    KM_NOSLEEP);
                if (newbdevsw == NULL)
                        return SET_ERROR(ENOMEM);
                memcpy(newbdevsw, bdevsw, max_bdevsws * sizeof(bdevsw[0]));
                atomic_store_release(&bdevsw, newbdevsw);
                atomic_store_release(&max_bdevsws, MAXDEVSW);
        }

        if (bdevsw[*devmajor] != NULL)
                return SET_ERROR(EEXIST);

        KASSERT(bdevswref[*devmajor].dr_lc == NULL);
        lc = kmem_zalloc(sizeof(*lc), KM_SLEEP);
        localcount_init(lc);
        bdevswref[*devmajor].dr_lc = lc;

        atomic_store_release(&bdevsw[*devmajor], devsw);

        return 0;
}

static int
cdevsw_attach(const struct cdevsw *devsw, devmajor_t *devmajor)
{
        const struct cdevsw **newcdevsw = NULL;
        struct devswref *newcdevswref = NULL;
        struct localcount *lc;
        devmajor_t cmajor;
        int i;

        KASSERT(mutex_owned(&device_lock));

        if (*devmajor < 0) {
                for (cmajor = sys_cdevsws; cmajor < max_cdevsws; cmajor++) {
                        if (cdevsw[cmajor] != NULL)
                                continue;
                        for (i = 0; i < max_devsw_convs; i++) {
                                if (devsw_conv[i].d_cmajor == cmajor)
                                        break;
                        }
                        if (i != max_devsw_convs)
                                continue;
                        break;
                }
                *devmajor = cmajor;
        }

        if (*devmajor >= MAXDEVSW) {
                printf("%s: character majors exhausted\n", __func__);
                return SET_ERROR(ENOMEM);
        }

        if (cdevswref == NULL) {
                newcdevswref = kmem_zalloc(MAXDEVSW * sizeof(newcdevswref[0]),
                    KM_NOSLEEP);
                if (newcdevswref == NULL)
                        return SET_ERROR(ENOMEM);
                atomic_store_release(&cdevswref, newcdevswref);
        }

        if (*devmajor >= max_cdevsws) {
                KASSERT(cdevsw == cdevsw0);
                newcdevsw = kmem_zalloc(MAXDEVSW * sizeof(newcdevsw[0]),
                    KM_NOSLEEP);
                if (newcdevsw == NULL)
                        return SET_ERROR(ENOMEM);
                memcpy(newcdevsw, cdevsw, max_cdevsws * sizeof(cdevsw[0]));
                atomic_store_release(&cdevsw, newcdevsw);
                atomic_store_release(&max_cdevsws, MAXDEVSW);
        }

        if (cdevsw[*devmajor] != NULL)
                return SET_ERROR(EEXIST);

        KASSERT(cdevswref[*devmajor].dr_lc == NULL);
        lc = kmem_zalloc(sizeof(*lc), KM_SLEEP);
        localcount_init(lc);
        cdevswref[*devmajor].dr_lc = lc;

        atomic_store_release(&cdevsw[*devmajor], devsw);

        return 0;
}

static void
devsw_detach_locked(const struct bdevsw *bdev, const struct cdevsw *cdev)
{
        int bi = -1, ci = -1/*XXXGCC*/, di;
        struct cfdriver *cd;
        device_t dv;

        KASSERT(mutex_owned(&device_lock));

        /*
         * If this is wired to an autoconf device, make sure the device
         * has no more instances.  No locking here because under
         * correct use of devsw_detach, none of this state can change
         * at this point.
         */
        if (cdev != NULL && (cd = cdev->d_cfdriver) != NULL) {
                for (di = 0; di < cd->cd_ndevs; di++) {
                        KASSERTMSG((dv = cd->cd_devs[di]) == NULL,
                            "detaching character device driver %s"
                            " still has attached unit %s",
                            cd->cd_name, device_xname(dv));
                }
        }
        if (bdev != NULL && (cd = bdev->d_cfdriver) != NULL) {
                for (di = 0; di < cd->cd_ndevs; di++) {
                        KASSERTMSG((dv = cd->cd_devs[di]) == NULL,
                            "detaching block device driver %s"
                            " still has attached unit %s",
                            cd->cd_name, device_xname(dv));
                }
        }

        /* Prevent new references.  */
        if (bdev != NULL) {
                for (bi = 0; bi < max_bdevsws; bi++) {
                        if (bdevsw[bi] != bdev)
                                continue;
                        atomic_store_relaxed(&bdevsw[bi], NULL);
                        break;
                }
                KASSERT(bi < max_bdevsws);
        }
        if (cdev != NULL) {
                for (ci = 0; ci < max_cdevsws; ci++) {
                        if (cdevsw[ci] != cdev)
                                continue;
                        atomic_store_relaxed(&cdevsw[ci], NULL);
                        break;
                }
                KASSERT(ci < max_cdevsws);
        }

        if (bdev == NULL && cdev == NULL) /* XXX possible? */
                return;

        /*
         * Wait for all bdevsw_lookup_acquire, cdevsw_lookup_acquire
         * calls to notice that the devsw is gone.
         *
         * XXX Despite the use of the pserialize_read_enter/exit API
         * elsewhere in this file, we use xc_barrier here instead of
         * pserialize_perform -- because devsw_init is too early for
         * pserialize_create.  Either pserialize_create should be made
         * to work earlier, or it should be nixed altogether.  Until
         * that is fixed, xc_barrier will serve the same purpose.
         */
        xc_barrier(0);

        /*
         * Wait for all references to drain.  It is the caller's
         * responsibility to ensure that at this point, there are no
         * extant open instances and all new d_open calls will fail.
         *
         * Note that localcount_drain may release and reacquire
         * device_lock.
         */
        if (bdev != NULL) {
                localcount_drain(bdevswref[bi].dr_lc,
                    &devsw_cv, &device_lock);
                localcount_fini(bdevswref[bi].dr_lc);
                kmem_free(bdevswref[bi].dr_lc, sizeof(*bdevswref[bi].dr_lc));
                bdevswref[bi].dr_lc = NULL;
        }
        if (cdev != NULL) {
                localcount_drain(cdevswref[ci].dr_lc,
                    &devsw_cv, &device_lock);
                localcount_fini(cdevswref[ci].dr_lc);
                kmem_free(cdevswref[ci].dr_lc, sizeof(*cdevswref[ci].dr_lc));
                cdevswref[ci].dr_lc = NULL;
        }
}

void
devsw_detach(const struct bdevsw *bdev, const struct cdevsw *cdev)
{

        mutex_enter(&device_lock);
        devsw_detach_locked(bdev, cdev);
        mutex_exit(&device_lock);
}

/*
 * Look up a block device by number.
 *
 * => Caller must ensure that the device is attached.
 */
const struct bdevsw *
bdevsw_lookup(dev_t dev)
{
        devmajor_t bmajor;

        if (dev == NODEV)
                return NULL;
        bmajor = major(dev);
        if (bmajor < 0 || bmajor >= atomic_load_relaxed(&max_bdevsws))
                return NULL;

        return atomic_load_consume(&bdevsw)[bmajor];
}

static const struct bdevsw *
bdevsw_lookup_acquire(dev_t dev, struct localcount **lcp)
{
        devmajor_t bmajor;
        const struct bdevsw *bdev = NULL, *const *curbdevsw;
        struct devswref *curbdevswref;
        int s;

        if (dev == NODEV)
                return NULL;
        bmajor = major(dev);
        if (bmajor < 0)
                return NULL;

        s = pserialize_read_enter();

        /*
         * max_bdevsws never goes down, so it is safe to rely on this
         * condition without any locking for the array access below.
         * Test sys_bdevsws first so we can avoid the memory barrier in
         * that case.
         */
        if (bmajor >= sys_bdevsws &&
            bmajor >= atomic_load_acquire(&max_bdevsws))
                goto out;
        curbdevsw = atomic_load_consume(&bdevsw);
        if ((bdev = atomic_load_consume(&curbdevsw[bmajor])) == NULL)
                goto out;

        curbdevswref = atomic_load_consume(&bdevswref);
        if (curbdevswref == NULL) {
                *lcp = NULL;
        } else if ((*lcp = curbdevswref[bmajor].dr_lc) != NULL) {
                localcount_acquire(*lcp);
        }
out:
        pserialize_read_exit(s);
        return bdev;
}

static void
bdevsw_release(const struct bdevsw *bdev, struct localcount *lc)
{

        if (lc == NULL)
                return;
        localcount_release(lc, &devsw_cv, &device_lock);
}

/*
 * Look up a character device by number.
 *
 * => Caller must ensure that the device is attached.
 */
const struct cdevsw *
cdevsw_lookup(dev_t dev)
{
        devmajor_t cmajor;

        if (dev == NODEV)
                return NULL;
        cmajor = major(dev);
        if (cmajor < 0 || cmajor >= atomic_load_relaxed(&max_cdevsws))
                return NULL;

        return atomic_load_consume(&cdevsw)[cmajor];
}

static const struct cdevsw *
cdevsw_lookup_acquire(dev_t dev, struct localcount **lcp)
{
        devmajor_t cmajor;
        const struct cdevsw *cdev = NULL, *const *curcdevsw;
        struct devswref *curcdevswref;
        int s;

        if (dev == NODEV)
                return NULL;
        cmajor = major(dev);
        if (cmajor < 0)
                return NULL;

        s = pserialize_read_enter();

        /*
         * max_cdevsws never goes down, so it is safe to rely on this
         * condition without any locking for the array access below.
         * Test sys_cdevsws first so we can avoid the memory barrier in
         * that case.
         */
        if (cmajor >= sys_cdevsws &&
            cmajor >= atomic_load_acquire(&max_cdevsws))
                goto out;
        curcdevsw = atomic_load_consume(&cdevsw);
        if ((cdev = atomic_load_consume(&curcdevsw[cmajor])) == NULL)
                goto out;

        curcdevswref = atomic_load_consume(&cdevswref);
        if (curcdevswref == NULL) {
                *lcp = NULL;
        } else if ((*lcp = curcdevswref[cmajor].dr_lc) != NULL) {
                localcount_acquire(*lcp);
        }
out:
        pserialize_read_exit(s);
        return cdev;
}

static void
cdevsw_release(const struct cdevsw *cdev, struct localcount *lc)
{

        if (lc == NULL)
                return;
        localcount_release(lc, &devsw_cv, &device_lock);
}

/*
 * Look up a block device by reference to its operations set.
 *
 * => Caller must ensure that the device is not detached, and therefore
 *    that the returned major is still valid when dereferenced.
 */
devmajor_t
bdevsw_lookup_major(const struct bdevsw *bdev)
{
        const struct bdevsw *const *curbdevsw;
        devmajor_t bmajor, bmax;

        bmax = atomic_load_acquire(&max_bdevsws);
        curbdevsw = atomic_load_consume(&bdevsw);
        for (bmajor = 0; bmajor < bmax; bmajor++) {
                if (atomic_load_relaxed(&curbdevsw[bmajor]) == bdev)
                        return bmajor;
        }

        return NODEVMAJOR;
}

/*
 * Look up a character device by reference to its operations set.
 *
 * => Caller must ensure that the device is not detached, and therefore
 *    that the returned major is still valid when dereferenced.
 */
devmajor_t
cdevsw_lookup_major(const struct cdevsw *cdev)
{
        const struct cdevsw *const *curcdevsw;
        devmajor_t cmajor, cmax;

        cmax = atomic_load_acquire(&max_cdevsws);
        curcdevsw = atomic_load_consume(&cdevsw);
        for (cmajor = 0; cmajor < cmax; cmajor++) {
                if (atomic_load_relaxed(&curcdevsw[cmajor]) == cdev)
                        return cmajor;
        }

        return NODEVMAJOR;
}

/*
 * Convert from block major number to name.
 *
 * => Caller must ensure that the device is not detached, and therefore
 *    that the name pointer is still valid when dereferenced.
 */
const char *
devsw_blk2name(devmajor_t bmajor)
{
        const char *name;
        devmajor_t cmajor;
        int i;

        name = NULL;
        cmajor = -1;

        mutex_enter(&device_lock);
        if (bmajor < 0 || bmajor >= max_bdevsws || bdevsw[bmajor] == NULL) {
                mutex_exit(&device_lock);
                return NULL;
        }
        for (i = 0; i < max_devsw_convs; i++) {
                if (devsw_conv[i].d_bmajor == bmajor) {
                        cmajor = devsw_conv[i].d_cmajor;
                        break;
                }
        }
        if (cmajor >= 0 && cmajor < max_cdevsws && cdevsw[cmajor] != NULL)
                name = devsw_conv[i].d_name;
        mutex_exit(&device_lock);

        return name;
}

/*
 * Convert char major number to device driver name.
 */
const char *
cdevsw_getname(devmajor_t major)
{
        const char *name;
        int i;

        name = NULL;

        if (major < 0)
                return NULL;

        mutex_enter(&device_lock);
        for (i = 0; i < max_devsw_convs; i++) {
                if (devsw_conv[i].d_cmajor == major) {
                        name = devsw_conv[i].d_name;
                        break;
                }
        }
        mutex_exit(&device_lock);
        return name;
}

/*
 * Convert block major number to device driver name.
 */
const char *
bdevsw_getname(devmajor_t major)
{
        const char *name;
        int i;

        name = NULL;

        if (major < 0)
                return NULL;

        mutex_enter(&device_lock);
        for (i = 0; i < max_devsw_convs; i++) {
                if (devsw_conv[i].d_bmajor == major) {
                        name = devsw_conv[i].d_name;
                        break;
                }
        }
        mutex_exit(&device_lock);
        return name;
}

/*
 * Convert from device name to block major number.
 *
 * => Caller must ensure that the device is not detached, and therefore
 *    that the major number is still valid when dereferenced.
 */
devmajor_t
devsw_name2blk(const char *name, char *devname, size_t devnamelen)
{
        struct devsw_conv *conv;
        devmajor_t bmajor;
        int i;

        if (name == NULL)
                return NODEVMAJOR;

        mutex_enter(&device_lock);
        for (i = 0; i < max_devsw_convs; i++) {
                size_t len;

                conv = &devsw_conv[i];
                if (conv->d_name == NULL)
                        continue;
                len = strlen(conv->d_name);
                if (strncmp(conv->d_name, name, len) != 0)
                        continue;
                if (name[len] != '\0' && !isdigit((unsigned char)name[len]))
                        continue;
                bmajor = conv->d_bmajor;
                if (bmajor < 0 || bmajor >= max_bdevsws ||
                    bdevsw[bmajor] == NULL)
                        break;
                if (devname != NULL) {
#ifdef DEVSW_DEBUG
                        if (strlen(conv->d_name) >= devnamelen)
                                printf("%s: too short buffer\n", __func__);
#endif /* DEVSW_DEBUG */
                        strncpy(devname, conv->d_name, devnamelen);
                        devname[devnamelen - 1] = '\0';
                }
                mutex_exit(&device_lock);
                return bmajor;
        }

        mutex_exit(&device_lock);
        return NODEVMAJOR;
}

/*
 * Convert from device name to char major number.
 *
 * => Caller must ensure that the device is not detached, and therefore
 *    that the major number is still valid when dereferenced.
 */
devmajor_t
devsw_name2chr(const char *name, char *devname, size_t devnamelen)
{
        struct devsw_conv *conv;
        devmajor_t cmajor;
        int i;

        if (name == NULL)
                return NODEVMAJOR;

        mutex_enter(&device_lock);
        for (i = 0; i < max_devsw_convs; i++) {
                size_t len;

                conv = &devsw_conv[i];
                if (conv->d_name == NULL)
                        continue;
                len = strlen(conv->d_name);
                if (strncmp(conv->d_name, name, len) != 0)
                        continue;
                if (name[len] != '\0' && !isdigit((unsigned char)name[len]))
                        continue;
                cmajor = conv->d_cmajor;
                if (cmajor < 0 || cmajor >= max_cdevsws ||
                    cdevsw[cmajor] == NULL)
                        break;
                if (devname != NULL) {
#ifdef DEVSW_DEBUG
                        if (strlen(conv->d_name) >= devnamelen)
                                printf("%s: too short buffer", __func__);
#endif /* DEVSW_DEBUG */
                        strncpy(devname, conv->d_name, devnamelen);
                        devname[devnamelen - 1] = '\0';
                }
                mutex_exit(&device_lock);
                return cmajor;
        }

        mutex_exit(&device_lock);
        return NODEVMAJOR;
}

/*
 * Convert from character dev_t to block dev_t.
 *
 * => Caller must ensure that the device is not detached, and therefore
 *    that the major number is still valid when dereferenced.
 */
dev_t
devsw_chr2blk(dev_t cdev)
{
        devmajor_t bmajor, cmajor;
        int i;
        dev_t rv;

        cmajor = major(cdev);
        bmajor = NODEVMAJOR;
        rv = NODEV;

        mutex_enter(&device_lock);
        if (cmajor < 0 || cmajor >= max_cdevsws || cdevsw[cmajor] == NULL) {
                mutex_exit(&device_lock);
                return NODEV;
        }
        for (i = 0; i < max_devsw_convs; i++) {
                if (devsw_conv[i].d_cmajor == cmajor) {
                        bmajor = devsw_conv[i].d_bmajor;
                        break;
                }
        }
        if (bmajor >= 0 && bmajor < max_bdevsws && bdevsw[bmajor] != NULL)
                rv = makedev(bmajor, minor(cdev));
        mutex_exit(&device_lock);

        return rv;
}

/*
 * Convert from block dev_t to character dev_t.
 *
 * => Caller must ensure that the device is not detached, and therefore
 *    that the major number is still valid when dereferenced.
 */
dev_t
devsw_blk2chr(dev_t bdev)
{
        devmajor_t bmajor, cmajor;
        int i;
        dev_t rv;

        bmajor = major(bdev);
        cmajor = NODEVMAJOR;
        rv = NODEV;

        mutex_enter(&device_lock);
        if (bmajor < 0 || bmajor >= max_bdevsws || bdevsw[bmajor] == NULL) {
                mutex_exit(&device_lock);
                return NODEV;
        }
        for (i = 0; i < max_devsw_convs; i++) {
                if (devsw_conv[i].d_bmajor == bmajor) {
                        cmajor = devsw_conv[i].d_cmajor;
                        break;
                }
        }
        if (cmajor >= 0 && cmajor < max_cdevsws && cdevsw[cmajor] != NULL)
                rv = makedev(cmajor, minor(bdev));
        mutex_exit(&device_lock);

        return rv;
}

/*
 * Device access methods.
 */

#define DEV_LOCK(d)                                             \
        if ((mpflag = (d->d_flag & D_MPSAFE)) == 0) {           \
                KERNEL_LOCK(1, NULL);                           \
        }

#define DEV_UNLOCK(d)                                           \
        if (mpflag == 0) {                                      \
                KERNEL_UNLOCK_ONE(NULL);                        \
        }

int
bdev_open(dev_t dev, int flag, int devtype, lwp_t *l)
{
        const struct bdevsw *d;
        struct localcount *lc;
        device_t dv = NULL/*XXXGCC*/;
        int unit = -1/*XXXGCC*/, rv, mpflag;

        d = bdevsw_lookup_acquire(dev, &lc);
        if (d == NULL)
                return SET_ERROR(ENXIO);

        if (d->d_devtounit) {
                /*
                 * If the device node corresponds to an autoconf device
                 * instance, acquire a reference to it so that during
                 * d_open, device_lookup is stable.
                 *
                 * XXX This should also arrange to instantiate cloning
                 * pseudo-devices if appropriate, but that requires
                 * reviewing them all to find and verify a common
                 * pattern.
                 */
                if ((unit = (*d->d_devtounit)(dev)) == -1) {
                        rv = SET_ERROR(ENXIO);
                        goto out;
                }
                if ((dv = device_lookup_acquire(d->d_cfdriver, unit)) ==
                    NULL) {
                        rv = SET_ERROR(ENXIO);
                        goto out;
                }
                SDT_PROBE6(sdt, bdev, open, acquire,
                    d, dev, flag, devtype, unit, dv);
        }

        DEV_LOCK(d);
        SDT_PROBE4(sdt, bdev, open, entry,  d, dev, flag, devtype);
        rv = (*d->d_open)(dev, flag, devtype, l);
        SDT_PROBE5(sdt, bdev, open, return,  d, dev, flag, devtype, rv);
        DEV_UNLOCK(d);

        if (d->d_devtounit) {
                SDT_PROBE6(sdt, bdev, open, release,
                    d, dev, flag, devtype, unit, dv);
                device_release(dv);
        }

out:    bdevsw_release(d, lc);

        return rv;
}

int
bdev_cancel(dev_t dev, int flag, int devtype, struct lwp *l)
{
        const struct bdevsw *d;
        int rv, mpflag;

        if ((d = bdevsw_lookup(dev)) == NULL)
                return SET_ERROR(ENXIO);
        if (d->d_cancel == NULL)
                return SET_ERROR(ENODEV);

        DEV_LOCK(d);
        SDT_PROBE4(sdt, bdev, cancel, entry,  d, dev, flag, devtype);
        rv = (*d->d_cancel)(dev, flag, devtype, l);
        SDT_PROBE5(sdt, bdev, cancel, return,  d, dev, flag, devtype, rv);
        DEV_UNLOCK(d);

        return rv;
}

int
bdev_close(dev_t dev, int flag, int devtype, lwp_t *l)
{
        const struct bdevsw *d;
        int rv, mpflag;

        if ((d = bdevsw_lookup(dev)) == NULL)
                return SET_ERROR(ENXIO);

        DEV_LOCK(d);
        SDT_PROBE4(sdt, bdev, close, entry,  d, dev, flag, devtype);
        rv = (*d->d_close)(dev, flag, devtype, l);
        SDT_PROBE5(sdt, bdev, close, return,  d, dev, flag, devtype, rv);
        DEV_UNLOCK(d);

        return rv;
}

SDT_PROVIDER_DECLARE(io);
SDT_PROBE_DEFINE1(io, kernel, , start, "struct buf *"/*bp*/);

void
bdev_strategy(struct buf *bp)
{
        const struct bdevsw *d;
        int mpflag;

        SDT_PROBE1(io, kernel, , start, bp);

        if ((d = bdevsw_lookup(bp->b_dev)) == NULL) {
                bp->b_error = SET_ERROR(ENXIO);
                bp->b_resid = bp->b_bcount;
                biodone_vfs(bp); /* biodone() iff vfs present */
                return;
        }

        DEV_LOCK(d);
        SDT_PROBE3(sdt, bdev, strategy, entry,  d, bp->b_dev, bp);
        (*d->d_strategy)(bp);
        SDT_PROBE3(sdt, bdev, strategy, return,  d, bp->b_dev, bp);
        DEV_UNLOCK(d);
}

int
bdev_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
{
        const struct bdevsw *d;
        int rv, mpflag;

        if ((d = bdevsw_lookup(dev)) == NULL)
                return SET_ERROR(ENXIO);

        DEV_LOCK(d);
        SDT_PROBE5(sdt, bdev, ioctl, entry,  d, dev, cmd, data, flag);
        rv = (*d->d_ioctl)(dev, cmd, data, flag, l);
        SDT_PROBE6(sdt, bdev, ioctl, return,  d, dev, cmd, data, flag, rv);
        DEV_UNLOCK(d);

        return rv;
}

int
bdev_dump(dev_t dev, daddr_t addr, void *data, size_t sz)
{
        const struct bdevsw *d;
        int rv;

        /*
         * Dump can be called without the device open.  Since it can
         * currently only be called with the system paused (and in a
         * potentially unstable state), we don't perform any locking.
         */
        if ((d = bdevsw_lookup(dev)) == NULL)
                return SET_ERROR(ENXIO);

        /* DEV_LOCK(d); */
        rv = (*d->d_dump)(dev, addr, data, sz);
        /* DEV_UNLOCK(d); */

        return rv;
}

int
bdev_flags(dev_t dev)
{
        const struct bdevsw *d;

        if ((d = bdevsw_lookup(dev)) == NULL)
                return 0;
        return d->d_flag & ~D_TYPEMASK;
}

int
bdev_type(dev_t dev)
{
        const struct bdevsw *d;

        if ((d = bdevsw_lookup(dev)) == NULL)
                return D_OTHER;
        return d->d_flag & D_TYPEMASK;
}

int
bdev_size(dev_t dev)
{
        const struct bdevsw *d;
        int rv, mpflag = 0;

        if ((d = bdevsw_lookup(dev)) == NULL ||
            d->d_psize == NULL)
                return -1;

        /*
         * Don't to try lock the device if we're dumping.
         * XXX: is there a better way to test this?
         */
        if ((boothowto & RB_DUMP) == 0)
                DEV_LOCK(d);
        SDT_PROBE2(sdt, bdev, psize, entry,  d, dev);
        rv = (*d->d_psize)(dev);
        SDT_PROBE3(sdt, bdev, psize, return,  d, dev, rv);
        if ((boothowto & RB_DUMP) == 0)
                DEV_UNLOCK(d);

        return rv;
}

int
bdev_discard(dev_t dev, off_t pos, off_t len)
{
        const struct bdevsw *d;
        int rv, mpflag;

        if ((d = bdevsw_lookup(dev)) == NULL)
                return SET_ERROR(ENXIO);

        DEV_LOCK(d);
        SDT_PROBE4(sdt, bdev, discard, entry,  d, dev, pos, len);
        rv = (*d->d_discard)(dev, pos, len);
        SDT_PROBE5(sdt, bdev, discard, return,  d, dev, pos, len, rv);
        DEV_UNLOCK(d);

        return rv;
}

void
bdev_detached(dev_t dev)
{
        const struct bdevsw *d;
        device_t dv;
        int unit;

        if ((d = bdevsw_lookup(dev)) == NULL)
                return;
        if (d->d_devtounit == NULL)
                return;
        if ((unit = (*d->d_devtounit)(dev)) == -1)
                return;
        if ((dv = device_lookup(d->d_cfdriver, unit)) == NULL)
                return;
        config_detach_commit(dv);
}

int
cdev_open(dev_t dev, int flag, int devtype, lwp_t *l)
{
        const struct cdevsw *d;
        struct localcount *lc;
        device_t dv = NULL/*XXXGCC*/;
        int unit = -1/*XXXGCC*/, rv, mpflag;

        d = cdevsw_lookup_acquire(dev, &lc);
        if (d == NULL)
                return SET_ERROR(ENXIO);

        if (d->d_devtounit) {
                /*
                 * If the device node corresponds to an autoconf device
                 * instance, acquire a reference to it so that during
                 * d_open, device_lookup is stable.
                 *
                 * XXX This should also arrange to instantiate cloning
                 * pseudo-devices if appropriate, but that requires
                 * reviewing them all to find and verify a common
                 * pattern.
                 */
                if ((unit = (*d->d_devtounit)(dev)) == -1) {
                        rv = SET_ERROR(ENXIO);
                        goto out;
                }
                if ((dv = device_lookup_acquire(d->d_cfdriver, unit)) ==
                    NULL) {
                        rv = SET_ERROR(ENXIO);
                        goto out;
                }
                SDT_PROBE6(sdt, cdev, open, acquire,
                    d, dev, flag, devtype, unit, dv);
        }

        DEV_LOCK(d);
        SDT_PROBE4(sdt, cdev, open, entry,  d, dev, flag, devtype);
        rv = (*d->d_open)(dev, flag, devtype, l);
        SDT_PROBE5(sdt, cdev, open, return,  d, dev, flag, devtype, rv);
        DEV_UNLOCK(d);

        if (d->d_devtounit) {
                SDT_PROBE6(sdt, cdev, open, release,
                    d, dev, flag, devtype, unit, dv);
                device_release(dv);
        }

out:    cdevsw_release(d, lc);

        return rv;
}

int
cdev_cancel(dev_t dev, int flag, int devtype, struct lwp *l)
{
        const struct cdevsw *d;
        int rv, mpflag;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return SET_ERROR(ENXIO);
        if (d->d_cancel == NULL)
                return SET_ERROR(ENODEV);

        DEV_LOCK(d);
        SDT_PROBE4(sdt, cdev, cancel, entry,  d, dev, flag, devtype);
        rv = (*d->d_cancel)(dev, flag, devtype, l);
        SDT_PROBE5(sdt, cdev, cancel, return,  d, dev, flag, devtype, rv);
        DEV_UNLOCK(d);

        return rv;
}

int
cdev_close(dev_t dev, int flag, int devtype, lwp_t *l)
{
        const struct cdevsw *d;
        int rv, mpflag;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return SET_ERROR(ENXIO);

        DEV_LOCK(d);
        SDT_PROBE4(sdt, cdev, close, entry,  d, dev, flag, devtype);
        rv = (*d->d_close)(dev, flag, devtype, l);
        SDT_PROBE5(sdt, cdev, close, return,  d, dev, flag, devtype, rv);
        DEV_UNLOCK(d);

        return rv;
}

int
cdev_read(dev_t dev, struct uio *uio, int flag)
{
        const struct cdevsw *d;
        int rv, mpflag;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return SET_ERROR(ENXIO);

        DEV_LOCK(d);
        SDT_PROBE4(sdt, cdev, read, entry,  d, dev, uio, flag);
        rv = (*d->d_read)(dev, uio, flag);
        SDT_PROBE5(sdt, cdev, read, return,  d, dev, uio, flag, rv);
        DEV_UNLOCK(d);

        return rv;
}

int
cdev_write(dev_t dev, struct uio *uio, int flag)
{
        const struct cdevsw *d;
        int rv, mpflag;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return SET_ERROR(ENXIO);

        DEV_LOCK(d);
        SDT_PROBE4(sdt, cdev, write, entry,  d, dev, uio, flag);
        rv = (*d->d_write)(dev, uio, flag);
        SDT_PROBE5(sdt, cdev, write, return,  d, dev, uio, flag, rv);
        DEV_UNLOCK(d);

        return rv;
}

int
cdev_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
{
        const struct cdevsw *d;
        int rv, mpflag;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return SET_ERROR(ENXIO);

        DEV_LOCK(d);
        SDT_PROBE5(sdt, cdev, ioctl, entry,  d, dev, cmd, data, flag);
        rv = (*d->d_ioctl)(dev, cmd, data, flag, l);
        SDT_PROBE6(sdt, cdev, ioctl, return,  d, dev, cmd, data, flag, rv);
        DEV_UNLOCK(d);

        return rv;
}

void
cdev_stop(struct tty *tp, int flag)
{
        const struct cdevsw *d;
        int mpflag;

        if ((d = cdevsw_lookup(tp->t_dev)) == NULL)
                return;

        DEV_LOCK(d);
        SDT_PROBE4(sdt, cdev, stop, entry,  d, tp->t_dev, tp, flag);
        (*d->d_stop)(tp, flag);
        SDT_PROBE4(sdt, cdev, stop, return,  d, tp->t_dev, tp, flag);
        DEV_UNLOCK(d);
}

struct tty *
cdev_tty(dev_t dev)
{
        const struct cdevsw *d;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return NULL;

        /* XXX Check if necessary. */
        if (d->d_tty == NULL)
                return NULL;

        return (*d->d_tty)(dev);
}

int
cdev_poll(dev_t dev, int flag, lwp_t *l)
{
        const struct cdevsw *d;
        int rv, mpflag;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return POLLERR;

        DEV_LOCK(d);
        SDT_PROBE3(sdt, cdev, poll, entry,  d, dev, flag);
        rv = (*d->d_poll)(dev, flag, l);
        SDT_PROBE4(sdt, cdev, poll, return,  d, dev, flag, rv);
        DEV_UNLOCK(d);

        return rv;
}

paddr_t
cdev_mmap(dev_t dev, off_t off, int flag)
{
        const struct cdevsw *d;
        paddr_t rv;
        int mpflag;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return (paddr_t)-1LL;

        DEV_LOCK(d);
        SDT_PROBE4(sdt, cdev, mmap, entry,  d, dev, off, flag);
        rv = (*d->d_mmap)(dev, off, flag);
        SDT_PROBE5(sdt, cdev, mmap, return,  d, dev, off, flag, rv);
        DEV_UNLOCK(d);

        return rv;
}

int
cdev_kqfilter(dev_t dev, struct knote *kn)
{
        const struct cdevsw *d;
        int rv, mpflag;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return SET_ERROR(ENXIO);

        DEV_LOCK(d);
        SDT_PROBE3(sdt, cdev, kqfilter, entry,  d, dev, kn);
        rv = (*d->d_kqfilter)(dev, kn);
        SDT_PROBE4(sdt, cdev, kqfilter, return,  d, dev, kn, rv);
        DEV_UNLOCK(d);

        return rv;
}

int
cdev_discard(dev_t dev, off_t pos, off_t len)
{
        const struct cdevsw *d;
        int rv, mpflag;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return SET_ERROR(ENXIO);

        DEV_LOCK(d);
        SDT_PROBE4(sdt, cdev, discard, entry,  d, dev, pos, len);
        rv = (*d->d_discard)(dev, pos, len);
        SDT_PROBE5(sdt, cdev, discard, return,  d, dev, pos, len, rv);
        DEV_UNLOCK(d);

        return rv;
}

int
cdev_flags(dev_t dev)
{
        const struct cdevsw *d;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return 0;
        return d->d_flag & ~D_TYPEMASK;
}

int
cdev_type(dev_t dev)
{
        const struct cdevsw *d;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return D_OTHER;
        return d->d_flag & D_TYPEMASK;
}

void
cdev_detached(dev_t dev)
{
        const struct cdevsw *d;
        device_t dv;
        int unit;

        if ((d = cdevsw_lookup(dev)) == NULL)
                return;
        if (d->d_devtounit == NULL)
                return;
        if ((unit = (*d->d_devtounit)(dev)) == -1)
                return;
        if ((dv = device_lookup(d->d_cfdriver, unit)) == NULL)
                return;
        config_detach_commit(dv);
}

/*
 * nommap(dev, off, prot)
 *
 *      mmap routine that always fails, for non-mmappable devices.
 */
paddr_t
nommap(dev_t dev, off_t off, int prot)
{

        return (paddr_t)-1;
}

/*
 * dev_minor_unit(dev)
 *
 *      Returns minor(dev) as an int.  Intended for use with struct
 *      bdevsw, cdevsw::d_devtounit for drivers whose /dev nodes are
 *      implemented by reference to an autoconf instance with the minor
 *      number.
 */
int
dev_minor_unit(dev_t dev)
{

        return minor(dev);
}