/*      $NetBSD: ccd.c,v 1.194 2025/12/24 16:58:15 kre Exp $    */

/*-
 * Copyright (c) 1996, 1997, 1998, 1999, 2007, 2009 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe, and by Andrew Doran.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Copyright (c) 1988 University of Utah.
 * Copyright (c) 1990, 1993
 *      The Regents of the University of California.  All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * the Systems Programming Group of the University of Utah Computer
 * Science Department.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * from: Utah $Hdr: cd.c 1.6 90/11/28$
 *
 *      @(#)cd.c        8.2 (Berkeley) 11/16/93
 */

/*
 * "Concatenated" disk driver.
 *
 * Notes on concurrency:
 *
 * => sc_dvlock serializes access to the device nodes, excluding block I/O.
 *
 * => sc_iolock serializes access to (sc_flags & CCDF_INITED), disk stats,
 *    sc_stop, sc_bufq and b_resid from master buffers.
 *
 * => a combination of CCDF_INITED, sc_inflight, and sc_iolock is used to
 *    serialize I/O and configuration changes.
 *
 * => the in-core disk label does not change while the device is open.
 *
 * On memory consumption: ccd fans out I/O requests and so needs to
 * allocate memory.  If the system is desperately low on memory, we
 * single thread I/O.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: ccd.c,v 1.194 2025/12/24 16:58:15 kre Exp $");

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/proc.h>
#include <sys/errno.h>
#include <sys/buf.h>
#include <sys/kmem.h>
#include <sys/pool.h>
#include <sys/module.h>
#include <sys/namei.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/disklabel.h>
#include <sys/device.h>
#include <sys/disk.h>
#include <sys/syslog.h>
#include <sys/fcntl.h>
#include <sys/vnode.h>
#include <sys/conf.h>
#include <sys/mutex.h>
#include <sys/queue.h>
#include <sys/kauth.h>
#include <sys/kthread.h>
#include <sys/bufq.h>
#include <sys/sysctl.h>
#include <sys/compat_stub.h>

#include <uvm/uvm_extern.h>

#include <dev/ccdvar.h>
#include <dev/dkvar.h>

#include <miscfs/specfs/specdev.h> /* for v_rdev */

#include "ioconf.h"

#if defined(CCDDEBUG) && !defined(DEBUG)
#define DEBUG
#endif

#ifdef DEBUG
#define CCDB_FOLLOW     0x01
#define CCDB_INIT       0x02
#define CCDB_IO         0x04
#define CCDB_LABEL      0x08
#define CCDB_VNODE      0x10
int ccddebug = 0x00;
#endif

#define ccdunit(x)      DISKUNIT(x)

struct ccdbuf {
        struct buf      cb_buf;         /* new I/O buf */
        struct buf      *cb_obp;        /* ptr. to original I/O buf */
        struct ccd_softc *cb_sc;        /* pointer to ccd softc */
        int             cb_comp;        /* target component */
        SIMPLEQ_ENTRY(ccdbuf) cb_q;     /* fifo of component buffers */
};

/* component buffer pool */
static pool_cache_t ccd_cache;

#define CCD_GETBUF(wait)        pool_cache_get(ccd_cache, wait)
#define CCD_PUTBUF(cbp)         pool_cache_put(ccd_cache, cbp)

#define CCDLABELDEV(dev)        \
        (MAKEDISKDEV(major((dev)), ccdunit((dev)), RAW_PART))

/* called by main() at boot time */
void    ccddetach(void);

/* called by biodone() at interrupt time */
static void     ccdiodone(struct buf *);

static void     ccdinterleave(struct ccd_softc *);
static int      ccdinit(struct ccd_softc *, char **, struct vnode **,
                    struct lwp *);
static struct ccdbuf *ccdbuffer(struct ccd_softc *, struct buf *,
                    daddr_t, void *, long, int);
static void     ccdgetdefaultlabel(struct ccd_softc *, struct disklabel *);
static void     ccdgetdisklabel(dev_t);
static void     ccdmakedisklabel(struct ccd_softc *);
static int      ccdstart(struct ccd_softc *, struct buf *, int);
static void     ccdthread(void *);

static dev_type_open(ccdopen);
static dev_type_close(ccdclose);
static dev_type_read(ccdread);
static dev_type_write(ccdwrite);
static dev_type_ioctl(ccdioctl);
static dev_type_strategy(ccdstrategy);
static dev_type_size(ccdsize);

const struct bdevsw ccd_bdevsw = {
        .d_open = ccdopen,
        .d_close = ccdclose,
        .d_strategy = ccdstrategy,
        .d_ioctl = ccdioctl,
        .d_dump = nodump,
        .d_psize = ccdsize,
        .d_discard = nodiscard,
        .d_flag = D_DISK | D_MPSAFE
};

const struct cdevsw ccd_cdevsw = {
        .d_open = ccdopen,
        .d_close = ccdclose,
        .d_read = ccdread,
        .d_write = ccdwrite,
        .d_ioctl = ccdioctl,
        .d_stop = nostop,
        .d_tty = notty,
        .d_poll = nopoll,
        .d_mmap = nommap,
        .d_kqfilter = nokqfilter,
        .d_discard = nodiscard,
        .d_flag = D_DISK | D_MPSAFE
};

static const struct dkdriver ccddkdriver = {
        .d_strategy = ccdstrategy,
        .d_minphys = minphys
}; 

#ifdef DEBUG
static  void printiinfo(struct ccdiinfo *);
#endif

static LIST_HEAD(, ccd_softc) ccds = LIST_HEAD_INITIALIZER(ccds);
static kmutex_t ccd_lock;

SYSCTL_SETUP_PROTO(sysctl_kern_ccd_setup);

static struct ccd_softc *
ccdcreate(int unit) {
        struct ccd_softc *sc = kmem_zalloc(sizeof(*sc), KM_SLEEP);

        /* Initialize per-softc structures. */
        snprintf(sc->sc_xname, sizeof(sc->sc_xname), "ccd%d", unit);
        sc->sc_unit = unit;
        mutex_init(&sc->sc_dvlock, MUTEX_DEFAULT, IPL_NONE);
        sc->sc_iolock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
        cv_init(&sc->sc_stop, "ccdstop");
        cv_init(&sc->sc_push, "ccdthr");
        disk_init(&sc->sc_dkdev, sc->sc_xname, &ccddkdriver);
        return sc;
}

static void
ccddestroy(struct ccd_softc *sc) {
        mutex_obj_free(sc->sc_iolock);
        mutex_exit(&sc->sc_dvlock);
        mutex_destroy(&sc->sc_dvlock);
        cv_destroy(&sc->sc_stop);
        cv_destroy(&sc->sc_push);
        disk_destroy(&sc->sc_dkdev);
        kmem_free(sc, sizeof(*sc));
}

static struct ccd_softc *
ccdget(int unit, int make) {
        struct ccd_softc *sc;
        if (unit < 0) {
#ifdef DIAGNOSTIC
                panic("%s: unit %d!", __func__, unit);
#endif
                return NULL;
        }
        mutex_enter(&ccd_lock);
        LIST_FOREACH(sc, &ccds, sc_link) {
                if (sc->sc_unit == unit) {
                        mutex_exit(&ccd_lock);
                        return sc;
                }
        }
        mutex_exit(&ccd_lock);
        if (!make)
                return NULL;
        if ((sc = ccdcreate(unit)) == NULL)
                return NULL;
        mutex_enter(&ccd_lock);
        LIST_INSERT_HEAD(&ccds, sc, sc_link);
        mutex_exit(&ccd_lock);
        return sc;
}

static void
ccdput(struct ccd_softc *sc) {
        mutex_enter(&ccd_lock);
        LIST_REMOVE(sc, sc_link);
        mutex_exit(&ccd_lock);
        ccddestroy(sc);
}

/*
 * Called by main() during pseudo-device attachment.  All we need
 * to do is allocate enough space for devices to be configured later.
 */
void
ccdattach(int num)
{
        mutex_init(&ccd_lock, MUTEX_DEFAULT, IPL_NONE);

        /* Initialize the component buffer pool. */
        ccd_cache = pool_cache_init(sizeof(struct ccdbuf), 0,
            0, 0, "ccdbuf", NULL, IPL_BIO, NULL, NULL, NULL);
}

void
ccddetach(void)
{
        pool_cache_destroy(ccd_cache);
        mutex_destroy(&ccd_lock);
}

static int
ccdinit(struct ccd_softc *cs, char **cpaths, struct vnode **vpp,
    struct lwp *l)
{
        struct ccdcinfo *ci = NULL;
        int ix;
        struct ccdgeom *ccg = &cs->sc_geom;
        char *tmppath;
        int error, path_alloced;
        uint64_t psize, minsize;
        unsigned secsize, maxsecsize;
        struct disk_geom *dg;

#ifdef DEBUG
        if (ccddebug & (CCDB_FOLLOW|CCDB_INIT))
                printf("%s: ccdinit\n", cs->sc_xname);
#endif

        /* Allocate space for the component info. */
        cs->sc_cinfo = kmem_alloc(cs->sc_nccdisks * sizeof(*cs->sc_cinfo),
            KM_SLEEP);
        tmppath = kmem_alloc(MAXPATHLEN, KM_SLEEP);

        cs->sc_size = 0;

        /*
         * Verify that each component piece exists and record
         * relevant information about it.
         */
        maxsecsize = 0;
        minsize = 0;
        for (ix = 0, path_alloced = 0; ix < cs->sc_nccdisks; ix++) {
                ci = &cs->sc_cinfo[ix];
                ci->ci_vp = vpp[ix];

                /*
                 * Copy in the pathname of the component.
                 */
                memset(tmppath, 0, MAXPATHLEN); /* sanity */
                error = copyinstr(cpaths[ix], tmppath,
                    MAXPATHLEN, &ci->ci_pathlen);
                if (ci->ci_pathlen == 0)
                        error = EINVAL;
                if (error) {
#ifdef DEBUG
                        if (ccddebug & (CCDB_FOLLOW|CCDB_INIT))
                                printf("%s: can't copy path, error = %d\n",
                                    cs->sc_xname, error);
#endif
                        goto out;
                }
                ci->ci_path = kmem_alloc(ci->ci_pathlen, KM_SLEEP);
                memcpy(ci->ci_path, tmppath, ci->ci_pathlen);
                path_alloced++;

                /*
                 * XXX: Cache the component's dev_t.
                 */
                ci->ci_dev = vpp[ix]->v_rdev;

                /*
                 * Get partition information for the component.
                 */
                error = getdisksize(vpp[ix], &psize, &secsize);
                if (error) {
#ifdef DEBUG
                        if (ccddebug & (CCDB_FOLLOW|CCDB_INIT))
                                 printf("%s: %s: disksize failed, error = %d\n",
                                     cs->sc_xname, ci->ci_path, error);
#endif
                        goto out;
                }

                /*
                 * Calculate the size, truncating to an interleave
                 * boundary if necessary.
                 */
                maxsecsize = secsize > maxsecsize ? secsize : maxsecsize;
                if (cs->sc_ileave > 1)
                        psize -= psize % cs->sc_ileave;

                if (psize == 0) {
#ifdef DEBUG
                        if (ccddebug & (CCDB_FOLLOW|CCDB_INIT))
                                printf("%s: %s: size == 0\n",
                                    cs->sc_xname, ci->ci_path);
#endif
                        error = ENODEV;
                        goto out;
                }

                if (minsize == 0 || psize < minsize)
                        minsize = psize;
                ci->ci_size = psize;
                cs->sc_size += psize;
        }

        /*
         * Don't allow the interleave to be smaller than
         * the biggest component sector.
         */
        if ((cs->sc_ileave > 0) &&
            (cs->sc_ileave < (maxsecsize / DEV_BSIZE))) {
#ifdef DEBUG
                if (ccddebug & (CCDB_FOLLOW|CCDB_INIT))
                        printf("%s: interleave must be at least %d\n",
                            cs->sc_xname, (maxsecsize / DEV_BSIZE));
#endif
                error = EINVAL;
                goto out;
        }

        /*
         * If uniform interleave is desired set all sizes to that of
         * the smallest component.
         */
        if (cs->sc_flags & CCDF_UNIFORM) {
                for (ci = cs->sc_cinfo;
                     ci < &cs->sc_cinfo[cs->sc_nccdisks]; ci++)
                        ci->ci_size = minsize;

                cs->sc_size = cs->sc_nccdisks * minsize;
        }

        /*
         * Construct the interleave table.
         */
        ccdinterleave(cs);

        /*
         * Create pseudo-geometry based on 1MB cylinders.  It's
         * pretty close.
         */
        ccg->ccg_secsize = DEV_BSIZE;
        ccg->ccg_ntracks = 1;
        ccg->ccg_nsectors = 1024 * (1024 / ccg->ccg_secsize);
        ccg->ccg_ncylinders = cs->sc_size / ccg->ccg_nsectors;

        dg = &cs->sc_dkdev.dk_geom;
        memset(dg, 0, sizeof(*dg));
        dg->dg_secperunit = cs->sc_size;
        dg->dg_secsize = ccg->ccg_secsize;
        dg->dg_nsectors = ccg->ccg_nsectors;
        dg->dg_ntracks = ccg->ccg_ntracks;
        dg->dg_ncylinders = ccg->ccg_ncylinders;

        if (cs->sc_ileave > 0)
                aprint_normal("%s: Interleaving %d component%s "
                    "(%d block interleave)\n", cs->sc_xname,
                    cs->sc_nccdisks, (cs->sc_nccdisks != 0 ? "s" : ""),
                    cs->sc_ileave);
        else
                aprint_normal("%s: Concatenating %d component%s\n",
                    cs->sc_xname,
                    cs->sc_nccdisks, (cs->sc_nccdisks != 0 ? "s" : ""));
        for (ix = 0; ix < cs->sc_nccdisks; ix++) {
                ci = &cs->sc_cinfo[ix];
                aprint_normal("%s: %s (%ju blocks)\n", cs->sc_xname,
                    ci->ci_path, (uintmax_t)ci->ci_size);
        }
        aprint_normal("%s: total %ju blocks\n", cs->sc_xname, cs->sc_size);

        /*
         * Create thread to handle deferred I/O.
         */
        cs->sc_zap = false;
        error = kthread_create(PRI_BIO, KTHREAD_MPSAFE, NULL, ccdthread,
            cs, &cs->sc_thread, "%s", cs->sc_xname);
        if (error) {
                printf("ccdinit: can't create thread: %d\n", error);
                goto out;
        }

        /*
         * Only now that everything is set up can we enable the device.
         */
        mutex_enter(cs->sc_iolock);
        cs->sc_flags |= CCDF_INITED;
        mutex_exit(cs->sc_iolock);
        kmem_free(tmppath, MAXPATHLEN);
        return (0);

 out:
        for (ix = 0; ix < path_alloced; ix++) {
                kmem_free(cs->sc_cinfo[ix].ci_path,
                    cs->sc_cinfo[ix].ci_pathlen);
        }
        kmem_free(cs->sc_cinfo, cs->sc_nccdisks * sizeof(struct ccdcinfo));
        kmem_free(tmppath, MAXPATHLEN);
        return (error);
}

static void
ccdinterleave(struct ccd_softc *cs)
{
        struct ccdcinfo *ci, *smallci;
        struct ccdiinfo *ii;
        daddr_t bn, lbn;
        int ix;
        u_long size;

#ifdef DEBUG
        if (ccddebug & CCDB_INIT)
                printf("ccdinterleave(%p): ileave %d\n", cs, cs->sc_ileave);
#endif
        /*
         * Allocate an interleave table.
         * Chances are this is too big, but we don't care.
         */
        size = (cs->sc_nccdisks + 1) * sizeof(struct ccdiinfo);
        cs->sc_itable = kmem_zalloc(size, KM_SLEEP);

        /*
         * Trivial case: no interleave (actually interleave of disk size).
         * Each table entry represents a single component in its entirety.
         */
        if (cs->sc_ileave == 0) {
                bn = 0;
                ii = cs->sc_itable;

                for (ix = 0; ix < cs->sc_nccdisks; ix++) {
                        /* Allocate space for ii_index. */
                        ii->ii_indexsz = sizeof(int);
                        ii->ii_index = kmem_alloc(ii->ii_indexsz, KM_SLEEP);
                        ii->ii_ndisk = 1;
                        ii->ii_startblk = bn;
                        ii->ii_startoff = 0;
                        ii->ii_index[0] = ix;
                        bn += cs->sc_cinfo[ix].ci_size;
                        ii++;
                }
                ii->ii_ndisk = 0;
#ifdef DEBUG
                if (ccddebug & CCDB_INIT)
                        printiinfo(cs->sc_itable);
#endif
                return;
        }

        /*
         * The following isn't fast or pretty; it doesn't have to be.
         */
        size = 0;
        bn = lbn = 0;
        for (ii = cs->sc_itable; ; ii++) {
                /* Allocate space for ii_index. */
                ii->ii_indexsz = sizeof(int) * cs->sc_nccdisks;
                ii->ii_index = kmem_alloc(ii->ii_indexsz, KM_SLEEP);

                /*
                 * Locate the smallest of the remaining components
                 */
                smallci = NULL;
                for (ci = cs->sc_cinfo;
                     ci < &cs->sc_cinfo[cs->sc_nccdisks]; ci++)
                        if (ci->ci_size > size &&
                            (smallci == NULL ||
                             ci->ci_size < smallci->ci_size))
                                smallci = ci;

                /*
                 * Nobody left, all done
                 */
                if (smallci == NULL) {
                        ii->ii_ndisk = 0;
                        break;
                }

                /*
                 * Record starting logical block and component offset
                 */
                ii->ii_startblk = bn / cs->sc_ileave;
                ii->ii_startoff = lbn;

                /*
                 * Determine how many disks take part in this interleave
                 * and record their indices.
                 */
                ix = 0;
                for (ci = cs->sc_cinfo;
                     ci < &cs->sc_cinfo[cs->sc_nccdisks]; ci++)
                        if (ci->ci_size >= smallci->ci_size)
                                ii->ii_index[ix++] = ci - cs->sc_cinfo;
                ii->ii_ndisk = ix;
                bn += ix * (smallci->ci_size - size);
                lbn = smallci->ci_size / cs->sc_ileave;
                size = smallci->ci_size;
        }
#ifdef DEBUG
        if (ccddebug & CCDB_INIT)
                printiinfo(cs->sc_itable);
#endif
}

/* ARGSUSED */
static int
ccdopen(dev_t dev, int flags, int fmt, struct lwp *l)
{
        int unit = ccdunit(dev);
        struct ccd_softc *cs;
        struct disklabel *lp;
        int error = 0, part, pmask;

#ifdef DEBUG
        if (ccddebug & CCDB_FOLLOW)
                printf("ccdopen(0x%"PRIx64", 0x%x)\n", dev, flags);
#endif
        if ((cs = ccdget(unit, 1)) == NULL)
                return ENXIO;

        mutex_enter(&cs->sc_dvlock);

        lp = cs->sc_dkdev.dk_label;

        part = DISKPART(dev);
        pmask = (1 << part);

        /*
         * If we're initialized, check to see if there are any other
         * open partitions.  If not, then it's safe to update
         * the in-core disklabel.  Only read the disklabel if it is
         * not already valid.
         */
        if ((cs->sc_flags & (CCDF_INITED|CCDF_VLABEL)) == CCDF_INITED &&
            cs->sc_dkdev.dk_openmask == 0)
                ccdgetdisklabel(dev);

        /* Check that the partition exists. */
        if (part != RAW_PART) {
                if (((cs->sc_flags & CCDF_INITED) == 0) ||
                    ((part >= lp->d_npartitions) ||
                     (lp->d_partitions[part].p_fstype == FS_UNUSED))) {
                        error = ENXIO;
                        goto done;
                }
        }

        /* Prevent our unit from being unconfigured while open. */
        switch (fmt) {
        case S_IFCHR:
                cs->sc_dkdev.dk_copenmask |= pmask;
                break;

        case S_IFBLK:
                cs->sc_dkdev.dk_bopenmask |= pmask;
                break;
        }
        cs->sc_dkdev.dk_openmask =
            cs->sc_dkdev.dk_copenmask | cs->sc_dkdev.dk_bopenmask;

 done:
        mutex_exit(&cs->sc_dvlock);
        return (error);
}

/* ARGSUSED */
static int
ccdclose(dev_t dev, int flags, int fmt, struct lwp *l)
{
        int unit = ccdunit(dev);
        struct ccd_softc *cs;
        int part;

#ifdef DEBUG
        if (ccddebug & CCDB_FOLLOW)
                printf("ccdclose(0x%"PRIx64", 0x%x)\n", dev, flags);
#endif

        if ((cs = ccdget(unit, 0)) == NULL)
                return ENXIO;

        mutex_enter(&cs->sc_dvlock);

        part = DISKPART(dev);

        /* ...that much closer to allowing unconfiguration... */
        switch (fmt) {
        case S_IFCHR:
                cs->sc_dkdev.dk_copenmask &= ~(1 << part);
                break;

        case S_IFBLK:
                cs->sc_dkdev.dk_bopenmask &= ~(1 << part);
                break;
        }
        cs->sc_dkdev.dk_openmask =
            cs->sc_dkdev.dk_copenmask | cs->sc_dkdev.dk_bopenmask;

        if (cs->sc_dkdev.dk_openmask == 0) {
                if ((cs->sc_flags & CCDF_KLABEL) == 0)
                        cs->sc_flags &= ~CCDF_VLABEL;
        }

        mutex_exit(&cs->sc_dvlock);
        return (0);
}

static void
ccdthread(void *cookie)
{
        int error;
        struct ccd_softc *cs;
        struct buf *bp;

        cs = cookie;

#ifdef DEBUG
        if (ccddebug & CCDB_FOLLOW)
                printf("ccdthread: hello\n");
#endif

        mutex_enter(cs->sc_iolock);
        while (__predict_true(!cs->sc_zap)) {
                bp = bufq_get(cs->sc_bufq);
                if (bp == NULL) {
                        /* Nothing to do. */
                        cv_wait(&cs->sc_push, cs->sc_iolock);
                        continue;
                }
#ifdef DEBUG
                if (ccddebug & CCDB_FOLLOW)
                        printf("ccdthread: dispatching I/O\n");
#endif
                error = ccdstart(cs, bp, PR_WAITOK);
                KASSERT(error == 0);
                mutex_enter(cs->sc_iolock);
        }
        cs->sc_thread = NULL;
        mutex_exit(cs->sc_iolock);
#ifdef DEBUG
        if (ccddebug & CCDB_FOLLOW)
                printf("ccdthread: goodbye\n");
#endif
        kthread_exit(0);
}

static void
ccdstrategy(struct buf *bp)
{
        int unit = ccdunit(bp->b_dev);
        struct ccd_softc *cs;
        if ((cs = ccdget(unit, 0)) == NULL)
                return;

        /* Must be open or reading label. */
        KASSERT(cs->sc_dkdev.dk_openmask != 0 ||
            (cs->sc_flags & CCDF_RLABEL) != 0);

        mutex_enter(cs->sc_iolock);
        /* Synchronize with device init/uninit. */
        if (__predict_false((cs->sc_flags & CCDF_INITED) == 0)) {
                mutex_exit(cs->sc_iolock);
#ifdef DEBUG
                if (ccddebug & CCDB_FOLLOW)
                        printf("ccdstrategy: unit %d: not inited\n", unit);
#endif
                bp->b_error = ENXIO;
                bp->b_resid = bp->b_bcount;
                biodone(bp);
                return;
        }

        if (ccdstart(cs, bp, PR_NOWAIT) != 0) {
                /* Defer to thread if system is low on memory. */
                bufq_put(cs->sc_bufq, bp);
                cv_broadcast(&cs->sc_push);
                mutex_exit(cs->sc_iolock);
        }
}

static int
ccdstart(struct ccd_softc *cs, struct buf *bp, int wait)
{
        daddr_t blkno;
        int wlabel;
        struct disklabel *lp;
        long bcount, rcount;
        struct ccdbuf *cbp;
        char *addr;
        daddr_t bn;
        vnode_t *vp;
        SIMPLEQ_HEAD(, ccdbuf) cbufq;

        KASSERT(mutex_owned(cs->sc_iolock));
        KASSERT(bp != NULL);

        disk_busy(&cs->sc_dkdev);

#ifdef DEBUG
        if (ccddebug & CCDB_FOLLOW)
                printf("ccdstart(%s, %p)\n", cs->sc_xname, bp);
#endif

        /* If it's a nil transfer, wake up the top half now. */
        if (bp->b_bcount == 0)
                goto done;

        lp = cs->sc_dkdev.dk_label;

        /*
         * Do bounds checking and adjust transfer.  If there's an
         * error, the bounds check will flag that for us.  Convert
         * the partition relative block number to an absolute.
         */
        blkno = bp->b_blkno;
        wlabel = cs->sc_flags & (CCDF_WLABEL|CCDF_LABELLING);
        if (DISKPART(bp->b_dev) != RAW_PART) {
                if (bounds_check_with_label(&cs->sc_dkdev, bp, wlabel) <= 0)
                        goto done;
                blkno += lp->d_partitions[DISKPART(bp->b_dev)].p_offset;
        }
        mutex_exit(cs->sc_iolock);
        bp->b_rawblkno = blkno;

        /* Allocate the component buffers. */
        SIMPLEQ_INIT(&cbufq);
        bp->b_resid = bp->b_bcount;
        bn = bp->b_rawblkno;
        addr = bp->b_data;
        for (bcount = bp->b_bcount; bcount > 0; bcount -= rcount) {
                cbp = ccdbuffer(cs, bp, bn, addr, bcount, wait);
                KASSERT(cbp != NULL || wait == PR_NOWAIT);
                if (cbp == NULL) {
                        while ((cbp = SIMPLEQ_FIRST(&cbufq)) != NULL) {
                                SIMPLEQ_REMOVE_HEAD(&cbufq, cb_q);
                                CCD_PUTBUF(cbp);
                        }
                        mutex_enter(cs->sc_iolock);
                        disk_unbusy(&cs->sc_dkdev, 0, 0);
                        return ENOMEM;
                }
                SIMPLEQ_INSERT_TAIL(&cbufq, cbp, cb_q);
                rcount = cbp->cb_buf.b_bcount;
                bn += btodb(rcount);
                addr += rcount;
        }

        /* All buffers set up, now fire off the requests. */
        while ((cbp = SIMPLEQ_FIRST(&cbufq)) != NULL) {
                SIMPLEQ_REMOVE_HEAD(&cbufq, cb_q);
                vp = cbp->cb_buf.b_vp;
                if ((cbp->cb_buf.b_flags & B_READ) == 0) {
                        mutex_enter(vp->v_interlock);
                        vp->v_numoutput++;
                        mutex_exit(vp->v_interlock);
                }
                (void)VOP_STRATEGY(vp, &cbp->cb_buf);
        }
        return 0;

 done:
        disk_unbusy(&cs->sc_dkdev, 0, 0);
        cv_broadcast(&cs->sc_stop);
        cv_broadcast(&cs->sc_push);
        mutex_exit(cs->sc_iolock);
        bp->b_resid = bp->b_bcount;
        biodone(bp);
        return 0;
}

/*
 * Build a component buffer header.
 */
static struct ccdbuf *
ccdbuffer(struct ccd_softc *cs, struct buf *bp, daddr_t bn, void *addr,
    long bcount, int wait)
{
        struct ccdcinfo *ci;
        struct ccdbuf *cbp;
        daddr_t cbn, cboff;
        u_int64_t cbc;
        int ccdisk;

#ifdef DEBUG
        if (ccddebug & CCDB_IO)
                printf("ccdbuffer(%p, %p, %" PRId64 ", %p, %ld)\n",
                       cs, bp, bn, addr, bcount);
#endif
        /*
         * Determine which component bn falls in.
         */
        cbn = bn;
        cboff = 0;

        /*
         * Serially concatenated
         */
        if (cs->sc_ileave == 0) {
                daddr_t sblk;

                sblk = 0;
                for (ccdisk = 0, ci = &cs->sc_cinfo[ccdisk];
                    cbn >= sblk + ci->ci_size;
                    ccdisk++, ci = &cs->sc_cinfo[ccdisk])
                        sblk += ci->ci_size;
                cbn -= sblk;
        }
        /*
         * Interleaved
         */
        else {
                struct ccdiinfo *ii;
                int off;

                cboff = cbn % cs->sc_ileave;
                cbn /= cs->sc_ileave;
                for (ii = cs->sc_itable; ii->ii_ndisk; ii++)
                        if (ii->ii_startblk > cbn)
                                break;
                ii--;
                off = cbn - ii->ii_startblk;
                if (ii->ii_ndisk == 1) {
                        ccdisk = ii->ii_index[0];
                        cbn = ii->ii_startoff + off;
                } else {
                        ccdisk = ii->ii_index[off % ii->ii_ndisk];
                        cbn = ii->ii_startoff + off / ii->ii_ndisk;
                }
                cbn *= cs->sc_ileave;
                ci = &cs->sc_cinfo[ccdisk];
        }

        /*
         * Fill in the component buf structure.
         */
        cbp = CCD_GETBUF(wait);
        if (cbp == NULL)
                return NULL;
        buf_init(&cbp->cb_buf);
        cbp->cb_buf.b_flags = bp->b_flags;
        cbp->cb_buf.b_oflags = bp->b_oflags;
        cbp->cb_buf.b_cflags = bp->b_cflags;
        cbp->cb_buf.b_iodone = ccdiodone;
        cbp->cb_buf.b_proc = bp->b_proc;
        cbp->cb_buf.b_dev = ci->ci_dev;
        cbp->cb_buf.b_blkno = cbn + cboff;
        cbp->cb_buf.b_data = addr;
        cbp->cb_buf.b_vp = ci->ci_vp;
        cbp->cb_buf.b_objlock = ci->ci_vp->v_interlock;
        if (cs->sc_ileave == 0)
                cbc = dbtob((u_int64_t)(ci->ci_size - cbn));
        else
                cbc = dbtob((u_int64_t)(cs->sc_ileave - cboff));
        cbp->cb_buf.b_bcount = cbc < bcount ? cbc : bcount;

        /*
         * context for ccdiodone
         */
        cbp->cb_obp = bp;
        cbp->cb_sc = cs;
        cbp->cb_comp = ccdisk;

        BIO_COPYPRIO(&cbp->cb_buf, bp);

#ifdef DEBUG
        if (ccddebug & CCDB_IO)
                printf(" dev 0x%"PRIx64"(u%lu): cbp %p bn %" PRId64 " addr %p"
                       " bcnt %d\n",
                    ci->ci_dev, (unsigned long) (ci-cs->sc_cinfo), cbp,
                    cbp->cb_buf.b_blkno, cbp->cb_buf.b_data,
                    cbp->cb_buf.b_bcount);
#endif

        return (cbp);
}

/*
 * Called at interrupt time.
 * Mark the component as done and if all components are done,
 * take a ccd interrupt.
 */
static void
ccdiodone(struct buf *vbp)
{
        struct ccdbuf *cbp = (struct ccdbuf *) vbp;
        struct buf *bp = cbp->cb_obp;
        struct ccd_softc *cs = cbp->cb_sc;
        int count;

#ifdef DEBUG
        if (ccddebug & CCDB_FOLLOW)
                printf("ccdiodone(%p)\n", cbp);
        if (ccddebug & CCDB_IO) {
                printf("ccdiodone: bp %p bcount %d resid %d\n",
                       bp, bp->b_bcount, bp->b_resid);
                printf(" dev 0x%"PRIx64"(u%d), cbp %p bn %" PRId64 " addr %p"
                       " bcnt %d\n",
                       cbp->cb_buf.b_dev, cbp->cb_comp, cbp,
                       cbp->cb_buf.b_blkno, cbp->cb_buf.b_data,
                       cbp->cb_buf.b_bcount);
        }
#endif

        if (cbp->cb_buf.b_error != 0) {
                bp->b_error = cbp->cb_buf.b_error;
                printf("%s: error %d on component %d\n",
                       cs->sc_xname, bp->b_error, cbp->cb_comp);
        }
        count = cbp->cb_buf.b_bcount;
        buf_destroy(&cbp->cb_buf);
        CCD_PUTBUF(cbp);

        /*
         * If all done, "interrupt".
         */
        mutex_enter(cs->sc_iolock);
        bp->b_resid -= count;
        if (bp->b_resid < 0)
                panic("ccdiodone: count");
        if (bp->b_resid == 0) {
                /*
                 * Request is done for better or worse, wakeup the top half.
                 */
                if (bp->b_error != 0)
                        bp->b_resid = bp->b_bcount;
                disk_unbusy(&cs->sc_dkdev, (bp->b_bcount - bp->b_resid),
                    (bp->b_flags & B_READ));
                if (!disk_isbusy(&cs->sc_dkdev)) {
                        if (bufq_peek(cs->sc_bufq) != NULL) {
                                cv_broadcast(&cs->sc_push);
                        }
                        cv_broadcast(&cs->sc_stop);
                }
                mutex_exit(cs->sc_iolock);
                biodone(bp);
        } else
                mutex_exit(cs->sc_iolock);
}

/* ARGSUSED */
static int
ccdread(dev_t dev, struct uio *uio, int flags)
{
        int unit = ccdunit(dev);
        struct ccd_softc *cs;

#ifdef DEBUG
        if (ccddebug & CCDB_FOLLOW)
                printf("ccdread(0x%"PRIx64", %p)\n", dev, uio);
#endif
        if ((cs = ccdget(unit, 0)) == NULL)
                return 0;

        /* Unlocked advisory check, ccdstrategy check is synchronous. */
        if ((cs->sc_flags & CCDF_INITED) == 0)
                return (ENXIO);

        return (physio(ccdstrategy, NULL, dev, B_READ, minphys, uio));
}

/* ARGSUSED */
static int
ccdwrite(dev_t dev, struct uio *uio, int flags)
{
        int unit = ccdunit(dev);
        struct ccd_softc *cs;

#ifdef DEBUG
        if (ccddebug & CCDB_FOLLOW)
                printf("ccdwrite(0x%"PRIx64", %p)\n", dev, uio);
#endif
        if ((cs = ccdget(unit, 0)) == NULL)
                return ENOENT;

        /* Unlocked advisory check, ccdstrategy check is synchronous. */
        if ((cs->sc_flags & CCDF_INITED) == 0)
                return (ENXIO);

        return (physio(ccdstrategy, NULL, dev, B_WRITE, minphys, uio));
}

int (*compat_ccd_ioctl_60)(dev_t, u_long, void *, int, struct lwp *,
    int (*)(dev_t, u_long, void *, int, struct lwp *)) = (void *)enosys;

static int
ccdioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
{
        int unit = ccdunit(dev);
        int i, j, lookedup = 0, error = 0;
        int part, pmask, make, hook;
        struct ccd_softc *cs;
        struct ccd_ioctl *ccio = (struct ccd_ioctl *)data;
        kauth_cred_t uc;
        char **cpp;
        struct pathbuf *pb;
        struct vnode **vpp;
#ifdef __HAVE_OLD_DISKLABEL
        struct disklabel newlabel;
#endif

        switch (cmd) {
        case CCDIOCSET:
                make = 1;
                break;
        default:
                MODULE_HOOK_CALL(ccd_ioctl_60_hook,
                                 (0, cmd, NULL, 0, NULL, NULL),
                                 enosys(), hook);
                if (hook == 0)
                        make = 1;
                else
                        make = 0;
                break;
        }

        if ((cs = ccdget(unit, make)) == NULL)
                return ENOENT;
        uc = kauth_cred_get();

        MODULE_HOOK_CALL(ccd_ioctl_60_hook,
                         (dev, cmd, data, flag, l, ccdioctl),
                         enosys(), error);
        if (error != ENOSYS)
                return error;

        /* Must be open for writes for these commands... */
        switch (cmd) {
        case CCDIOCSET:
        case CCDIOCCLR:
        case DIOCSDINFO:
        case DIOCWDINFO:
        case DIOCCACHESYNC:
        case DIOCAWEDGE:
        case DIOCDWEDGE:
        case DIOCRMWEDGES:
        case DIOCMWEDGES:
#ifdef __HAVE_OLD_DISKLABEL
        case ODIOCSDINFO:
        case ODIOCWDINFO:
#endif
        case DIOCKLABEL:
        case DIOCWLABEL:
                if ((flag & FWRITE) == 0)
                        return (EBADF);
        }

        /* Must be initialized for these... */
        switch (cmd) {
        case CCDIOCCLR:
        case DIOCGDINFO:
        case DIOCGSTRATEGY:
        case DIOCGCACHE:
        case DIOCCACHESYNC:
        case DIOCAWEDGE:
        case DIOCDWEDGE:
        case DIOCLWEDGES:
        case DIOCMWEDGES:
        case DIOCSDINFO:
        case DIOCWDINFO:
        case DIOCGPARTINFO:
        case DIOCWLABEL:
        case DIOCKLABEL:
        case DIOCGDEFLABEL:
#ifdef __HAVE_OLD_DISKLABEL
        case ODIOCGDINFO:
        case ODIOCSDINFO:
        case ODIOCWDINFO:
        case ODIOCGDEFLABEL:
#endif
                if ((cs->sc_flags & CCDF_INITED) == 0)
                        return ENXIO;
        }

        error = disk_ioctl(&cs->sc_dkdev, dev, cmd, data, flag, l);
        if (error != EPASSTHROUGH)
                return error;

        error = 0;
        switch (cmd) {
        case DIOCGSTRATEGY:
            {
                struct disk_strategy *dks = (void *)data;

                mutex_enter(cs->sc_iolock);
                if (cs->sc_bufq != NULL)
                        strlcpy(dks->dks_name,
                            bufq_getstrategyname(cs->sc_bufq),
                            sizeof(dks->dks_name));
                else
                        error = EINVAL;
                mutex_exit(cs->sc_iolock);
                dks->dks_paramlen = 0;
                break;
            }

        case DIOCWDINFO:
        case DIOCSDINFO:
#ifdef __HAVE_OLD_DISKLABEL
        case ODIOCWDINFO:
        case ODIOCSDINFO:
#endif
        {
                struct disklabel *lp;
#ifdef __HAVE_OLD_DISKLABEL
                if (cmd == ODIOCSDINFO || cmd == ODIOCWDINFO) {
                        memset(&newlabel, 0, sizeof newlabel);
                        memcpy(&newlabel, data, sizeof (struct olddisklabel));
                        lp = &newlabel;
                } else
#endif
                lp = (struct disklabel *)data;

                cs->sc_flags |= CCDF_LABELLING;

                error = setdisklabel(cs->sc_dkdev.dk_label,
                    lp, 0, cs->sc_dkdev.dk_cpulabel);
                if (error == 0) {
                        if (cmd == DIOCWDINFO
#ifdef __HAVE_OLD_DISKLABEL
                            || cmd == ODIOCWDINFO
#endif
                           )
                                error = writedisklabel(CCDLABELDEV(dev),
                                    ccdstrategy, cs->sc_dkdev.dk_label,
                                    cs->sc_dkdev.dk_cpulabel);
                }

                cs->sc_flags &= ~CCDF_LABELLING;
                break;
        }

        case DIOCKLABEL:
                if (*(int *)data != 0)
                        cs->sc_flags |= CCDF_KLABEL;
                else
                        cs->sc_flags &= ~CCDF_KLABEL;
                break;

        case DIOCWLABEL:
                if (*(int *)data != 0)
                        cs->sc_flags |= CCDF_WLABEL;
                else
                        cs->sc_flags &= ~CCDF_WLABEL;
                break;

        case DIOCGDEFLABEL:
                ccdgetdefaultlabel(cs, (struct disklabel *)data);
                break;

#ifdef __HAVE_OLD_DISKLABEL
        case ODIOCGDEFLABEL:
                ccdgetdefaultlabel(cs, &newlabel);
                if (newlabel.d_npartitions > OLDMAXPARTITIONS)
                        return ENOTTY;
                memcpy(data, &newlabel, sizeof (struct olddisklabel));
                break;
#endif
        default:
                error = ENOTTY;
                        break;
        }

        if (error != ENOTTY)
                return error;

        mutex_enter(&cs->sc_dvlock);

        error = 0;
        switch (cmd) {
        case CCDIOCSET:
                if (cs->sc_flags & CCDF_INITED) {
                        error = EBUSY;
                        goto out;
                }

                /* Validate the flags. */
                if ((ccio->ccio_flags & CCDF_USERMASK) != ccio->ccio_flags) {
                        error = EINVAL;
                        goto out;
                }

                if (ccio->ccio_ndisks > CCD_MAXNDISKS ||
                    ccio->ccio_ndisks == 0) {
                        error = EINVAL;
                        goto out;
                }

                /* Fill in some important bits. */
                cs->sc_ileave = ccio->ccio_ileave;
                cs->sc_nccdisks = ccio->ccio_ndisks;
                cs->sc_flags = ccio->ccio_flags & CCDF_USERMASK;

                /*
                 * Allocate space for and copy in the array of
                 * component pathnames and device numbers.
                 */
                cpp = kmem_alloc(ccio->ccio_ndisks * sizeof(*cpp), KM_SLEEP);
                vpp = kmem_alloc(ccio->ccio_ndisks * sizeof(*vpp), KM_SLEEP);
                error = copyin(ccio->ccio_disks, cpp,
                    ccio->ccio_ndisks * sizeof(*cpp));
                if (error) {
                        kmem_free(vpp, ccio->ccio_ndisks * sizeof(*vpp));
                        kmem_free(cpp, ccio->ccio_ndisks * sizeof(*cpp));
                        goto out;
                }

#ifdef DEBUG
                if (ccddebug & CCDB_INIT)
                        for (i = 0; i < ccio->ccio_ndisks; ++i)
                                printf("ccdioctl: component %d: %p\n",
                                    i, cpp[i]);
#endif

                for (i = 0; i < ccio->ccio_ndisks; ++i) {
#ifdef DEBUG
                        if (ccddebug & CCDB_INIT)
                                printf("ccdioctl: lookedup = %d\n", lookedup);
#endif
                        error = pathbuf_copyin(cpp[i], &pb);
                        if (error == 0) {
                                error = vn_bdev_openpath(pb, &vpp[i], l);
                                pathbuf_destroy(pb);
                        }
                        if (error != 0) {
                                for (j = 0; j < lookedup; ++j)
                                        (void)vn_close(vpp[j], FREAD|FWRITE,
                                            uc);
                                kmem_free(vpp, ccio->ccio_ndisks *
                                    sizeof(*vpp));
                                kmem_free(cpp, ccio->ccio_ndisks *
                                    sizeof(*cpp));

                                /*
                                 * No component data is allocated,
                                 * nothing is to be freed.
                                */
                                cs->sc_nccdisks = 0;
                                goto out;
                        }
                        ++lookedup;
                }

                /* Attach the disk. */
                disk_attach(&cs->sc_dkdev);
                bufq_alloc(&cs->sc_bufq, "fcfs", 0);

                /*
                 * Initialize the ccd.  Fills in the softc for us.
                 */
                if ((error = ccdinit(cs, cpp, vpp, l)) != 0) {
                        for (j = 0; j < lookedup; ++j)
                                (void)vn_close(vpp[j], FREAD|FWRITE,
                                    uc);
                        kmem_free(vpp, ccio->ccio_ndisks * sizeof(*vpp));
                        kmem_free(cpp, ccio->ccio_ndisks * sizeof(*cpp));
                        disk_detach(&cs->sc_dkdev);
                        mutex_exit(&cs->sc_dvlock);
                        bufq_free(cs->sc_bufq);
                        return error;
                }

                /* We can free the temporary variables now. */
                kmem_free(vpp, ccio->ccio_ndisks * sizeof(*vpp));
                kmem_free(cpp, ccio->ccio_ndisks * sizeof(*cpp));

                /*
                 * The ccd has been successfully initialized, so
                 * we can place it into the array.  Don't try to
                 * read the disklabel until the disk has been attached,
                 * because space for the disklabel is allocated
                 * in disk_attach();
                 */
                ccio->ccio_unit = unit;
                ccio->ccio_size = cs->sc_size;

                /* Try and read the disklabel. */
                ccdgetdisklabel(dev);
                disk_set_info(NULL, &cs->sc_dkdev, NULL);

                /* discover wedges */
                mutex_exit(&cs->sc_dvlock);
                dkwedge_discover(&cs->sc_dkdev);
                return 0;

        case CCDIOCCLR:
                /*
                 * Don't unconfigure if any other partitions are open
                 * or if both the character and block flavors of this
                 * partition are open.
                 */
                part = DISKPART(dev);
                pmask = (1 << part);
                if ((cs->sc_dkdev.dk_openmask & ~pmask) ||
                    ((cs->sc_dkdev.dk_bopenmask & pmask) &&
                    (cs->sc_dkdev.dk_copenmask & pmask))) {
                        error = EBUSY;
                        goto out;
                }

                /* Delete all of our wedges. */
                dkwedge_delall(&cs->sc_dkdev);

                /* Stop new I/O, wait for in-flight I/O to complete. */
                mutex_enter(cs->sc_iolock);
                cs->sc_flags &= ~(CCDF_INITED|CCDF_VLABEL);
                cs->sc_zap = true;
                while (disk_isbusy(&cs->sc_dkdev) ||
                    bufq_peek(cs->sc_bufq) != NULL ||
                    cs->sc_thread != NULL) {
                        cv_broadcast(&cs->sc_push);
                        (void)cv_timedwait(&cs->sc_stop, cs->sc_iolock, hz);
                }
                mutex_exit(cs->sc_iolock);

                /*
                 * Free ccd_softc information and clear entry.
                 */

                /* Close the components and free their pathnames. */
                for (i = 0; i < cs->sc_nccdisks; ++i) {
                        /*
                         * XXX: this close could potentially fail and
                         * cause Bad Things.  Maybe we need to force
                         * the close to happen?
                         */
#ifdef DEBUG
                        if (ccddebug & CCDB_VNODE)
                                vprint("CCDIOCCLR: vnode info",
                                    cs->sc_cinfo[i].ci_vp);
#endif
                        (void)vn_close(cs->sc_cinfo[i].ci_vp, FREAD|FWRITE,
                            uc);
                        kmem_free(cs->sc_cinfo[i].ci_path,
                            cs->sc_cinfo[i].ci_pathlen);
                }

                if (cs->sc_nccdisks != 0) {
                        /* Free interleave index. */
                        for (i = 0; cs->sc_itable[i].ii_ndisk; ++i) {
                                kmem_free(cs->sc_itable[i].ii_index,
                                    cs->sc_itable[i].ii_indexsz);
                        }
                        /* Free component info and interleave table. */
                        kmem_free(cs->sc_cinfo, cs->sc_nccdisks *
                            sizeof(struct ccdcinfo));
                        kmem_free(cs->sc_itable, (cs->sc_nccdisks + 1) *
                            sizeof(struct ccdiinfo));
                }

                aprint_normal("%s: detached\n", cs->sc_xname);

                /* Detach the disk. */
                disk_detach(&cs->sc_dkdev);
                bufq_free(cs->sc_bufq);

                /* also releases sc_dvlock */
                ccdput(cs);

                /* Don't break, otherwise cs is read again. */
                return 0;

        case DIOCGCACHE:
            {
                int dkcache = 0;

                /*
                 * We pass this call down to all components and report
                 * intersection of the flags returned by the components.
                 * If any errors out, we return error. CCD components
                 * can not change unless the device is unconfigured, so
                 * device feature flags will remain static. RCE/WCE can change
                 * of course, if set directly on underlying device.
                 */
                for (error = 0, i = 0; i < cs->sc_nccdisks; i++) {
                        error = VOP_IOCTL(cs->sc_cinfo[i].ci_vp, cmd, &j,
                                      flag, uc);
                        if (error)
                                break;

                        if (i == 0)
                                dkcache = j;
                        else
                                dkcache = DKCACHE_COMBINE(dkcache, j);
                }

                *((int *)data) = dkcache;
                break;
            }

        case DIOCCACHESYNC:
                /*
                 * We pass this call down to all components and report
                 * the first error we encounter.
                 */
                for (error = 0, i = 0; i < cs->sc_nccdisks; i++) {
                        j = VOP_IOCTL(cs->sc_cinfo[i].ci_vp, cmd, data,
                                      flag, uc);
                        if (j != 0 && error == 0)
                                error = j;
                }
                break;

default:
        error = ENOTTY;
                break;
        }

 out:
        mutex_exit(&cs->sc_dvlock);
        return (error);
}

static int
ccdsize(dev_t dev)
{
        struct ccd_softc *cs;
        struct disklabel *lp;
        int part, unit, omask, size;

        unit = ccdunit(dev);
        if ((cs = ccdget(unit, 0)) == NULL)
                return -1;

        if ((cs->sc_flags & CCDF_INITED) == 0)
                return (-1);

        part = DISKPART(dev);
        omask = cs->sc_dkdev.dk_openmask & (1 << part);
        lp = cs->sc_dkdev.dk_label;

        if (omask == 0 && ccdopen(dev, 0, S_IFBLK, curlwp))
                return (-1);

        if (lp->d_partitions[part].p_fstype != FS_SWAP)
                size = -1;
        else
                size = lp->d_partitions[part].p_size *
                    (lp->d_secsize / DEV_BSIZE);

        if (omask == 0 && ccdclose(dev, 0, S_IFBLK, curlwp))
                return (-1);

        return (size);
}

static void
ccdgetdefaultlabel(struct ccd_softc *cs, struct disklabel *lp)
{
        struct ccdgeom *ccg = &cs->sc_geom;

        memset(lp, 0, sizeof(*lp));

        if (cs->sc_size > UINT32_MAX)
                lp->d_secperunit = UINT32_MAX;
        else
                lp->d_secperunit = cs->sc_size;
        lp->d_secsize = ccg->ccg_secsize;
        lp->d_nsectors = ccg->ccg_nsectors;
        lp->d_ntracks = ccg->ccg_ntracks;
        lp->d_ncylinders = ccg->ccg_ncylinders;
        lp->d_secpercyl = lp->d_ntracks * lp->d_nsectors;

        strncpy(lp->d_typename, "ccd", sizeof(lp->d_typename));
        lp->d_type = DKTYPE_CCD;
        strncpy(lp->d_packname, "fictitious", sizeof(lp->d_packname));
        lp->d_rpm = 3600;
        lp->d_interleave = 1;
        lp->d_flags = 0;

        lp->d_partitions[RAW_PART].p_offset = 0;
        lp->d_partitions[RAW_PART].p_size = lp->d_secperunit;
        lp->d_partitions[RAW_PART].p_fstype = FS_UNUSED;
        lp->d_npartitions = RAW_PART + 1;

        lp->d_magic = DISKMAGIC;
        lp->d_magic2 = DISKMAGIC;
        lp->d_checksum = dkcksum(cs->sc_dkdev.dk_label);
}

/*
 * Read the disklabel from the ccd.  If one is not present, fake one
 * up.
 */
static void
ccdgetdisklabel(dev_t dev)
{
        int unit = ccdunit(dev);
        struct ccd_softc *cs;
        const char *errstring;
        struct disklabel *lp;
        struct cpu_disklabel *clp;

        if ((cs = ccdget(unit, 0)) == NULL)
                return;
        lp = cs->sc_dkdev.dk_label;
        clp = cs->sc_dkdev.dk_cpulabel;
        KASSERT(mutex_owned(&cs->sc_dvlock));

        memset(clp, 0, sizeof(*clp));

        ccdgetdefaultlabel(cs, lp);

        /*
         * Call the generic disklabel extraction routine.
         */
        cs->sc_flags |= CCDF_RLABEL;
        if ((cs->sc_flags & CCDF_NOLABEL) != 0)
                errstring = "CCDF_NOLABEL set; ignoring on-disk label";
        else
                errstring = readdisklabel(CCDLABELDEV(dev), ccdstrategy,
                    cs->sc_dkdev.dk_label, cs->sc_dkdev.dk_cpulabel);
        if (errstring)
                ccdmakedisklabel(cs);
        else {
                int i;
                struct partition *pp;

                /*
                 * Sanity check whether the found disklabel is valid.
                 *
                 * This is necessary since total size of ccd may vary
                 * when an interleave is changed even though exactly
                 * same components are used, and old disklabel may used
                 * if that is found.
                 */
                if (lp->d_secperunit < UINT32_MAX ?
                        lp->d_secperunit != cs->sc_size :
                        lp->d_secperunit > cs->sc_size)
                        printf("WARNING: %s: "
                            "total sector size in disklabel (%ju) != "
                            "the size of ccd (%ju)\n", cs->sc_xname,
                            (uintmax_t)lp->d_secperunit,
                            (uintmax_t)cs->sc_size);
                for (i = 0; i < lp->d_npartitions; i++) {
                        pp = &lp->d_partitions[i];
                        if (pp->p_offset + pp->p_size > cs->sc_size)
                                printf("WARNING: %s: end of partition `%c' "
                                    "exceeds the size of ccd (%ju)\n",
                                    cs->sc_xname, 'a' + i, (uintmax_t)cs->sc_size);
                }
        }

#ifdef DEBUG
        /* It's actually extremely common to have unlabeled ccds. */
        if (ccddebug & CCDB_LABEL)
                if (errstring != NULL)
                        printf("%s: %s\n", cs->sc_xname, errstring);
#endif

        /* In-core label now valid. */
        cs->sc_flags = (cs->sc_flags | CCDF_VLABEL) & ~CCDF_RLABEL;
}

/*
 * Take care of things one might want to take care of in the event
 * that a disklabel isn't present.
 */
static void
ccdmakedisklabel(struct ccd_softc *cs)
{
        struct disklabel *lp = cs->sc_dkdev.dk_label;

        /*
         * For historical reasons, if there's no disklabel present
         * the raw partition must be marked FS_BSDFFS.
         */
        lp->d_partitions[RAW_PART].p_fstype = FS_BSDFFS;

        strncpy(lp->d_packname, "default label", sizeof(lp->d_packname));

        lp->d_checksum = dkcksum(lp);
}

#ifdef DEBUG
static void
printiinfo(struct ccdiinfo *ii)
{
        int ix, i;

        for (ix = 0; ii->ii_ndisk; ix++, ii++) {
                printf(" itab[%d]: #dk %d sblk %" PRId64 " soff %" PRId64,
                    ix, ii->ii_ndisk, ii->ii_startblk, ii->ii_startoff);
                for (i = 0; i < ii->ii_ndisk; i++)
                        printf(" %d", ii->ii_index[i]);
                printf("\n");
        }
}
#endif

MODULE(MODULE_CLASS_DRIVER, ccd, "dk_subr,bufq_fcfs");

static int
ccd_modcmd(modcmd_t cmd, void *arg)
{
        int error = 0;
#ifdef _MODULE
        int bmajor = -1, cmajor = -1;
#endif


        switch (cmd) {
        case MODULE_CMD_INIT:
#ifdef _MODULE
                ccdattach(0);

                error = devsw_attach("ccd", &ccd_bdevsw, &bmajor,
                    &ccd_cdevsw, &cmajor);
#endif
                break;

        case MODULE_CMD_FINI:
#ifdef _MODULE
                mutex_enter(&ccd_lock);
                if (!LIST_EMPTY(&ccds)) {
                        mutex_exit(&ccd_lock);
                        error = EBUSY;
                } else {
                        mutex_exit(&ccd_lock);
                        devsw_detach(&ccd_bdevsw, &ccd_cdevsw);
                        ccddetach();
                }
#endif
                break;

        case MODULE_CMD_STAT:
                return ENOTTY;

        default:
                return ENOTTY;
        }

        return error;
}

static int
ccd_units_sysctl(SYSCTLFN_ARGS)
{
        struct sysctlnode node;
        struct ccd_softc *sc;
        int error, i, nccd, *units;
        size_t size;

        nccd = 0;
        mutex_enter(&ccd_lock);
        LIST_FOREACH(sc, &ccds, sc_link)
                nccd++;
        mutex_exit(&ccd_lock);

        if (nccd != 0) {
                size = nccd * sizeof(*units);
                units = kmem_zalloc(size, KM_SLEEP);
                i = 0;
                mutex_enter(&ccd_lock);
                LIST_FOREACH(sc, &ccds, sc_link) {
                        if (i >= nccd)
                                break;
                        units[i++] = sc->sc_unit;
                }
                mutex_exit(&ccd_lock);
        } else {
                units = NULL;
                size = 0;
        }

        node = *rnode;
        node.sysctl_data = units;
        node.sysctl_size = size;

        error = sysctl_lookup(SYSCTLFN_CALL(&node));
        if (units)
                kmem_free(units, size);
        return error;
}

static int
ccd_info_sysctl(SYSCTLFN_ARGS)
{
        struct sysctlnode node;
        struct ccddiskinfo ccd;
        struct ccd_softc *sc;
        int unit, error;

        if (newp == NULL || newlen != sizeof(int))
                return EINVAL;

        error = sysctl_copyin(l, newp, &unit, sizeof unit);
        if (error)
                return error;
        newlen = 0;
        ccd.ccd_ndisks = ~0;
        mutex_enter(&ccd_lock);
        LIST_FOREACH(sc, &ccds, sc_link) {
                if (sc->sc_unit == unit) {
                        ccd.ccd_ileave = sc->sc_ileave;
                        ccd.ccd_size = sc->sc_size;
                        ccd.ccd_ndisks = sc->sc_nccdisks;
                        ccd.ccd_flags = sc->sc_flags;
                        break;
                }
        }
        mutex_exit(&ccd_lock);

        if (ccd.ccd_ndisks == ~0)
                return ENOENT;

        node = *rnode;
        node.sysctl_data = &ccd;
        node.sysctl_size = sizeof(ccd);

        return sysctl_lookup(SYSCTLFN_CALL(&node));
}

static int
ccd_components_sysctl(SYSCTLFN_ARGS)
{
        struct sysctlnode node;
        int error, unit;
        size_t size;
        char *names, *p, *ep;
        struct ccd_softc *sc;

        if (newp == NULL || newlen != sizeof(int))
                return EINVAL;

        size = 0;
        error = sysctl_copyin(l, newp, &unit, sizeof unit);
        if (error)
                return error;
        newlen = 0;
        mutex_enter(&ccd_lock);
        LIST_FOREACH(sc, &ccds, sc_link)
                if (sc->sc_unit == unit) {
                        for (size_t i = 0; i < sc->sc_nccdisks; i++)
                                size += strlen(sc->sc_cinfo[i].ci_path) + 1;
                        break;
                }
        mutex_exit(&ccd_lock);

        if (size == 0)
                return ENOENT;
        names = kmem_zalloc(size, KM_SLEEP);
        p = names;
        ep = names + size;
        mutex_enter(&ccd_lock);
        LIST_FOREACH(sc, &ccds, sc_link)
                if (sc->sc_unit == unit) {
                        for (size_t i = 0; i < sc->sc_nccdisks; i++) {
                                char *d = sc->sc_cinfo[i].ci_path;
                                while (p < ep && (*p++ = *d++) != '\0')
                                        continue;
                        }
                        break;
                }
        mutex_exit(&ccd_lock);

        node = *rnode;
        node.sysctl_data = names;
        node.sysctl_size = ep - names;

        error = sysctl_lookup(SYSCTLFN_CALL(&node));
        kmem_free(names, size);
        return error;
}

SYSCTL_SETUP(sysctl_kern_ccd_setup, "sysctl kern.ccd subtree setup")
{
        const struct sysctlnode *node = NULL;

        sysctl_createv(clog, 0, NULL, &node,
            CTLFLAG_PERMANENT,
            CTLTYPE_NODE, "ccd",
            SYSCTL_DESCR("ConCatenated Disk state"),
            NULL, 0, NULL, 0,
            CTL_KERN, CTL_CREATE, CTL_EOL);

        if (node == NULL)
                return;

        sysctl_createv(clog, 0, &node, NULL,
            CTLFLAG_PERMANENT | CTLFLAG_READONLY,
            CTLTYPE_STRUCT, "units",
            SYSCTL_DESCR("List of ccd unit numbers"),
            ccd_units_sysctl, 0, NULL, 0,
            CTL_CREATE, CTL_EOL);
        sysctl_createv(clog, 0, &node, NULL,
            CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
            CTLTYPE_STRUCT, "info",
            SYSCTL_DESCR("Information about a CCD unit"),
            ccd_info_sysctl, 0, NULL, 0,
            CTL_CREATE, CTL_EOL);
        sysctl_createv(clog, 0, &node, NULL,
            CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
            CTLTYPE_STRUCT, "components",
            SYSCTL_DESCR("Information about CCD components"),
            ccd_components_sysctl, 0, NULL, 0,
            CTL_CREATE, CTL_EOL);
}