/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2016 The FreeBSD Foundation
 *
 * This software was developed by Edward Tomasz Napierala under sponsorship
 * from the FreeBSD Foundation.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 */
/*
 * USB Mass Storage Class Bulk-Only (BBB) Transport target.
 *
 * http://www.usb.org/developers/docs/devclass_docs/usbmassbulk_10.pdf
 *
 * This code implements the USB Mass Storage frontend driver for the CAM
 * Target Layer (ctl(4)) subsystem.
 */

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/refcount.h>
#include <sys/stdint.h>
#include <sys/sysctl.h>
#include <sys/systm.h>

#include <dev/usb/usb.h>
#include <dev/usb/usbdi.h>
#include "usbdevs.h"
#include "usb_if.h"

#include <cam/scsi/scsi_all.h>
#include <cam/scsi/scsi_da.h>
#include <cam/ctl/ctl_io.h>
#include <cam/ctl/ctl.h>
#include <cam/ctl/ctl_backend.h>
#include <cam/ctl/ctl_error.h>
#include <cam/ctl/ctl_frontend.h>
#include <cam/ctl/ctl_debug.h>
#include <cam/ctl/ctl_ha.h>
#include <cam/ctl/ctl_ioctl.h>
#include <cam/ctl/ctl_private.h>

SYSCTL_NODE(_hw_usb, OID_AUTO, cfumass, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
    "CAM Target Layer USB Mass Storage Frontend");
static int debug = 1;
SYSCTL_INT(_hw_usb_cfumass, OID_AUTO, debug, CTLFLAG_RWTUN,
    &debug, 1, "Enable debug messages");
static int max_lun = 0;
SYSCTL_INT(_hw_usb_cfumass, OID_AUTO, max_lun, CTLFLAG_RWTUN,
    &max_lun, 1, "Maximum advertised LUN number");
static int ignore_stop = 1;
SYSCTL_INT(_hw_usb_cfumass, OID_AUTO, ignore_stop, CTLFLAG_RWTUN,
    &ignore_stop, 1, "Ignore START STOP UNIT with START and LOEJ bits cleared");

/*
 * The driver uses a single, global CTL port.  It could create its ports
 * in cfumass_attach() instead, but that would make it impossible to specify
 * "port cfumass0" in ctl.conf(5), as the port generally wouldn't exist
 * at the time ctld(8) gets run.
 */
struct ctl_port cfumass_port;
bool            cfumass_port_online;
volatile u_int  cfumass_refcount;

#ifndef CFUMASS_BULK_SIZE 
#define CFUMASS_BULK_SIZE       (1U << 17)      /* bytes */
#endif

/*
 * USB transfer definitions.
 */
#define CFUMASS_T_COMMAND       0
#define CFUMASS_T_DATA_OUT      1
#define CFUMASS_T_DATA_IN       2
#define CFUMASS_T_STATUS        3
#define CFUMASS_T_MAX           4

/*
 * USB interface specific control requests.
 */
#define UR_RESET        0xff    /* Bulk-Only Mass Storage Reset */
#define UR_GET_MAX_LUN  0xfe    /* Get Max LUN */

/*
 * Command Block Wrapper.
 */
struct cfumass_cbw_t {
        uDWord  dCBWSignature;
#define CBWSIGNATURE            0x43425355 /* "USBC" */
        uDWord  dCBWTag;
        uDWord  dCBWDataTransferLength;
        uByte   bCBWFlags;
#define CBWFLAGS_OUT            0x00
#define CBWFLAGS_IN             0x80
        uByte   bCBWLUN;
        uByte   bCDBLength;
#define CBWCBLENGTH             16
        uByte   CBWCB[CBWCBLENGTH];
} __packed;

#define CFUMASS_CBW_SIZE        31
CTASSERT(sizeof(struct cfumass_cbw_t) == CFUMASS_CBW_SIZE);

/*
 * Command Status Wrapper.
 */
struct cfumass_csw_t {
        uDWord  dCSWSignature;
#define CSWSIGNATURE            0x53425355 /* "USBS" */
        uDWord  dCSWTag;
        uDWord  dCSWDataResidue;
        uByte   bCSWStatus;
#define CSWSTATUS_GOOD          0x0
#define CSWSTATUS_FAILED        0x1
#define CSWSTATUS_PHASE         0x2
} __packed;

#define CFUMASS_CSW_SIZE        13
CTASSERT(sizeof(struct cfumass_csw_t) == CFUMASS_CSW_SIZE);

struct cfumass_softc {
        device_t                sc_dev;
        struct usb_device       *sc_udev;
        struct usb_xfer         *sc_xfer[CFUMASS_T_MAX];

        struct cfumass_cbw_t *sc_cbw;
        struct cfumass_csw_t *sc_csw;

        struct mtx      sc_mtx;
        int             sc_online;
        int             sc_ctl_initid;

        /*
         * This is used to communicate between CTL callbacks
         * and USB callbacks; basically, it holds the state
         * for the current command ("the" command, since there
         * is no queueing in USB Mass Storage).
         */
        bool            sc_current_stalled;

        /*
         * The following are set upon receiving a SCSI command.
         */
        int             sc_current_tag;
        int             sc_current_transfer_length;
        int             sc_current_flags;

        /*
         * The following are set in ctl_datamove().
         */
        int             sc_current_residue;
        union ctl_io    *sc_ctl_io;

        /*
         * The following is set in cfumass_done().
         */
        int             sc_current_status;

        /*
         * Number of requests queued to CTL.
         */
        volatile u_int  sc_queued;
};

/*
 * USB interface.
 */
static device_probe_t           cfumass_probe;
static device_attach_t          cfumass_attach;
static device_detach_t          cfumass_detach;
static device_suspend_t         cfumass_suspend;
static device_resume_t          cfumass_resume;
static usb_handle_request_t     cfumass_handle_request;

static usb_callback_t           cfumass_t_command_callback;
static usb_callback_t           cfumass_t_data_callback;
static usb_callback_t           cfumass_t_status_callback;

static device_method_t cfumass_methods[] = {
        /* USB interface. */
        DEVMETHOD(usb_handle_request, cfumass_handle_request),

        /* Device interface. */
        DEVMETHOD(device_probe, cfumass_probe),
        DEVMETHOD(device_attach, cfumass_attach),
        DEVMETHOD(device_detach, cfumass_detach),
        DEVMETHOD(device_suspend, cfumass_suspend),
        DEVMETHOD(device_resume, cfumass_resume),

        DEVMETHOD_END
};

static driver_t cfumass_driver = {
        .name = "cfumass",
        .methods = cfumass_methods,
        .size = sizeof(struct cfumass_softc),
};

DRIVER_MODULE(cfumass, uhub, cfumass_driver, NULL, NULL);
MODULE_VERSION(cfumass, 0);
MODULE_DEPEND(cfumass, usb, 1, 1, 1);
MODULE_DEPEND(cfumass, usb_template, 1, 1, 1);

static struct usb_config cfumass_config[CFUMASS_T_MAX] = {
        [CFUMASS_T_COMMAND] = {
                .type = UE_BULK,
                .endpoint = UE_ADDR_ANY,
                .direction = UE_DIR_OUT,
                .bufsize = sizeof(struct cfumass_cbw_t),
                .callback = &cfumass_t_command_callback,
                .usb_mode = USB_MODE_DEVICE,
        },

        [CFUMASS_T_DATA_OUT] = {
                .type = UE_BULK,
                .endpoint = UE_ADDR_ANY,
                .direction = UE_DIR_OUT,
                .bufsize = CFUMASS_BULK_SIZE,
                .flags = {.proxy_buffer = 1, .short_xfer_ok = 1,
                    .ext_buffer = 1},
                .callback = &cfumass_t_data_callback,
                .usb_mode = USB_MODE_DEVICE,
        },

        [CFUMASS_T_DATA_IN] = {
                .type = UE_BULK,
                .endpoint = UE_ADDR_ANY,
                .direction = UE_DIR_IN,
                .bufsize = CFUMASS_BULK_SIZE,
                .flags = {.proxy_buffer = 1, .short_xfer_ok = 1,
                    .ext_buffer = 1},
                .callback = &cfumass_t_data_callback,
                .usb_mode = USB_MODE_DEVICE,
        },

        [CFUMASS_T_STATUS] = {
                .type = UE_BULK,
                .endpoint = UE_ADDR_ANY,
                .direction = UE_DIR_IN,
                .bufsize = sizeof(struct cfumass_csw_t),
                .flags = {.short_xfer_ok = 1},
                .callback = &cfumass_t_status_callback,
                .usb_mode = USB_MODE_DEVICE,
        },
};

/*
 * CTL frontend interface.
 */
static int      cfumass_init(void);
static int      cfumass_shutdown(void);
static void     cfumass_online(void *arg);
static void     cfumass_offline(void *arg);
static void     cfumass_datamove(union ctl_io *io);
static void     cfumass_done(union ctl_io *io);

static struct ctl_frontend cfumass_frontend = {
        .name = "umass",
        .init = cfumass_init,
        .shutdown = cfumass_shutdown,
};
CTL_FRONTEND_DECLARE(ctlcfumass, cfumass_frontend);

#define CFUMASS_DEBUG(S, X, ...)                                        \
        do {                                                            \
                if (debug > 1) {                                        \
                        device_printf(S->sc_dev, "%s: " X "\n",         \
                            __func__, ## __VA_ARGS__);                  \
                }                                                       \
        } while (0)

#define CFUMASS_WARN(S, X, ...)                                         \
        do {                                                            \
                if (debug > 0) {                                        \
                        device_printf(S->sc_dev, "WARNING: %s: " X "\n",\
                            __func__, ## __VA_ARGS__);                  \
                }                                                       \
        } while (0)

#define CFUMASS_LOCK(X)         mtx_lock(&X->sc_mtx)
#define CFUMASS_UNLOCK(X)       mtx_unlock(&X->sc_mtx)

static void     cfumass_transfer_start(struct cfumass_softc *sc,
                    uint8_t xfer_index);
static void     cfumass_terminate(struct cfumass_softc *sc);

static int
cfumass_probe(device_t dev)
{
        struct usb_attach_arg *uaa;
        struct usb_interface_descriptor *id;

        uaa = device_get_ivars(dev);

        if (uaa->usb_mode != USB_MODE_DEVICE)
                return (ENXIO);

        /*
         * Check for a compliant device.
         */
        id = usbd_get_interface_descriptor(uaa->iface);
        if ((id == NULL) ||
            (id->bInterfaceClass != UICLASS_MASS) ||
            (id->bInterfaceSubClass != UISUBCLASS_SCSI) ||
            (id->bInterfaceProtocol != UIPROTO_MASS_BBB)) {
                return (ENXIO);
        }

        return (BUS_PROBE_GENERIC);
}

static int
cfumass_attach(device_t dev)
{
        struct cfumass_softc *sc;
        struct usb_attach_arg *uaa;
        int error;

        sc = device_get_softc(dev);
        uaa = device_get_ivars(dev);

        sc->sc_dev = dev;
        sc->sc_udev = uaa->device;

        CFUMASS_DEBUG(sc, "go");

        usbd_set_power_mode(uaa->device, USB_POWER_MODE_SAVE);
        device_set_usb_desc(dev);

        mtx_init(&sc->sc_mtx, "cfumass", NULL, MTX_DEF);
        refcount_acquire(&cfumass_refcount);

        error = usbd_transfer_setup(uaa->device,
            &uaa->info.bIfaceIndex, sc->sc_xfer, cfumass_config,
            CFUMASS_T_MAX, sc, &sc->sc_mtx);
        if (error != 0) {
                CFUMASS_WARN(sc, "usbd_transfer_setup() failed: %s",
                    usbd_errstr(error));
                refcount_release(&cfumass_refcount);
                return (ENXIO);
        }

        sc->sc_cbw =
            usbd_xfer_get_frame_buffer(sc->sc_xfer[CFUMASS_T_COMMAND], 0);
        sc->sc_csw =
            usbd_xfer_get_frame_buffer(sc->sc_xfer[CFUMASS_T_STATUS], 0);

        sc->sc_ctl_initid = ctl_add_initiator(&cfumass_port, -1, 0, NULL);
        if (sc->sc_ctl_initid < 0) {
                CFUMASS_WARN(sc, "ctl_add_initiator() failed with error %d",
                    sc->sc_ctl_initid);
                usbd_transfer_unsetup(sc->sc_xfer, CFUMASS_T_MAX);
                refcount_release(&cfumass_refcount);
                return (ENXIO);
        }

        refcount_init(&sc->sc_queued, 0);

        CFUMASS_LOCK(sc);
        cfumass_transfer_start(sc, CFUMASS_T_COMMAND);
        CFUMASS_UNLOCK(sc);

        return (0);
}

static int
cfumass_detach(device_t dev)
{
        struct cfumass_softc *sc;
        int error;

        sc = device_get_softc(dev);

        CFUMASS_DEBUG(sc, "go");

        CFUMASS_LOCK(sc);
        cfumass_terminate(sc);
        CFUMASS_UNLOCK(sc);
        usbd_transfer_unsetup(sc->sc_xfer, CFUMASS_T_MAX);

        if (sc->sc_ctl_initid != -1) {
                error = ctl_remove_initiator(&cfumass_port, sc->sc_ctl_initid);
                if (error != 0) {
                        CFUMASS_WARN(sc, "ctl_remove_initiator() failed "
                            "with error %d", error);
                }
                sc->sc_ctl_initid = -1;
        }

        mtx_destroy(&sc->sc_mtx);
        refcount_release(&cfumass_refcount);

        return (0);
}

static int
cfumass_suspend(device_t dev)
{
        struct cfumass_softc *sc;

        sc = device_get_softc(dev);
        CFUMASS_DEBUG(sc, "go");

        return (0);
}

static int
cfumass_resume(device_t dev)
{
        struct cfumass_softc *sc;

        sc = device_get_softc(dev);
        CFUMASS_DEBUG(sc, "go");

        return (0);
}

static void
cfumass_transfer_start(struct cfumass_softc *sc, uint8_t xfer_index)
{

        usbd_transfer_start(sc->sc_xfer[xfer_index]);
}

static void
cfumass_transfer_stop_and_drain(struct cfumass_softc *sc, uint8_t xfer_index)
{

        usbd_transfer_stop(sc->sc_xfer[xfer_index]);
        CFUMASS_UNLOCK(sc);
        usbd_transfer_drain(sc->sc_xfer[xfer_index]);
        CFUMASS_LOCK(sc);
}

static void
cfumass_terminate(struct cfumass_softc *sc)
{
        int last;

        for (;;) {
                cfumass_transfer_stop_and_drain(sc, CFUMASS_T_COMMAND);
                cfumass_transfer_stop_and_drain(sc, CFUMASS_T_DATA_IN);
                cfumass_transfer_stop_and_drain(sc, CFUMASS_T_DATA_OUT);

                if (sc->sc_ctl_io != NULL) {
                        CFUMASS_DEBUG(sc, "terminating CTL transfer");
                        ctl_set_data_phase_error(&sc->sc_ctl_io->scsiio);
                        ctl_datamove_done(sc->sc_ctl_io, false);
                        sc->sc_ctl_io = NULL;
                }

                cfumass_transfer_stop_and_drain(sc, CFUMASS_T_STATUS);

                refcount_acquire(&sc->sc_queued);
                last = refcount_release(&sc->sc_queued);
                if (last != 0)
                        break;

                CFUMASS_DEBUG(sc, "%d CTL tasks pending", sc->sc_queued);
                msleep(__DEVOLATILE(void *, &sc->sc_queued), &sc->sc_mtx,
                    0, "cfumass_reset", hz / 100);
        }
}

static int
cfumass_handle_request(device_t dev,
    const void *preq, void **pptr, uint16_t *plen,
    uint16_t offset, uint8_t *pstate)
{
        static uint8_t max_lun_tmp;
        struct cfumass_softc *sc;
        const struct usb_device_request *req;
        uint8_t is_complete;

        sc = device_get_softc(dev);
        req = preq;
        is_complete = *pstate;

        CFUMASS_DEBUG(sc, "go");

        if (is_complete)
                return (ENXIO);

        if ((req->bmRequestType == UT_WRITE_CLASS_INTERFACE) &&
            (req->bRequest == UR_RESET)) {
                CFUMASS_WARN(sc, "received Bulk-Only Mass Storage Reset");
                *plen = 0;

                CFUMASS_LOCK(sc);
                cfumass_terminate(sc);
                cfumass_transfer_start(sc, CFUMASS_T_COMMAND);
                CFUMASS_UNLOCK(sc);

                CFUMASS_DEBUG(sc, "Bulk-Only Mass Storage Reset done");
                return (0);
        }

        if ((req->bmRequestType == UT_READ_CLASS_INTERFACE) &&
            (req->bRequest == UR_GET_MAX_LUN)) {
                CFUMASS_DEBUG(sc, "received Get Max LUN");
                if (offset == 0) {
                        *plen = 1;
                        /*
                         * The protocol doesn't support LUN numbers higher
                         * than 15.  Also, some initiators (namely Windows XP
                         * SP3 Version 2002) can't properly query the number
                         * of LUNs, resulting in inaccessible "fake" ones - thus
                         * the default limit of one LUN.
                         */
                        if (max_lun < 0 || max_lun > 15) {
                                CFUMASS_WARN(sc,
                                    "invalid hw.usb.cfumass.max_lun, must be "
                                    "between 0 and 15; defaulting to 0");
                                max_lun_tmp = 0;
                        } else {
                                max_lun_tmp = max_lun;
                        }
                        *pptr = &max_lun_tmp;
                } else {
                        *plen = 0;
                }
                return (0);
        }

        return (ENXIO);
}

static int
cfumass_quirk(struct cfumass_softc *sc, unsigned char *cdb, int cdb_len)
{
        struct scsi_start_stop_unit *sssu;

        switch (cdb[0]) {
        case START_STOP_UNIT:
                /*
                 * Some initiators - eg OSX, Darwin Kernel Version 15.6.0,
                 * root:xnu-3248.60.11~2/RELEASE_X86_64 - attempt to stop
                 * the unit on eject, but fail to start it when it's plugged
                 * back.  Just ignore the command.
                 */

                if (cdb_len < sizeof(*sssu)) {
                        CFUMASS_DEBUG(sc, "received START STOP UNIT with "
                            "bCDBLength %d, should be %zd",
                            cdb_len, sizeof(*sssu));
                        break;
                }

                sssu = (struct scsi_start_stop_unit *)cdb;
                if ((sssu->how & SSS_PC_MASK) != 0)
                        break;

                if ((sssu->how & SSS_START) != 0)
                        break;

                if ((sssu->how & SSS_LOEJ) != 0)
                        break;
                
                if (ignore_stop == 0) {
                        break;
                } else if (ignore_stop == 1) {
                        CFUMASS_WARN(sc, "ignoring START STOP UNIT request");
                } else {
                        CFUMASS_DEBUG(sc, "ignoring START STOP UNIT request");
                }

                sc->sc_current_status = 0;
                cfumass_transfer_start(sc, CFUMASS_T_STATUS);

                return (1);
        default:
                break;
        }

        return (0);
}

static void
cfumass_t_command_callback(struct usb_xfer *xfer, usb_error_t usb_error)
{
        struct cfumass_softc *sc;
        uint32_t signature;
        union ctl_io *io;
        int error = 0;

        sc = usbd_xfer_softc(xfer);

        KASSERT(sc->sc_ctl_io == NULL,
            ("sc_ctl_io is %p, should be NULL", sc->sc_ctl_io));

        switch (USB_GET_STATE(xfer)) {
        case USB_ST_TRANSFERRED:
                CFUMASS_DEBUG(sc, "USB_ST_TRANSFERRED");

                signature = UGETDW(sc->sc_cbw->dCBWSignature);
                if (signature != CBWSIGNATURE) {
                        CFUMASS_WARN(sc, "wrong dCBWSignature 0x%08x, "
                            "should be 0x%08x", signature, CBWSIGNATURE);
                        break;
                }

                if (sc->sc_cbw->bCDBLength <= 0 ||
                    sc->sc_cbw->bCDBLength > sizeof(sc->sc_cbw->CBWCB)) {
                        CFUMASS_WARN(sc, "invalid bCDBLength %d, should be <= %zd",
                            sc->sc_cbw->bCDBLength, sizeof(sc->sc_cbw->CBWCB));
                        break;
                }

                sc->sc_current_stalled = false;
                sc->sc_current_status = 0;
                sc->sc_current_tag = UGETDW(sc->sc_cbw->dCBWTag);
                sc->sc_current_transfer_length =
                    UGETDW(sc->sc_cbw->dCBWDataTransferLength);
                sc->sc_current_flags = sc->sc_cbw->bCBWFlags;

                /*
                 * Make sure to report proper residue if the datamove wasn't
                 * required, or wasn't called due to SCSI error.
                 */
                sc->sc_current_residue = sc->sc_current_transfer_length;

                if (cfumass_quirk(sc,
                    sc->sc_cbw->CBWCB, sc->sc_cbw->bCDBLength) != 0)
                        break;

                if (!cfumass_port_online) {
                        CFUMASS_DEBUG(sc, "cfumass port is offline; stalling");
                        usbd_xfer_set_stall(xfer);
                        break;
                }

                /*
                 * Those CTL functions cannot be called with mutex held.
                 */
                CFUMASS_UNLOCK(sc);
                io = ctl_alloc_io(cfumass_port.ctl_pool_ref);
                ctl_zero_io(io);
                io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = sc;
                io->io_hdr.io_type = CTL_IO_SCSI;
                io->io_hdr.nexus.initid = sc->sc_ctl_initid;
                io->io_hdr.nexus.targ_port = cfumass_port.targ_port;
                io->io_hdr.nexus.targ_lun = ctl_decode_lun(sc->sc_cbw->bCBWLUN);
                io->scsiio.tag_num = UGETDW(sc->sc_cbw->dCBWTag);
                io->scsiio.tag_type = CTL_TAG_UNTAGGED;
                io->scsiio.cdb_len = sc->sc_cbw->bCDBLength;
                memcpy(io->scsiio.cdb, sc->sc_cbw->CBWCB, sc->sc_cbw->bCDBLength);
                refcount_acquire(&sc->sc_queued);
                error = ctl_queue(io);
                if (error != CTL_RETVAL_COMPLETE) {
                        CFUMASS_WARN(sc,
                            "ctl_queue() failed; error %d; stalling", error);
                        ctl_free_io(io);
                        refcount_release(&sc->sc_queued);
                        CFUMASS_LOCK(sc);
                        usbd_xfer_set_stall(xfer);
                        break;
                }

                CFUMASS_LOCK(sc);
                break;

        case USB_ST_SETUP:
tr_setup:
                CFUMASS_DEBUG(sc, "USB_ST_SETUP");

                usbd_xfer_set_frame_len(xfer, 0, sizeof(*sc->sc_cbw));
                usbd_transfer_submit(xfer);
                break;

        default:
                if (usb_error == USB_ERR_CANCELLED) {
                        CFUMASS_DEBUG(sc, "USB_ERR_CANCELLED");
                        break;
                }

                CFUMASS_DEBUG(sc, "USB_ST_ERROR: %s", usbd_errstr(usb_error));

                goto tr_setup;
        }
}

static void
cfumass_t_data_callback(struct usb_xfer *xfer, usb_error_t usb_error)
{
        struct cfumass_softc *sc = usbd_xfer_softc(xfer);
        union ctl_io *io = sc->sc_ctl_io;
        uint32_t max_bulk;
        struct ctl_sg_entry sg_entry, *sglist;
        int actlen, sumlen, sg_count;

        switch (USB_GET_STATE(xfer)) {
        case USB_ST_TRANSFERRED:
                CFUMASS_DEBUG(sc, "USB_ST_TRANSFERRED");

                usbd_xfer_status(xfer, &actlen, &sumlen, NULL, NULL);
                sc->sc_current_residue -= actlen;
                io->scsiio.ext_data_filled += actlen;
                io->scsiio.kern_data_resid -= actlen;
                if (actlen < sumlen ||
                    sc->sc_current_residue == 0 ||
                    io->scsiio.kern_data_resid == 0) {
                        sc->sc_ctl_io = NULL;
                        ctl_datamove_done(io, false);
                        break;
                }
                /* FALLTHROUGH */

        case USB_ST_SETUP:
tr_setup:
                CFUMASS_DEBUG(sc, "USB_ST_SETUP");

                if (io->scsiio.kern_sg_entries > 0) {
                        sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
                        sg_count = io->scsiio.kern_sg_entries;
                } else {
                        sglist = &sg_entry;
                        sglist->addr = io->scsiio.kern_data_ptr;
                        sglist->len = io->scsiio.kern_data_len;
                        sg_count = 1;
                }

                sumlen = io->scsiio.ext_data_filled -
                    io->scsiio.kern_rel_offset;
                while (sumlen >= sglist->len && sg_count > 0) {
                        sumlen -= sglist->len;
                        sglist++;
                        sg_count--;
                }
                KASSERT(sg_count > 0, ("Run out of S/G list entries"));

                max_bulk = usbd_xfer_max_len(xfer);
                actlen = min(sglist->len - sumlen, max_bulk);
                actlen = min(actlen, sc->sc_current_transfer_length -
                    io->scsiio.ext_data_filled);
                CFUMASS_DEBUG(sc, "requested %d, done %d, max_bulk %d, "
                    "segment %zd => transfer %d",
                    sc->sc_current_transfer_length, io->scsiio.ext_data_filled,
                    max_bulk, sglist->len - sumlen, actlen);

                usbd_xfer_set_frame_data(xfer, 0,
                    (uint8_t *)sglist->addr + sumlen, actlen);
                usbd_transfer_submit(xfer);
                break;

        default:
                if (usb_error == USB_ERR_CANCELLED) {
                        CFUMASS_DEBUG(sc, "USB_ERR_CANCELLED");
                        break;
                }
                CFUMASS_DEBUG(sc, "USB_ST_ERROR: %s", usbd_errstr(usb_error));
                goto tr_setup;
        }
}

static void
cfumass_t_status_callback(struct usb_xfer *xfer, usb_error_t usb_error)
{
        struct cfumass_softc *sc;

        sc = usbd_xfer_softc(xfer);

        KASSERT(sc->sc_ctl_io == NULL,
            ("sc_ctl_io is %p, should be NULL", sc->sc_ctl_io));

        switch (USB_GET_STATE(xfer)) {
        case USB_ST_TRANSFERRED:
                CFUMASS_DEBUG(sc, "USB_ST_TRANSFERRED");

                cfumass_transfer_start(sc, CFUMASS_T_COMMAND);
                break;

        case USB_ST_SETUP:
tr_setup:
                CFUMASS_DEBUG(sc, "USB_ST_SETUP");

                if (sc->sc_current_residue > 0 && !sc->sc_current_stalled) {
                        CFUMASS_DEBUG(sc, "non-zero residue, stalling");
                        usbd_xfer_set_stall(xfer);
                        sc->sc_current_stalled = true;
                }

                USETDW(sc->sc_csw->dCSWSignature, CSWSIGNATURE);
                USETDW(sc->sc_csw->dCSWTag, sc->sc_current_tag);
                USETDW(sc->sc_csw->dCSWDataResidue, sc->sc_current_residue);
                sc->sc_csw->bCSWStatus = sc->sc_current_status;

                usbd_xfer_set_frame_len(xfer, 0, sizeof(*sc->sc_csw));
                usbd_transfer_submit(xfer);
                break;

        default:
                if (usb_error == USB_ERR_CANCELLED) {
                        CFUMASS_DEBUG(sc, "USB_ERR_CANCELLED");
                        break;
                }

                CFUMASS_DEBUG(sc, "USB_ST_ERROR: %s",
                    usbd_errstr(usb_error));

                goto tr_setup;
        }
}

static void
cfumass_online(void *arg __unused)
{

        cfumass_port_online = true;
}

static void
cfumass_offline(void *arg __unused)
{

        cfumass_port_online = false;
}

static void
cfumass_datamove(union ctl_io *io)
{
        struct cfumass_softc *sc;

        sc = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;

        CFUMASS_DEBUG(sc, "go");

        CFUMASS_LOCK(sc);

        KASSERT(sc->sc_ctl_io == NULL,
            ("sc_ctl_io is %p, should be NULL", sc->sc_ctl_io));
        sc->sc_ctl_io = io;

        if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) {
                /*
                 * Verify that CTL wants us to send the data in the direction
                 * expected by the initiator.
                 */
                if (sc->sc_current_flags != CBWFLAGS_IN) {
                        CFUMASS_WARN(sc, "wrong bCBWFlags 0x%x, should be 0x%x",
                            sc->sc_current_flags, CBWFLAGS_IN);
                        goto fail;
                }

                cfumass_transfer_start(sc, CFUMASS_T_DATA_IN);
        } else {
                if (sc->sc_current_flags != CBWFLAGS_OUT) {
                        CFUMASS_WARN(sc, "wrong bCBWFlags 0x%x, should be 0x%x",
                            sc->sc_current_flags, CBWFLAGS_OUT);
                        goto fail;
                }

                cfumass_transfer_start(sc, CFUMASS_T_DATA_OUT);
        }

        CFUMASS_UNLOCK(sc);
        return;

fail:
        ctl_set_data_phase_error(&io->scsiio);
        ctl_datamove_done(io, true);
        sc->sc_ctl_io = NULL;
}

static void
cfumass_done(union ctl_io *io)
{
        struct cfumass_softc *sc;

        sc = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;

        CFUMASS_DEBUG(sc, "go");

        KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
            ("invalid CTL status %#x", io->io_hdr.status));
        KASSERT(sc->sc_ctl_io == NULL,
            ("sc_ctl_io is %p, should be NULL", sc->sc_ctl_io));

        if (io->io_hdr.io_type == CTL_IO_TASK &&
            io->taskio.task_action == CTL_TASK_I_T_NEXUS_RESET) {
                /*
                 * Implicit task termination has just completed; nothing to do.
                 */
                ctl_free_io(io);
                return;
        }

        /*
         * Do not return status for aborted commands.
         * There are exceptions, but none supported by CTL yet.
         */
        if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
             (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
            (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
                ctl_free_io(io);
                return;
        }

        if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)
                sc->sc_current_status = 0;
        else
                sc->sc_current_status = 1;

        /* XXX: How should we report BUSY, RESERVATION CONFLICT, etc? */
        if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SCSI_ERROR &&
            io->scsiio.scsi_status == SCSI_STATUS_CHECK_COND)
                ctl_queue_sense(io);
        else
                ctl_free_io(io);

        CFUMASS_LOCK(sc);
        cfumass_transfer_start(sc, CFUMASS_T_STATUS);
        CFUMASS_UNLOCK(sc);

        refcount_release(&sc->sc_queued);
}

int
cfumass_init(void)
{
        int error;

        cfumass_port.frontend = &cfumass_frontend;
        cfumass_port.port_type = CTL_PORT_UMASS;
        cfumass_port.num_requested_ctl_io = 1;
        cfumass_port.port_name = "cfumass";
        cfumass_port.physical_port = 0;
        cfumass_port.virtual_port = 0;
        cfumass_port.port_online = cfumass_online;
        cfumass_port.port_offline = cfumass_offline;
        cfumass_port.onoff_arg = NULL;
        cfumass_port.fe_datamove = cfumass_datamove;
        cfumass_port.fe_done = cfumass_done;
        cfumass_port.targ_port = -1;

        error = ctl_port_register(&cfumass_port);
        if (error != 0) {
                printf("%s: ctl_port_register() failed "
                    "with error %d", __func__, error);
        }

        cfumass_port_online = true;
        refcount_init(&cfumass_refcount, 0);

        return (error);
}

int
cfumass_shutdown(void)
{
        int error;

        if (cfumass_refcount > 0) {
                if (debug > 1) {
                        printf("%s: still have %u attachments; "
                            "returning EBUSY\n", __func__, cfumass_refcount);
                }
                return (EBUSY);
        }

        error = ctl_port_deregister(&cfumass_port);
        if (error != 0) {
                printf("%s: ctl_port_deregister() failed "
                    "with error %d\n", __func__, error);
        }

        return (error);
}