/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2012, Bryan Venteicher <bryanv@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice unmodified, this list of conditions, and the following
 *    disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/* Driver for VirtIO SCSI devices. */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/sglist.h>
#include <sys/sysctl.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/callout.h>
#include <sys/queue.h>
#include <sys/sbuf.h>
#include <sys/stdarg.h>

#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus.h>
#include <sys/rman.h>

#include <cam/cam.h>
#include <cam/cam_ccb.h>
#include <cam/cam_sim.h>
#include <cam/cam_periph.h>
#include <cam/cam_xpt_sim.h>
#include <cam/cam_debug.h>
#include <cam/scsi/scsi_all.h>
#include <cam/scsi/scsi_message.h>

#include <dev/virtio/virtio.h>
#include <dev/virtio/virtqueue.h>
#include <dev/virtio/scsi/virtio_scsi.h>
#include <dev/virtio/scsi/virtio_scsivar.h>

#include "virtio_if.h"

static int      vtscsi_modevent(module_t, int, void *);

static int      vtscsi_probe(device_t);
static int      vtscsi_attach(device_t);
static int      vtscsi_detach(device_t);
static int      vtscsi_suspend(device_t);
static int      vtscsi_resume(device_t);

static int      vtscsi_negotiate_features(struct vtscsi_softc *);
static int      vtscsi_setup_features(struct vtscsi_softc *);
static void     vtscsi_read_config(struct vtscsi_softc *,
                    struct virtio_scsi_config *);
static int      vtscsi_maximum_segments(struct vtscsi_softc *, int);
static int      vtscsi_alloc_virtqueues(struct vtscsi_softc *);
static void     vtscsi_check_sizes(struct vtscsi_softc *);
static void     vtscsi_write_device_config(struct vtscsi_softc *);
static int      vtscsi_reinit(struct vtscsi_softc *);

static int      vtscsi_alloc_cam(struct vtscsi_softc *);
static int      vtscsi_register_cam(struct vtscsi_softc *);
static void     vtscsi_free_cam(struct vtscsi_softc *);
static void     vtscsi_cam_async(void *, uint32_t, struct cam_path *, void *);
static int      vtscsi_register_async(struct vtscsi_softc *);
static void     vtscsi_deregister_async(struct vtscsi_softc *);
static void     vtscsi_cam_action(struct cam_sim *, union ccb *);
static void     vtscsi_cam_poll(struct cam_sim *);

static void     vtscsi_cam_scsi_io(struct vtscsi_softc *, struct cam_sim *,
                    union ccb *);
static void     vtscsi_cam_get_tran_settings(struct vtscsi_softc *,
                    union ccb *);
static void     vtscsi_cam_reset_bus(struct vtscsi_softc *, union ccb *);
static void     vtscsi_cam_reset_dev(struct vtscsi_softc *, union ccb *);
static void     vtscsi_cam_abort(struct vtscsi_softc *, union ccb *);
static void     vtscsi_cam_path_inquiry(struct vtscsi_softc *,
                    struct cam_sim *, union ccb *);

static int      vtscsi_sg_append_scsi_buf(struct vtscsi_softc *,
                    struct sglist *, struct ccb_scsiio *);
static int      vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *,
                    struct vtscsi_request *, int *, int *);
static int      vtscsi_execute_scsi_cmd(struct vtscsi_softc *,
                    struct vtscsi_request *);
static int      vtscsi_start_scsi_cmd(struct vtscsi_softc *, union ccb *);
static void     vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *,
                    struct vtscsi_request *);
static int      vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *,
                    struct vtscsi_request *);
static void     vtscsi_timedout_scsi_cmd(void *);
static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *);
static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *,
                    struct ccb_scsiio *, struct virtio_scsi_cmd_resp *);
static void     vtscsi_complete_scsi_cmd(struct vtscsi_softc *,
                    struct vtscsi_request *);

static void     vtscsi_poll_ctrl_req(struct vtscsi_softc *,
                    struct vtscsi_request *);
static int      vtscsi_execute_ctrl_req(struct vtscsi_softc *,
                    struct vtscsi_request *, struct sglist *, int, int, int);
static void     vtscsi_complete_abort_task_cmd(struct vtscsi_softc *c,
                    struct vtscsi_request *);
static int      vtscsi_execute_abort_task_cmd(struct vtscsi_softc *,
                    struct vtscsi_request *);
static int      vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *,
                    struct vtscsi_request *);

static void     vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *);
static void     vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []);
static void     vtscsi_init_scsi_cmd_req(struct vtscsi_softc *,
                    struct ccb_scsiio *, struct virtio_scsi_cmd_req *);
static void     vtscsi_init_ctrl_tmf_req(struct vtscsi_softc *, struct ccb_hdr *,
                    uint32_t, uintptr_t, struct virtio_scsi_ctrl_tmf_req *);

static void     vtscsi_freeze_simq(struct vtscsi_softc *, int);
static int      vtscsi_thaw_simq(struct vtscsi_softc *, int);

static void     vtscsi_announce(struct vtscsi_softc *, uint32_t, target_id_t,
                    lun_id_t);
static void     vtscsi_execute_rescan(struct vtscsi_softc *, target_id_t,
                    lun_id_t);
static void     vtscsi_execute_rescan_bus(struct vtscsi_softc *);

static void     vtscsi_handle_event(struct vtscsi_softc *,
                    struct virtio_scsi_event *);
static int      vtscsi_enqueue_event_buf(struct vtscsi_softc *,
                    struct virtio_scsi_event *);
static int      vtscsi_init_event_vq(struct vtscsi_softc *);
static void     vtscsi_reinit_event_vq(struct vtscsi_softc *);
static void     vtscsi_drain_event_vq(struct vtscsi_softc *);

static void     vtscsi_complete_vqs_locked(struct vtscsi_softc *);
static void     vtscsi_complete_vqs(struct vtscsi_softc *);
static void     vtscsi_drain_vqs(struct vtscsi_softc *);
static void     vtscsi_cancel_request(struct vtscsi_softc *,
                    struct vtscsi_request *);
static void     vtscsi_drain_vq(struct vtscsi_softc *, struct virtqueue *);
static void     vtscsi_stop(struct vtscsi_softc *);
static int      vtscsi_reset_bus(struct vtscsi_softc *);

static void     vtscsi_init_request(struct vtscsi_softc *,
                    struct vtscsi_request *);
static int      vtscsi_alloc_requests(struct vtscsi_softc *);
static void     vtscsi_free_requests(struct vtscsi_softc *);
static void     vtscsi_enqueue_request(struct vtscsi_softc *,
                    struct vtscsi_request *);
static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *);

static void     vtscsi_complete_request(struct vtscsi_request *);
static void     vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *);

static void     vtscsi_control_vq_intr(void *);
static void     vtscsi_event_vq_intr(void *);
static void     vtscsi_request_vq_intr(void *);
static void     vtscsi_disable_vqs_intr(struct vtscsi_softc *);
static void     vtscsi_enable_vqs_intr(struct vtscsi_softc *);

static void     vtscsi_get_tunables(struct vtscsi_softc *);
static void     vtscsi_setup_sysctl(struct vtscsi_softc *);

static void     vtscsi_printf_req(struct vtscsi_request *, const char *,
                    const char *, ...);

#define vtscsi_modern(_sc) (((_sc)->vtscsi_features & VIRTIO_F_VERSION_1) != 0)
#define vtscsi_htog16(_sc, _val)        virtio_htog16(vtscsi_modern(_sc), _val)
#define vtscsi_htog32(_sc, _val)        virtio_htog32(vtscsi_modern(_sc), _val)
#define vtscsi_htog64(_sc, _val)        virtio_htog64(vtscsi_modern(_sc), _val)
#define vtscsi_gtoh16(_sc, _val)        virtio_gtoh16(vtscsi_modern(_sc), _val)
#define vtscsi_gtoh32(_sc, _val)        virtio_gtoh32(vtscsi_modern(_sc), _val)
#define vtscsi_gtoh64(_sc, _val)        virtio_gtoh64(vtscsi_modern(_sc), _val)

/* Global tunables. */
/*
 * The current QEMU VirtIO SCSI implementation does not cancel in-flight
 * IO during virtio_stop(). So in-flight requests still complete after the
 * device reset. We would have to wait for all the in-flight IO to complete,
 * which defeats the typical purpose of a bus reset. We could simulate the
 * bus reset with either I_T_NEXUS_RESET of all the targets, or with
 * LOGICAL_UNIT_RESET of all the LUNs (assuming there is space in the
 * control virtqueue). But this isn't very useful if things really go off
 * the rails, so default to disabled for now.
 */
static int vtscsi_bus_reset_disable = 1;
TUNABLE_INT("hw.vtscsi.bus_reset_disable", &vtscsi_bus_reset_disable);

static struct virtio_feature_desc vtscsi_feature_desc[] = {
        { VIRTIO_SCSI_F_INOUT,          "InOut"         },
        { VIRTIO_SCSI_F_HOTPLUG,        "Hotplug"       },
        { VIRTIO_SCSI_F_CHANGE,         "ChangeEvent"   },
        { VIRTIO_SCSI_F_T10_PI,         "T10PI"         },

        { 0, NULL }
};

static device_method_t vtscsi_methods[] = {
        /* Device methods. */
        DEVMETHOD(device_probe,         vtscsi_probe),
        DEVMETHOD(device_attach,        vtscsi_attach),
        DEVMETHOD(device_detach,        vtscsi_detach),
        DEVMETHOD(device_suspend,       vtscsi_suspend),
        DEVMETHOD(device_resume,        vtscsi_resume),

        DEVMETHOD_END
};

static driver_t vtscsi_driver = {
        "vtscsi",
        vtscsi_methods,
        sizeof(struct vtscsi_softc)
};

VIRTIO_DRIVER_MODULE(virtio_scsi, vtscsi_driver, vtscsi_modevent, NULL);
MODULE_VERSION(virtio_scsi, 1);
MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1);
MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1);

VIRTIO_SIMPLE_PNPINFO(virtio_scsi, VIRTIO_ID_SCSI, "VirtIO SCSI Adapter");

static int
vtscsi_modevent(module_t mod, int type, void *unused)
{
        int error;

        switch (type) {
        case MOD_LOAD:
        case MOD_QUIESCE:
        case MOD_UNLOAD:
        case MOD_SHUTDOWN:
                error = 0;
                break;
        default:
                error = EOPNOTSUPP;
                break;
        }

        return (error);
}

static int
vtscsi_probe(device_t dev)
{
        return (VIRTIO_SIMPLE_PROBE(dev, virtio_scsi));
}

static int
vtscsi_attach(device_t dev)
{
        struct vtscsi_softc *sc;
        struct virtio_scsi_config scsicfg;
        int error;

        sc = device_get_softc(dev);
        sc->vtscsi_dev = dev;
        virtio_set_feature_desc(dev, vtscsi_feature_desc);

        VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev));
        TAILQ_INIT(&sc->vtscsi_req_free);

        vtscsi_get_tunables(sc);
        vtscsi_setup_sysctl(sc);

        error = vtscsi_setup_features(sc);
        if (error) {
                device_printf(dev, "cannot setup features\n");
                goto fail;
        }

        vtscsi_read_config(sc, &scsicfg);

        sc->vtscsi_max_channel = scsicfg.max_channel;
        sc->vtscsi_max_target = scsicfg.max_target;
        sc->vtscsi_max_lun = scsicfg.max_lun;
        sc->vtscsi_event_buf_size = scsicfg.event_info_size;

        vtscsi_write_device_config(sc);

        sc->vtscsi_max_nsegs = vtscsi_maximum_segments(sc, scsicfg.seg_max);
        sc->vtscsi_sglist = sglist_alloc(sc->vtscsi_max_nsegs, M_NOWAIT);
        if (sc->vtscsi_sglist == NULL) {
                error = ENOMEM;
                device_printf(dev, "cannot allocate sglist\n");
                goto fail;
        }

        error = vtscsi_alloc_virtqueues(sc);
        if (error) {
                device_printf(dev, "cannot allocate virtqueues\n");
                goto fail;
        }

        vtscsi_check_sizes(sc);

        error = vtscsi_init_event_vq(sc);
        if (error) {
                device_printf(dev, "cannot populate the eventvq\n");
                goto fail;
        }

        error = vtscsi_alloc_requests(sc);
        if (error) {
                device_printf(dev, "cannot allocate requests\n");
                goto fail;
        }

        error = vtscsi_alloc_cam(sc);
        if (error) {
                device_printf(dev, "cannot allocate CAM structures\n");
                goto fail;
        }

        error = virtio_setup_intr(dev, INTR_TYPE_CAM);
        if (error) {
                device_printf(dev, "cannot setup virtqueue interrupts\n");
                goto fail;
        }

        vtscsi_enable_vqs_intr(sc);

        /*
         * Register with CAM after interrupts are enabled so we will get
         * notified of the probe responses.
         */
        error = vtscsi_register_cam(sc);
        if (error) {
                device_printf(dev, "cannot register with CAM\n");
                goto fail;
        }

fail:
        if (error)
                vtscsi_detach(dev);

        return (error);
}

static int
vtscsi_detach(device_t dev)
{
        struct vtscsi_softc *sc;

        sc = device_get_softc(dev);

        VTSCSI_LOCK(sc);
        sc->vtscsi_flags |= VTSCSI_FLAG_DETACH;
        if (device_is_attached(dev))
                vtscsi_stop(sc);
        VTSCSI_UNLOCK(sc);

        vtscsi_complete_vqs(sc);
        vtscsi_drain_vqs(sc);

        vtscsi_free_cam(sc);
        vtscsi_free_requests(sc);

        if (sc->vtscsi_sglist != NULL) {
                sglist_free(sc->vtscsi_sglist);
                sc->vtscsi_sglist = NULL;
        }

        VTSCSI_LOCK_DESTROY(sc);

        return (0);
}

static int
vtscsi_suspend(device_t dev)
{

        return (0);
}

static int
vtscsi_resume(device_t dev)
{

        return (0);
}

static int
vtscsi_negotiate_features(struct vtscsi_softc *sc)
{
        device_t dev;
        uint64_t features;

        dev = sc->vtscsi_dev;
        features = VTSCSI_FEATURES;

        sc->vtscsi_features = virtio_negotiate_features(dev, features);
        return (virtio_finalize_features(dev));
}

static int
vtscsi_setup_features(struct vtscsi_softc *sc)
{
        device_t dev;
        int error;

        dev = sc->vtscsi_dev;

        error = vtscsi_negotiate_features(sc);
        if (error)
                return (error);

        if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
                sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT;
        if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT))
                sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL;
        if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG))
                sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG;

        return (0);
}

#define VTSCSI_GET_CONFIG(_dev, _field, _cfg)                   \
        virtio_read_device_config(_dev,                         \
            offsetof(struct virtio_scsi_config, _field),        \
            &(_cfg)->_field, sizeof((_cfg)->_field))            \

static void
vtscsi_read_config(struct vtscsi_softc *sc,
    struct virtio_scsi_config *scsicfg)
{
        device_t dev;

        dev = sc->vtscsi_dev;

        bzero(scsicfg, sizeof(struct virtio_scsi_config));

        VTSCSI_GET_CONFIG(dev, num_queues, scsicfg);
        VTSCSI_GET_CONFIG(dev, seg_max, scsicfg);
        VTSCSI_GET_CONFIG(dev, max_sectors, scsicfg);
        VTSCSI_GET_CONFIG(dev, cmd_per_lun, scsicfg);
        VTSCSI_GET_CONFIG(dev, event_info_size, scsicfg);
        VTSCSI_GET_CONFIG(dev, sense_size, scsicfg);
        VTSCSI_GET_CONFIG(dev, cdb_size, scsicfg);
        VTSCSI_GET_CONFIG(dev, max_channel, scsicfg);
        VTSCSI_GET_CONFIG(dev, max_target, scsicfg);
        VTSCSI_GET_CONFIG(dev, max_lun, scsicfg);
}

#undef VTSCSI_GET_CONFIG

static int
vtscsi_maximum_segments(struct vtscsi_softc *sc, int seg_max)
{
        int nsegs;

        nsegs = VTSCSI_MIN_SEGMENTS;

        if (seg_max > 0) {
                nsegs += MIN(seg_max, maxphys / PAGE_SIZE + 1);
                if (sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT)
                        nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT);
        } else
                nsegs += 1;

        return (nsegs);
}

static int
vtscsi_alloc_virtqueues(struct vtscsi_softc *sc)
{
        device_t dev;
        struct vq_alloc_info vq_info[3];
        int nvqs;

        dev = sc->vtscsi_dev;
        nvqs = 3;

        VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtscsi_control_vq_intr, sc,
            &sc->vtscsi_control_vq, "%s control", device_get_nameunit(dev));

        VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtscsi_event_vq_intr, sc,
            &sc->vtscsi_event_vq, "%s event", device_get_nameunit(dev));

        VQ_ALLOC_INFO_INIT(&vq_info[2], sc->vtscsi_max_nsegs,
            vtscsi_request_vq_intr, sc, &sc->vtscsi_request_vq,
            "%s request", device_get_nameunit(dev));

        return (virtio_alloc_virtqueues(dev, nvqs, vq_info));
}

static void
vtscsi_check_sizes(struct vtscsi_softc *sc)
{
        int rqsize;

        if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0) {
                /*
                 * Ensure the assertions in virtqueue_enqueue(),
                 * even if the hypervisor reports a bad seg_max.
                 */
                rqsize = virtqueue_size(sc->vtscsi_request_vq);
                if (sc->vtscsi_max_nsegs > rqsize) {
                        device_printf(sc->vtscsi_dev,
                            "clamping seg_max (%d %d)\n", sc->vtscsi_max_nsegs,
                            rqsize);
                        sc->vtscsi_max_nsegs = rqsize;
                }
        }
}

static void
vtscsi_write_device_config(struct vtscsi_softc *sc)
{

        virtio_write_dev_config_4(sc->vtscsi_dev,
            offsetof(struct virtio_scsi_config, sense_size),
            VIRTIO_SCSI_SENSE_SIZE);

        /*
         * This is the size in the virtio_scsi_cmd_req structure. Note
         * this value (32) is larger than the maximum CAM CDB size (16).
         */
        virtio_write_dev_config_4(sc->vtscsi_dev,
            offsetof(struct virtio_scsi_config, cdb_size),
            VIRTIO_SCSI_CDB_SIZE);
}

static int
vtscsi_reinit(struct vtscsi_softc *sc)
{
        device_t dev;
        int error;

        dev = sc->vtscsi_dev;

        error = virtio_reinit(dev, sc->vtscsi_features);
        if (error == 0) {
                vtscsi_write_device_config(sc);
                virtio_reinit_complete(dev);
                vtscsi_reinit_event_vq(sc);

                vtscsi_enable_vqs_intr(sc);
        }

        vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d\n", error);

        return (error);
}

static int
vtscsi_alloc_cam(struct vtscsi_softc *sc)
{
        device_t dev;
        struct cam_devq *devq;
        int openings;

        dev = sc->vtscsi_dev;
        openings = sc->vtscsi_nrequests - VTSCSI_RESERVED_REQUESTS;

        devq = cam_simq_alloc(openings);
        if (devq == NULL) {
                device_printf(dev, "cannot allocate SIM queue\n");
                return (ENOMEM);
        }

        sc->vtscsi_sim = cam_sim_alloc(vtscsi_cam_action, vtscsi_cam_poll,
            "vtscsi", sc, device_get_unit(dev), VTSCSI_MTX(sc), 1,
            openings, devq);
        if (sc->vtscsi_sim == NULL) {
                cam_simq_free(devq);
                device_printf(dev, "cannot allocate SIM\n");
                return (ENOMEM);
        }

        return (0);
}

static int
vtscsi_register_cam(struct vtscsi_softc *sc)
{
        device_t dev;
        int registered, error;

        dev = sc->vtscsi_dev;
        registered = 0;

        VTSCSI_LOCK(sc);

        if (xpt_bus_register(sc->vtscsi_sim, dev, 0) != CAM_SUCCESS) {
                error = ENOMEM;
                device_printf(dev, "cannot register XPT bus\n");
                goto fail;
        }

        registered = 1;

        if (xpt_create_path(&sc->vtscsi_path, NULL,
            cam_sim_path(sc->vtscsi_sim), CAM_TARGET_WILDCARD,
            CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
                error = ENOMEM;
                device_printf(dev, "cannot create bus path\n");
                goto fail;
        }

        if (vtscsi_register_async(sc) != CAM_REQ_CMP) {
                error = EIO;
                device_printf(dev, "cannot register async callback\n");
                goto fail;
        }

        VTSCSI_UNLOCK(sc);

        return (0);

fail:
        if (sc->vtscsi_path != NULL) {
                xpt_free_path(sc->vtscsi_path);
                sc->vtscsi_path = NULL;
        }

        if (registered != 0)
                xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim));

        VTSCSI_UNLOCK(sc);

        return (error);
}

static void
vtscsi_free_cam(struct vtscsi_softc *sc)
{

        VTSCSI_LOCK(sc);

        if (sc->vtscsi_path != NULL) {
                vtscsi_deregister_async(sc);

                xpt_free_path(sc->vtscsi_path);
                sc->vtscsi_path = NULL;

                xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim));
        }

        if (sc->vtscsi_sim != NULL) {
                cam_sim_free(sc->vtscsi_sim, 1);
                sc->vtscsi_sim = NULL;
        }

        VTSCSI_UNLOCK(sc);
}

static void
vtscsi_cam_async(void *cb_arg, uint32_t code, struct cam_path *path, void *arg)
{
        struct cam_sim *sim;
        struct vtscsi_softc *sc;

        sim = cb_arg;
        sc = cam_sim_softc(sim);

        vtscsi_dprintf(sc, VTSCSI_TRACE, "code=%u\n", code);

        /*
         * TODO Once QEMU supports event reporting, we should
         *      (un)subscribe to events here.
         */
        switch (code) {
        case AC_FOUND_DEVICE:
                break;
        case AC_LOST_DEVICE:
                break;
        }
}

static int
vtscsi_register_async(struct vtscsi_softc *sc)
{
        struct ccb_setasync csa;

        memset(&csa, 0, sizeof(csa));
        xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5);
        csa.ccb_h.func_code = XPT_SASYNC_CB;
        csa.event_enable = AC_LOST_DEVICE | AC_FOUND_DEVICE;
        csa.callback = vtscsi_cam_async;
        csa.callback_arg = sc->vtscsi_sim;

        xpt_action((union ccb *) &csa);

        return (csa.ccb_h.status);
}

static void
vtscsi_deregister_async(struct vtscsi_softc *sc)
{
        struct ccb_setasync csa;

        memset(&csa, 0, sizeof(csa));
        xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5);
        csa.ccb_h.func_code = XPT_SASYNC_CB;
        csa.event_enable = 0;
        csa.callback = vtscsi_cam_async;
        csa.callback_arg = sc->vtscsi_sim;

        xpt_action((union ccb *) &csa);
}

static void
vtscsi_cam_action(struct cam_sim *sim, union ccb *ccb)
{
        struct vtscsi_softc *sc;
        struct ccb_hdr *ccbh;

        sc = cam_sim_softc(sim);
        ccbh = &ccb->ccb_h;

        VTSCSI_LOCK_OWNED(sc);

        if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) {
                /*
                 * The VTSCSI_MTX is briefly dropped between setting
                 * VTSCSI_FLAG_DETACH and deregistering with CAM, so
                 * drop any CCBs that come in during that window.
                 */
                ccbh->status = CAM_NO_HBA;
                xpt_done(ccb);
                return;
        }

        switch (ccbh->func_code) {
        case XPT_SCSI_IO:
                vtscsi_cam_scsi_io(sc, sim, ccb);
                break;

        case XPT_SET_TRAN_SETTINGS:
                ccbh->status = CAM_FUNC_NOTAVAIL;
                xpt_done(ccb);
                break;

        case XPT_GET_TRAN_SETTINGS:
                vtscsi_cam_get_tran_settings(sc, ccb);
                break;

        case XPT_RESET_BUS:
                vtscsi_cam_reset_bus(sc, ccb);
                break;

        case XPT_RESET_DEV:
                vtscsi_cam_reset_dev(sc, ccb);
                break;

        case XPT_ABORT:
                vtscsi_cam_abort(sc, ccb);
                break;

        case XPT_CALC_GEOMETRY:
                cam_calc_geometry(&ccb->ccg, 1);
                xpt_done(ccb);
                break;

        case XPT_PATH_INQ:
                vtscsi_cam_path_inquiry(sc, sim, ccb);
                break;

        default:
                vtscsi_dprintf(sc, VTSCSI_ERROR,
                    "invalid ccb=%p func=%#x\n", ccb, ccbh->func_code);

                ccbh->status = CAM_REQ_INVALID;
                xpt_done(ccb);
                break;
        }
}

static void
vtscsi_cam_poll(struct cam_sim *sim)
{
        struct vtscsi_softc *sc;

        sc = cam_sim_softc(sim);

        vtscsi_complete_vqs_locked(sc);
}

static void
vtscsi_cam_scsi_io(struct vtscsi_softc *sc, struct cam_sim *sim,
    union ccb *ccb)
{
        struct ccb_hdr *ccbh;
        struct ccb_scsiio *csio;
        int error;

        ccbh = &ccb->ccb_h;
        csio = &ccb->csio;

        if (csio->cdb_len > VIRTIO_SCSI_CDB_SIZE) {
                error = EINVAL;
                ccbh->status = CAM_REQ_INVALID;
                goto done;
        }

        if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_BOTH &&
            (sc->vtscsi_flags & VTSCSI_FLAG_BIDIRECTIONAL) == 0) {
                error = EINVAL;
                ccbh->status = CAM_REQ_INVALID;
                goto done;
        }

        error = vtscsi_start_scsi_cmd(sc, ccb);

done:
        if (error) {
                vtscsi_dprintf(sc, VTSCSI_ERROR,
                    "error=%d ccb=%p status=%#x\n", error, ccb, ccbh->status);
                xpt_done(ccb);
        }
}

static void
vtscsi_cam_get_tran_settings(struct vtscsi_softc *sc, union ccb *ccb)
{
        struct ccb_trans_settings *cts;
        struct ccb_trans_settings_scsi *scsi;

        cts = &ccb->cts;
        scsi = &cts->proto_specific.scsi;

        cts->protocol = PROTO_SCSI;
        cts->protocol_version = SCSI_REV_SPC3;
        cts->transport = XPORT_SAS;
        cts->transport_version = 0;

        scsi->valid = CTS_SCSI_VALID_TQ;
        scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;

        ccb->ccb_h.status = CAM_REQ_CMP;
        xpt_done(ccb);
}

static void
vtscsi_cam_reset_bus(struct vtscsi_softc *sc, union ccb *ccb)
{
        int error;

        error = vtscsi_reset_bus(sc);
        if (error == 0)
                ccb->ccb_h.status = CAM_REQ_CMP;
        else
                ccb->ccb_h.status = CAM_REQ_CMP_ERR;

        vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d ccb=%p status=%#x\n",
            error, ccb, ccb->ccb_h.status);

        xpt_done(ccb);
}

static void
vtscsi_cam_reset_dev(struct vtscsi_softc *sc, union ccb *ccb)
{
        struct ccb_hdr *ccbh;
        struct vtscsi_request *req;
        int error;

        ccbh = &ccb->ccb_h;

        req = vtscsi_dequeue_request(sc);
        if (req == NULL) {
                error = EAGAIN;
                vtscsi_freeze_simq(sc, VTSCSI_REQUEST);
                goto fail;
        }

        req->vsr_ccb = ccb;

        error = vtscsi_execute_reset_dev_cmd(sc, req);
        if (error == 0)
                return;

        vtscsi_enqueue_request(sc, req);

fail:
        vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n",
            error, req, ccb);

        if (error == EAGAIN)
                ccbh->status = CAM_RESRC_UNAVAIL;
        else
                ccbh->status = CAM_REQ_CMP_ERR;

        xpt_done(ccb);
}

static void
vtscsi_cam_abort(struct vtscsi_softc *sc, union ccb *ccb)
{
        struct vtscsi_request *req;
        struct ccb_hdr *ccbh;
        int error;

        ccbh = &ccb->ccb_h;

        req = vtscsi_dequeue_request(sc);
        if (req == NULL) {
                error = EAGAIN;
                vtscsi_freeze_simq(sc, VTSCSI_REQUEST);
                goto fail;
        }

        req->vsr_ccb = ccb;

        error = vtscsi_execute_abort_task_cmd(sc, req);
        if (error == 0)
                return;

        vtscsi_enqueue_request(sc, req);

fail:
        vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n",
            error, req, ccb);

        if (error == EAGAIN)
                ccbh->status = CAM_RESRC_UNAVAIL;
        else
                ccbh->status = CAM_REQ_CMP_ERR;

        xpt_done(ccb);
}

static void
vtscsi_cam_path_inquiry(struct vtscsi_softc *sc, struct cam_sim *sim,
    union ccb *ccb)
{
        device_t dev;
        struct ccb_pathinq *cpi;

        dev = sc->vtscsi_dev;
        cpi = &ccb->cpi;

        vtscsi_dprintf(sc, VTSCSI_TRACE, "sim=%p ccb=%p\n", sim, ccb);

        cpi->version_num = 1;
        cpi->hba_inquiry = PI_TAG_ABLE;
        cpi->target_sprt = 0;
        cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED;
        if (vtscsi_bus_reset_disable != 0)
                cpi->hba_misc |= PIM_NOBUSRESET;
        cpi->hba_eng_cnt = 0;

        cpi->max_target = sc->vtscsi_max_target;
        cpi->max_lun = sc->vtscsi_max_lun;
        cpi->initiator_id = cpi->max_target + 1;

        strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
        strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN);
        strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);

        cpi->unit_number = cam_sim_unit(sim);
        cpi->bus_id = cam_sim_bus(sim);

        cpi->base_transfer_speed = 300000;

        cpi->protocol = PROTO_SCSI;
        cpi->protocol_version = SCSI_REV_SPC3;
        cpi->transport = XPORT_SAS;
        cpi->transport_version = 0;

        cpi->maxio = (sc->vtscsi_max_nsegs - VTSCSI_MIN_SEGMENTS - 1) *
            PAGE_SIZE;

        cpi->hba_vendor = virtio_get_vendor(dev);
        cpi->hba_device = virtio_get_device(dev);
        cpi->hba_subvendor = virtio_get_subvendor(dev);
        cpi->hba_subdevice = virtio_get_subdevice(dev);

        ccb->ccb_h.status = CAM_REQ_CMP;
        xpt_done(ccb);
}

static int
vtscsi_sg_append_scsi_buf(struct vtscsi_softc *sc, struct sglist *sg,
    struct ccb_scsiio *csio)
{
        struct ccb_hdr *ccbh;
        struct bus_dma_segment *dseg;
        int i, error;

        ccbh = &csio->ccb_h;
        error = 0;

        switch ((ccbh->flags & CAM_DATA_MASK)) {
        case CAM_DATA_VADDR:
                error = sglist_append(sg, csio->data_ptr, csio->dxfer_len);
                break;
        case CAM_DATA_PADDR:
                error = sglist_append_phys(sg,
                    (vm_paddr_t)(vm_offset_t) csio->data_ptr, csio->dxfer_len);
                break;
        case CAM_DATA_SG:
                for (i = 0; i < csio->sglist_cnt && error == 0; i++) {
                        dseg = &((struct bus_dma_segment *)csio->data_ptr)[i];
                        error = sglist_append(sg,
                            (void *)(vm_offset_t) dseg->ds_addr, dseg->ds_len);
                }
                break;
        case CAM_DATA_SG_PADDR:
                for (i = 0; i < csio->sglist_cnt && error == 0; i++) {
                        dseg = &((struct bus_dma_segment *)csio->data_ptr)[i];
                        error = sglist_append_phys(sg,
                            (vm_paddr_t) dseg->ds_addr, dseg->ds_len);
                }
                break;
        case CAM_DATA_BIO:
                error = sglist_append_bio(sg, (struct bio *) csio->data_ptr);
                break;
        default:
                error = EINVAL;
                break;
        }

        return (error);
}

static int
vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *sc, struct vtscsi_request *req,
    int *readable, int *writable)
{
        struct sglist *sg;
        struct ccb_hdr *ccbh;
        struct ccb_scsiio *csio;
        struct virtio_scsi_cmd_req *cmd_req;
        struct virtio_scsi_cmd_resp *cmd_resp;
        int error;

        sg = sc->vtscsi_sglist;
        csio = &req->vsr_ccb->csio;
        ccbh = &csio->ccb_h;
        cmd_req = &req->vsr_cmd_req;
        cmd_resp = &req->vsr_cmd_resp;

        sglist_reset(sg);

        sglist_append(sg, cmd_req, sizeof(struct virtio_scsi_cmd_req));
        if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
                error = vtscsi_sg_append_scsi_buf(sc, sg, csio);
                /* At least one segment must be left for the response. */
                if (error || sg->sg_nseg == sg->sg_maxseg)
                        goto fail;
        }

        *readable = sg->sg_nseg;

        sglist_append(sg, cmd_resp, sizeof(struct virtio_scsi_cmd_resp));
        if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_IN) {
                error = vtscsi_sg_append_scsi_buf(sc, sg, csio);
                if (error)
                        goto fail;
        }

        *writable = sg->sg_nseg - *readable;

        vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p readable=%d "
            "writable=%d\n", req, ccbh, *readable, *writable);

        return (0);

fail:
        /*
         * This should never happen unless maxio was incorrectly set.
         */
        vtscsi_set_ccb_status(ccbh, CAM_REQ_TOO_BIG, 0);

        vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p "
            "nseg=%d maxseg=%d\n",
            error, req, ccbh, sg->sg_nseg, sg->sg_maxseg);

        return (EFBIG);
}

static int
vtscsi_execute_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req)
{
        struct sglist *sg;
        struct virtqueue *vq;
        struct ccb_scsiio *csio;
        struct ccb_hdr *ccbh;
        struct virtio_scsi_cmd_req *cmd_req;
        struct virtio_scsi_cmd_resp *cmd_resp;
        int readable, writable, error;

        sg = sc->vtscsi_sglist;
        vq = sc->vtscsi_request_vq;
        csio = &req->vsr_ccb->csio;
        ccbh = &csio->ccb_h;
        cmd_req = &req->vsr_cmd_req;
        cmd_resp = &req->vsr_cmd_resp;

        vtscsi_init_scsi_cmd_req(sc, csio, cmd_req);

        error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable);
        if (error)
                return (error);

        req->vsr_complete = vtscsi_complete_scsi_cmd;
        cmd_resp->response = -1;

        error = virtqueue_enqueue(vq, req, sg, readable, writable);
        if (error) {
                vtscsi_dprintf(sc, VTSCSI_ERROR,
                    "enqueue error=%d req=%p ccb=%p\n", error, req, ccbh);

                ccbh->status = CAM_REQUEUE_REQ;
                vtscsi_freeze_simq(sc, VTSCSI_REQUEST_VQ);
                return (error);
        }

        ccbh->status |= CAM_SIM_QUEUED;
        ccbh->ccbh_vtscsi_req = req;

        virtqueue_notify(vq);

        if (ccbh->timeout != CAM_TIME_INFINITY) {
                req->vsr_flags |= VTSCSI_REQ_FLAG_TIMEOUT_SET;
                callout_reset_sbt(&req->vsr_callout, SBT_1MS * ccbh->timeout,
                    0, vtscsi_timedout_scsi_cmd, req, 0);
        }

        vtscsi_dprintf_req(req, VTSCSI_TRACE, "enqueued req=%p ccb=%p\n",
            req, ccbh);

        return (0);
}

static int
vtscsi_start_scsi_cmd(struct vtscsi_softc *sc, union ccb *ccb)
{
        struct vtscsi_request *req;
        int error;

        req = vtscsi_dequeue_request(sc);
        if (req == NULL) {
                ccb->ccb_h.status = CAM_REQUEUE_REQ;
                vtscsi_freeze_simq(sc, VTSCSI_REQUEST);
                return (ENOBUFS);
        }

        req->vsr_ccb = ccb;

        error = vtscsi_execute_scsi_cmd(sc, req);
        if (error)
                vtscsi_enqueue_request(sc, req);

        return (error);
}

static void
vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *sc,
    struct vtscsi_request *req)
{
        struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
        struct vtscsi_request *to_req;
        uint8_t response;

        tmf_resp = &req->vsr_tmf_resp;
        response = tmf_resp->response;
        to_req = req->vsr_timedout_req;

        vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p to_req=%p response=%d\n",
            req, to_req, response);

        vtscsi_enqueue_request(sc, req);

        /*
         * The timedout request could have completed between when the
         * abort task was sent and when the host processed it.
         */
        if (to_req->vsr_state != VTSCSI_REQ_STATE_TIMEDOUT)
                return;

        /* The timedout request was successfully aborted. */
        if (response == VIRTIO_SCSI_S_FUNCTION_COMPLETE)
                return;

        /* Don't bother if the device is going away. */
        if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH)
                return;

        /* The timedout request will be aborted by the reset. */
        if (sc->vtscsi_flags & VTSCSI_FLAG_RESET)
                return;

        vtscsi_reset_bus(sc);
}

static int
vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *sc,
    struct vtscsi_request *to_req)
{
        struct sglist *sg;
        struct ccb_hdr *to_ccbh;
        struct vtscsi_request *req;
        struct virtio_scsi_ctrl_tmf_req *tmf_req;
        struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
        int error;

        sg = sc->vtscsi_sglist;
        to_ccbh = &to_req->vsr_ccb->ccb_h;

        req = vtscsi_dequeue_request(sc);
        if (req == NULL) {
                error = ENOBUFS;
                goto fail;
        }

        tmf_req = &req->vsr_tmf_req;
        tmf_resp = &req->vsr_tmf_resp;

        vtscsi_init_ctrl_tmf_req(sc, to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,
            (uintptr_t) to_ccbh, tmf_req);

        sglist_reset(sg);
        sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req));
        sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp));

        req->vsr_timedout_req = to_req;
        req->vsr_complete = vtscsi_complete_abort_timedout_scsi_cmd;
        tmf_resp->response = -1;

        error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1,
            VTSCSI_EXECUTE_ASYNC);
        if (error == 0)
                return (0);

        vtscsi_enqueue_request(sc, req);

fail:
        vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p "
            "timedout req=%p ccb=%p\n", error, req, to_req, to_ccbh);

        return (error);
}

static void
vtscsi_timedout_scsi_cmd(void *xreq)
{
        struct vtscsi_softc *sc;
        struct vtscsi_request *to_req;

        to_req = xreq;
        sc = to_req->vsr_softc;

        vtscsi_dprintf(sc, VTSCSI_INFO, "timedout req=%p ccb=%p state=%#x\n",
            to_req, to_req->vsr_ccb, to_req->vsr_state);

        /* Don't bother if the device is going away. */
        if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH)
                return;

        /*
         * Bail if the request is not in use. We likely raced when
         * stopping the callout handler or it has already been aborted.
         */
        if (to_req->vsr_state != VTSCSI_REQ_STATE_INUSE ||
            (to_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) == 0)
                return;

        /*
         * Complete the request queue in case the timedout request is
         * actually just pending.
         */
        vtscsi_complete_vq(sc, sc->vtscsi_request_vq);
        if (to_req->vsr_state == VTSCSI_REQ_STATE_FREE)
                return;

        sc->vtscsi_stats.scsi_cmd_timeouts++;
        to_req->vsr_state = VTSCSI_REQ_STATE_TIMEDOUT;

        if (vtscsi_abort_timedout_scsi_cmd(sc, to_req) == 0)
                return;

        vtscsi_dprintf(sc, VTSCSI_ERROR, "resetting bus\n");
        vtscsi_reset_bus(sc);
}

static cam_status
vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *cmd_resp)
{
        cam_status status;

        switch (cmd_resp->response) {
        case VIRTIO_SCSI_S_OK:
                status = CAM_REQ_CMP;
                break;
        case VIRTIO_SCSI_S_OVERRUN:
                status = CAM_DATA_RUN_ERR;
                break;
        case VIRTIO_SCSI_S_ABORTED:
                status = CAM_REQ_ABORTED;
                break;
        case VIRTIO_SCSI_S_BAD_TARGET:
                status = CAM_SEL_TIMEOUT;
                break;
        case VIRTIO_SCSI_S_RESET:
                status = CAM_SCSI_BUS_RESET;
                break;
        case VIRTIO_SCSI_S_BUSY:
                status = CAM_SCSI_BUSY;
                break;
        case VIRTIO_SCSI_S_TRANSPORT_FAILURE:
        case VIRTIO_SCSI_S_TARGET_FAILURE:
        case VIRTIO_SCSI_S_NEXUS_FAILURE:
                status = CAM_SCSI_IT_NEXUS_LOST;
                break;
        default: /* VIRTIO_SCSI_S_FAILURE */
                status = CAM_REQ_CMP_ERR;
                break;
        }

        return (status);
}

static cam_status
vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc,
    struct ccb_scsiio *csio, struct virtio_scsi_cmd_resp *cmd_resp)
{
        uint32_t resp_sense_length;
        cam_status status;

        csio->scsi_status = cmd_resp->status;
        csio->resid = vtscsi_htog32(sc, cmd_resp->resid);

        if (csio->scsi_status == SCSI_STATUS_OK)
                status = CAM_REQ_CMP;
        else
                status = CAM_SCSI_STATUS_ERROR;

        resp_sense_length = vtscsi_htog32(sc, cmd_resp->sense_len);

        if (resp_sense_length > 0) {
                status |= CAM_AUTOSNS_VALID;

                if (resp_sense_length < csio->sense_len)
                        csio->sense_resid = csio->sense_len - resp_sense_length;
                else
                        csio->sense_resid = 0;

                memcpy(&csio->sense_data, cmd_resp->sense,
                    csio->sense_len - csio->sense_resid);
        }

        vtscsi_dprintf(sc, status == CAM_REQ_CMP ? VTSCSI_TRACE : VTSCSI_ERROR,
            "ccb=%p scsi_status=%#x resid=%u sense_resid=%u\n",
            csio, csio->scsi_status, csio->resid, csio->sense_resid);

        return (status);
}

static void
vtscsi_complete_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req)
{
        struct ccb_hdr *ccbh;
        struct ccb_scsiio *csio;
        struct virtio_scsi_cmd_resp *cmd_resp;
        cam_status status;

        csio = &req->vsr_ccb->csio;
        ccbh = &csio->ccb_h;
        cmd_resp = &req->vsr_cmd_resp;

        KASSERT(ccbh->ccbh_vtscsi_req == req,
            ("ccb %p req mismatch %p/%p", ccbh, ccbh->ccbh_vtscsi_req, req));

        if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET)
                callout_stop(&req->vsr_callout);

        status = vtscsi_scsi_cmd_cam_status(cmd_resp);
        if (status == CAM_REQ_ABORTED) {
                if (req->vsr_state == VTSCSI_REQ_STATE_TIMEDOUT)
                        status = CAM_CMD_TIMEOUT;
        } else if (status == CAM_REQ_CMP)
                status = vtscsi_complete_scsi_cmd_response(sc, csio, cmd_resp);

        if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
                status |= CAM_DEV_QFRZN;
                xpt_freeze_devq(ccbh->path, 1);
        }

        if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0)
                status |= CAM_RELEASE_SIMQ;

        vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p status=%#x\n",
            req, ccbh, status);

        ccbh->status = status;
        xpt_done(req->vsr_ccb);
        vtscsi_enqueue_request(sc, req);
}

static void
vtscsi_poll_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req)
{

        /* XXX We probably shouldn't poll forever. */
        req->vsr_flags |= VTSCSI_REQ_FLAG_POLLED;
        do
                vtscsi_complete_vq(sc, sc->vtscsi_control_vq);
        while ((req->vsr_flags & VTSCSI_REQ_FLAG_COMPLETE) == 0);

        req->vsr_flags &= ~VTSCSI_REQ_FLAG_POLLED;
}

static int
vtscsi_execute_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req,
    struct sglist *sg, int readable, int writable, int flag)
{
        struct virtqueue *vq;
        int error;

        vq = sc->vtscsi_control_vq;

        MPASS(flag == VTSCSI_EXECUTE_POLL || req->vsr_complete != NULL);

        error = virtqueue_enqueue(vq, req, sg, readable, writable);
        if (error) {
                /*
                 * Return EAGAIN when the virtqueue does not have enough
                 * descriptors available.
                 */
                if (error == ENOSPC || error == EMSGSIZE)
                        error = EAGAIN;

                return (error);
        }

        virtqueue_notify(vq);
        if (flag == VTSCSI_EXECUTE_POLL)
                vtscsi_poll_ctrl_req(sc, req);

        return (0);
}

static void
vtscsi_complete_abort_task_cmd(struct vtscsi_softc *sc,
    struct vtscsi_request *req)
{
        union ccb *ccb;
        struct ccb_hdr *ccbh;
        struct virtio_scsi_ctrl_tmf_resp *tmf_resp;

        ccb = req->vsr_ccb;
        ccbh = &ccb->ccb_h;
        tmf_resp = &req->vsr_tmf_resp;

        switch (tmf_resp->response) {
        case VIRTIO_SCSI_S_FUNCTION_COMPLETE:
                ccbh->status = CAM_REQ_CMP;
                break;
        case VIRTIO_SCSI_S_FUNCTION_REJECTED:
                ccbh->status = CAM_UA_ABORT;
                break;
        default:
                ccbh->status = CAM_REQ_CMP_ERR;
                break;
        }

        xpt_done(ccb);
        vtscsi_enqueue_request(sc, req);
}

static int
vtscsi_execute_abort_task_cmd(struct vtscsi_softc *sc,
    struct vtscsi_request *req)
{
        struct sglist *sg;
        struct ccb_abort *cab;
        struct ccb_hdr *ccbh;
        struct ccb_hdr *abort_ccbh;
        struct vtscsi_request *abort_req;
        struct virtio_scsi_ctrl_tmf_req *tmf_req;
        struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
        int error;

        sg = sc->vtscsi_sglist;
        cab = &req->vsr_ccb->cab;
        ccbh = &cab->ccb_h;
        tmf_req = &req->vsr_tmf_req;
        tmf_resp = &req->vsr_tmf_resp;

        /* CCB header and request that's to be aborted. */
        abort_ccbh = &cab->abort_ccb->ccb_h;
        abort_req = abort_ccbh->ccbh_vtscsi_req;

        if (abort_ccbh->func_code != XPT_SCSI_IO || abort_req == NULL) {
                error = EINVAL;
                goto fail;
        }

        /* Only attempt to abort requests that could be in-flight. */
        if (abort_req->vsr_state != VTSCSI_REQ_STATE_INUSE) {
                error = EALREADY;
                goto fail;
        }

        abort_req->vsr_state = VTSCSI_REQ_STATE_ABORTED;
        if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET)
                callout_stop(&abort_req->vsr_callout);

        vtscsi_init_ctrl_tmf_req(sc, ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,
            (uintptr_t) abort_ccbh, tmf_req);

        sglist_reset(sg);
        sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req));
        sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp));

        req->vsr_complete = vtscsi_complete_abort_task_cmd;
        tmf_resp->response = -1;

        error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1,
            VTSCSI_EXECUTE_ASYNC);

fail:
        vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p abort_ccb=%p "
            "abort_req=%p\n", error, req, abort_ccbh, abort_req);

        return (error);
}

static void
vtscsi_complete_reset_dev_cmd(struct vtscsi_softc *sc,
    struct vtscsi_request *req)
{
        union ccb *ccb;
        struct ccb_hdr *ccbh;
        struct virtio_scsi_ctrl_tmf_resp *tmf_resp;

        ccb = req->vsr_ccb;
        ccbh = &ccb->ccb_h;
        tmf_resp = &req->vsr_tmf_resp;

        vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p response=%d\n",
            req, ccb, tmf_resp->response);

        if (tmf_resp->response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) {
                ccbh->status = CAM_REQ_CMP;
                vtscsi_announce(sc, AC_SENT_BDR, ccbh->target_id,
                    ccbh->target_lun);
        } else
                ccbh->status = CAM_REQ_CMP_ERR;

        xpt_done(ccb);
        vtscsi_enqueue_request(sc, req);
}

static int
vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *sc,
    struct vtscsi_request *req)
{
        struct sglist *sg;
        struct ccb_resetdev *crd;
        struct ccb_hdr *ccbh;
        struct virtio_scsi_ctrl_tmf_req *tmf_req;
        struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
        uint32_t subtype;
        int error;

        sg = sc->vtscsi_sglist;
        crd = &req->vsr_ccb->crd;
        ccbh = &crd->ccb_h;
        tmf_req = &req->vsr_tmf_req;
        tmf_resp = &req->vsr_tmf_resp;

        if (ccbh->target_lun == CAM_LUN_WILDCARD)
                subtype = VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET;
        else
                subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET;

        vtscsi_init_ctrl_tmf_req(sc, ccbh, subtype, 0, tmf_req);

        sglist_reset(sg);
        sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req));
        sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp));

        req->vsr_complete = vtscsi_complete_reset_dev_cmd;
        tmf_resp->response = -1;

        error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1,
            VTSCSI_EXECUTE_ASYNC);

        vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p ccb=%p\n",
            error, req, ccbh);

        return (error);
}

static void
vtscsi_get_request_lun(uint8_t lun[], target_id_t *target_id, lun_id_t *lun_id)
{

        *target_id = lun[1];
        *lun_id = (lun[2] << 8) | lun[3];
}

static void
vtscsi_set_request_lun(struct ccb_hdr *ccbh, uint8_t lun[])
{

        lun[0] = 1;
        lun[1] = ccbh->target_id;
        lun[2] = 0x40 | ((ccbh->target_lun >> 8) & 0x3F);
        lun[3] = ccbh->target_lun & 0xFF;
}

static void
vtscsi_init_scsi_cmd_req(struct vtscsi_softc *sc, struct ccb_scsiio *csio,
    struct virtio_scsi_cmd_req *cmd_req)
{
        uint8_t attr;

        switch (csio->tag_action) {
        case MSG_HEAD_OF_Q_TAG:
                attr = VIRTIO_SCSI_S_HEAD;
                break;
        case MSG_ORDERED_Q_TAG:
                attr = VIRTIO_SCSI_S_ORDERED;
                break;
        case MSG_ACA_TASK:
                attr = VIRTIO_SCSI_S_ACA;
                break;
        default: /* MSG_SIMPLE_Q_TAG */
                attr = VIRTIO_SCSI_S_SIMPLE;
                break;
        }

        vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun);
        cmd_req->tag = vtscsi_gtoh64(sc, (uintptr_t) csio);
        cmd_req->task_attr = attr;

        memcpy(cmd_req->cdb,
            csio->ccb_h.flags & CAM_CDB_POINTER ?
                csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes,
            csio->cdb_len);
}

static void
vtscsi_init_ctrl_tmf_req(struct vtscsi_softc *sc, struct ccb_hdr *ccbh,
    uint32_t subtype, uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req)
{

        vtscsi_set_request_lun(ccbh, tmf_req->lun);

        tmf_req->type = vtscsi_gtoh32(sc, VIRTIO_SCSI_T_TMF);
        tmf_req->subtype = vtscsi_gtoh32(sc, subtype);
        tmf_req->tag = vtscsi_gtoh64(sc, tag);
}

static void
vtscsi_freeze_simq(struct vtscsi_softc *sc, int reason)
{
        int frozen;

        frozen = sc->vtscsi_frozen;

        if (reason & VTSCSI_REQUEST &&
            (sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) == 0)
                sc->vtscsi_frozen |= VTSCSI_FROZEN_NO_REQUESTS;

        if (reason & VTSCSI_REQUEST_VQ &&
            (sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) == 0)
                sc->vtscsi_frozen |= VTSCSI_FROZEN_REQUEST_VQ_FULL;

        /* Freeze the SIMQ if transitioned to frozen. */
        if (frozen == 0 && sc->vtscsi_frozen != 0) {
                vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ frozen\n");
                xpt_freeze_simq(sc->vtscsi_sim, 1);
        }
}

static int
vtscsi_thaw_simq(struct vtscsi_softc *sc, int reason)
{
        int thawed;

        if (sc->vtscsi_frozen == 0 || reason == 0)
                return (0);

        if (reason & VTSCSI_REQUEST &&
            sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS)
                sc->vtscsi_frozen &= ~VTSCSI_FROZEN_NO_REQUESTS;

        if (reason & VTSCSI_REQUEST_VQ &&
            sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL)
                sc->vtscsi_frozen &= ~VTSCSI_FROZEN_REQUEST_VQ_FULL;

        thawed = sc->vtscsi_frozen == 0;
        if (thawed != 0)
                vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ thawed\n");

        return (thawed);
}

static void
vtscsi_announce(struct vtscsi_softc *sc, uint32_t ac_code,
    target_id_t target_id, lun_id_t lun_id)
{
        struct cam_path *path;

        /* Use the wildcard path from our softc for bus announcements. */
        if (target_id == CAM_TARGET_WILDCARD && lun_id == CAM_LUN_WILDCARD) {
                xpt_async(ac_code, sc->vtscsi_path, NULL);
                return;
        }

        if (xpt_create_path(&path, NULL, cam_sim_path(sc->vtscsi_sim),
            target_id, lun_id) != CAM_REQ_CMP) {
                vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot create path\n");
                return;
        }

        xpt_async(ac_code, path, NULL);
        xpt_free_path(path);
}

static void
vtscsi_execute_rescan(struct vtscsi_softc *sc, target_id_t target_id,
    lun_id_t lun_id)
{
        union ccb *ccb;
        cam_status status;

        ccb = xpt_alloc_ccb_nowait();
        if (ccb == NULL) {
                vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n");
                return;
        }

        status = xpt_create_path(&ccb->ccb_h.path, NULL,
            cam_sim_path(sc->vtscsi_sim), target_id, lun_id);
        if (status != CAM_REQ_CMP) {
                xpt_free_ccb(ccb);
                return;
        }

        xpt_rescan(ccb);
}

static void
vtscsi_execute_rescan_bus(struct vtscsi_softc *sc)
{

        vtscsi_execute_rescan(sc, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
}

static void
vtscsi_transport_reset_event(struct vtscsi_softc *sc,
    struct virtio_scsi_event *event)
{
        target_id_t target_id;
        lun_id_t lun_id;

        vtscsi_get_request_lun(event->lun, &target_id, &lun_id);

        switch (event->reason) {
        case VIRTIO_SCSI_EVT_RESET_RESCAN:
        case VIRTIO_SCSI_EVT_RESET_REMOVED:
                vtscsi_execute_rescan(sc, target_id, lun_id);
                break;
        default:
                device_printf(sc->vtscsi_dev,
                    "unhandled transport event reason: %d\n", event->reason);
                break;
        }
}

static void
vtscsi_handle_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event)
{
        int error __diagused;

        if ((event->event & VIRTIO_SCSI_T_EVENTS_MISSED) == 0) {
                switch (event->event) {
                case VIRTIO_SCSI_T_TRANSPORT_RESET:
                        vtscsi_transport_reset_event(sc, event);
                        break;
                default:
                        device_printf(sc->vtscsi_dev,
                            "unhandled event: %d\n", event->event);
                        break;
                }
        } else
                vtscsi_execute_rescan_bus(sc);

        /*
         * This should always be successful since the buffer
         * was just dequeued.
         */
        error = vtscsi_enqueue_event_buf(sc, event);
        KASSERT(error == 0,
            ("cannot requeue event buffer: %d", error));
}

static int
vtscsi_enqueue_event_buf(struct vtscsi_softc *sc,
    struct virtio_scsi_event *event)
{
        struct sglist *sg;
        struct virtqueue *vq;
        int size, error;

        sg = sc->vtscsi_sglist;
        vq = sc->vtscsi_event_vq;
        size = sc->vtscsi_event_buf_size;

        bzero(event, size);

        sglist_reset(sg);
        error = sglist_append(sg, event, size);
        if (error)
                return (error);

        error = virtqueue_enqueue(vq, event, sg, 0, sg->sg_nseg);
        if (error)
                return (error);

        virtqueue_notify(vq);

        return (0);
}

static int
vtscsi_init_event_vq(struct vtscsi_softc *sc)
{
        struct virtio_scsi_event *event;
        int i, size, error;

        /*
         * The first release of QEMU with VirtIO SCSI support would crash
         * when attempting to notify the event virtqueue. This was fixed
         * when hotplug support was added.
         */
        if (sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG)
                size = sc->vtscsi_event_buf_size;
        else
                size = 0;

        if (size < sizeof(struct virtio_scsi_event))
                return (0);

        for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) {
                event = &sc->vtscsi_event_bufs[i];

                error = vtscsi_enqueue_event_buf(sc, event);
                if (error)
                        break;
        }

        /*
         * Even just one buffer is enough. Missed events are
         * denoted with the VIRTIO_SCSI_T_EVENTS_MISSED flag.
         */
        if (i > 0)
                error = 0;

        return (error);
}

static void
vtscsi_reinit_event_vq(struct vtscsi_softc *sc)
{
        struct virtio_scsi_event *event;
        int i, error;

        if ((sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) == 0 ||
            sc->vtscsi_event_buf_size < sizeof(struct virtio_scsi_event))
                return;

        for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) {
                event = &sc->vtscsi_event_bufs[i];

                error = vtscsi_enqueue_event_buf(sc, event);
                if (error)
                        break;
        }

        KASSERT(i > 0, ("cannot reinit event vq: %d", error));
}

static void
vtscsi_drain_event_vq(struct vtscsi_softc *sc)
{
        struct virtqueue *vq;
        int last;

        vq = sc->vtscsi_event_vq;
        last = 0;

        while (virtqueue_drain(vq, &last) != NULL)
                ;

        KASSERT(virtqueue_empty(vq), ("eventvq not empty"));
}

static void
vtscsi_complete_vqs_locked(struct vtscsi_softc *sc)
{

        VTSCSI_LOCK_OWNED(sc);

        if (sc->vtscsi_request_vq != NULL)
                vtscsi_complete_vq(sc, sc->vtscsi_request_vq);
        if (sc->vtscsi_control_vq != NULL)
                vtscsi_complete_vq(sc, sc->vtscsi_control_vq);
}

static void
vtscsi_complete_vqs(struct vtscsi_softc *sc)
{

        VTSCSI_LOCK(sc);
        vtscsi_complete_vqs_locked(sc);
        VTSCSI_UNLOCK(sc);
}

static void
vtscsi_cancel_request(struct vtscsi_softc *sc, struct vtscsi_request *req)
{
        union ccb *ccb;
        int detach;

        ccb = req->vsr_ccb;

        vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p\n", req, ccb);

        /*
         * The callout must be drained when detaching since the request is
         * about to be freed. The VTSCSI_MTX must not be held for this in
         * case the callout is pending because there is a deadlock potential.
         * Otherwise, the virtqueue is being drained because of a bus reset
         * so we only need to attempt to stop the callouts.
         */
        detach = (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) != 0;
        if (detach != 0)
                VTSCSI_LOCK_NOTOWNED(sc);
        else
                VTSCSI_LOCK_OWNED(sc);

        if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) {
                if (detach != 0)
                        callout_drain(&req->vsr_callout);
                else
                        callout_stop(&req->vsr_callout);
        }

        if (ccb != NULL) {
                if (detach != 0) {
                        VTSCSI_LOCK(sc);
                        ccb->ccb_h.status = CAM_NO_HBA;
                } else
                        ccb->ccb_h.status = CAM_REQUEUE_REQ;
                xpt_done(ccb);
                if (detach != 0)
                        VTSCSI_UNLOCK(sc);
        }

        vtscsi_enqueue_request(sc, req);
}

static void
vtscsi_drain_vq(struct vtscsi_softc *sc, struct virtqueue *vq)
{
        struct vtscsi_request *req;
        int last;

        last = 0;

        vtscsi_dprintf(sc, VTSCSI_TRACE, "vq=%p\n", vq);

        while ((req = virtqueue_drain(vq, &last)) != NULL)
                vtscsi_cancel_request(sc, req);

        KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
}

static void
vtscsi_drain_vqs(struct vtscsi_softc *sc)
{

        if (sc->vtscsi_control_vq != NULL)
                vtscsi_drain_vq(sc, sc->vtscsi_control_vq);
        if (sc->vtscsi_request_vq != NULL)
                vtscsi_drain_vq(sc, sc->vtscsi_request_vq);
        if (sc->vtscsi_event_vq != NULL)
                vtscsi_drain_event_vq(sc);
}

static void
vtscsi_stop(struct vtscsi_softc *sc)
{

        vtscsi_disable_vqs_intr(sc);
        virtio_stop(sc->vtscsi_dev);
}

static int
vtscsi_reset_bus(struct vtscsi_softc *sc)
{
        int error;

        VTSCSI_LOCK_OWNED(sc);

        if (vtscsi_bus_reset_disable != 0) {
                device_printf(sc->vtscsi_dev, "bus reset disabled\n");
                return (0);
        }

        sc->vtscsi_flags |= VTSCSI_FLAG_RESET;

        /*
         * vtscsi_stop() will cause the in-flight requests to be canceled.
         * Those requests are then completed here so CAM will retry them
         * after the reset is complete.
         */
        vtscsi_stop(sc);
        vtscsi_complete_vqs_locked(sc);

        /* Rid the virtqueues of any remaining requests. */
        vtscsi_drain_vqs(sc);

        /*
         * Any resource shortage that froze the SIMQ cannot persist across
         * a bus reset so ensure it gets thawed here.
         */
        if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0)
                xpt_release_simq(sc->vtscsi_sim, 0);

        error = vtscsi_reinit(sc);
        if (error) {
                device_printf(sc->vtscsi_dev,
                    "reinitialization failed, stopping device...\n");
                vtscsi_stop(sc);
        } else
                vtscsi_announce(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
                    CAM_LUN_WILDCARD);

        sc->vtscsi_flags &= ~VTSCSI_FLAG_RESET;

        return (error);
}

static void
vtscsi_init_request(struct vtscsi_softc *sc, struct vtscsi_request *req)
{

#ifdef INVARIANTS
        int req_nsegs, resp_nsegs;

        req_nsegs = sglist_count(&req->vsr_ureq, sizeof(req->vsr_ureq));
        resp_nsegs = sglist_count(&req->vsr_uresp, sizeof(req->vsr_uresp));

        KASSERT(req_nsegs == 1, ("request crossed page boundary"));
        KASSERT(resp_nsegs == 1, ("response crossed page boundary"));
#endif

        req->vsr_softc = sc;
        callout_init_mtx(&req->vsr_callout, VTSCSI_MTX(sc), 0);
}

static int
vtscsi_alloc_requests(struct vtscsi_softc *sc)
{
        struct vtscsi_request *req;
        int i, nreqs;

        /*
         * Commands destined for either the request or control queues come
         * from the same SIM queue. Use the size of the request virtqueue
         * as it (should) be much more frequently used. Some additional
         * requests are allocated for internal (TMF) use.
         */
        nreqs = virtqueue_size(sc->vtscsi_request_vq);
        if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0)
                nreqs /= VTSCSI_MIN_SEGMENTS;
        nreqs += VTSCSI_RESERVED_REQUESTS;

        for (i = 0; i < nreqs; i++) {
                req = malloc(sizeof(struct vtscsi_request), M_DEVBUF,
                    M_NOWAIT);
                if (req == NULL)
                        return (ENOMEM);

                vtscsi_init_request(sc, req);

                sc->vtscsi_nrequests++;
                vtscsi_enqueue_request(sc, req);
        }

        return (0);
}

static void
vtscsi_free_requests(struct vtscsi_softc *sc)
{
        struct vtscsi_request *req;

        while ((req = vtscsi_dequeue_request(sc)) != NULL) {
                KASSERT(callout_active(&req->vsr_callout) == 0,
                    ("request callout still active"));

                sc->vtscsi_nrequests--;
                free(req, M_DEVBUF);
        }

        KASSERT(sc->vtscsi_nrequests == 0, ("leaked requests: %d",
            sc->vtscsi_nrequests));
}

static void
vtscsi_enqueue_request(struct vtscsi_softc *sc, struct vtscsi_request *req)
{

        KASSERT(req->vsr_softc == sc,
            ("non-matching request vsr_softc %p/%p", req->vsr_softc, sc));

        vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req);

        /* A request is available so the SIMQ could be released. */
        if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST) != 0)
                xpt_release_simq(sc->vtscsi_sim, 1);

        req->vsr_ccb = NULL;
        req->vsr_complete = NULL;
        req->vsr_ptr0 = NULL;
        req->vsr_state = VTSCSI_REQ_STATE_FREE;
        req->vsr_flags = 0;

        bzero(&req->vsr_ureq, sizeof(req->vsr_ureq));
        bzero(&req->vsr_uresp, sizeof(req->vsr_uresp));

        /*
         * We insert at the tail of the queue in order to make it
         * very unlikely a request will be reused if we race with
         * stopping its callout handler.
         */
        TAILQ_INSERT_TAIL(&sc->vtscsi_req_free, req, vsr_link);
}

static struct vtscsi_request *
vtscsi_dequeue_request(struct vtscsi_softc *sc)
{
        struct vtscsi_request *req;

        req = TAILQ_FIRST(&sc->vtscsi_req_free);
        if (req != NULL) {
                req->vsr_state = VTSCSI_REQ_STATE_INUSE;
                TAILQ_REMOVE(&sc->vtscsi_req_free, req, vsr_link);
        } else
                sc->vtscsi_stats.dequeue_no_requests++;

        vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req);

        return (req);
}

static void
vtscsi_complete_request(struct vtscsi_request *req)
{

        if (req->vsr_flags & VTSCSI_REQ_FLAG_POLLED)
                req->vsr_flags |= VTSCSI_REQ_FLAG_COMPLETE;

        if (req->vsr_complete != NULL)
                req->vsr_complete(req->vsr_softc, req);
}

static void
vtscsi_complete_vq(struct vtscsi_softc *sc, struct virtqueue *vq)
{
        struct vtscsi_request *req;

        VTSCSI_LOCK_OWNED(sc);

        while ((req = virtqueue_dequeue(vq, NULL)) != NULL)
                vtscsi_complete_request(req);
}

static void
vtscsi_control_vq_intr(void *xsc)
{
        struct vtscsi_softc *sc;
        struct virtqueue *vq;

        sc = xsc;
        vq = sc->vtscsi_control_vq;

again:
        VTSCSI_LOCK(sc);

        vtscsi_complete_vq(sc, sc->vtscsi_control_vq);

        if (virtqueue_enable_intr(vq) != 0) {
                virtqueue_disable_intr(vq);
                VTSCSI_UNLOCK(sc);
                goto again;
        }

        VTSCSI_UNLOCK(sc);
}

static void
vtscsi_event_vq_intr(void *xsc)
{
        struct vtscsi_softc *sc;
        struct virtqueue *vq;
        struct virtio_scsi_event *event;

        sc = xsc;
        vq = sc->vtscsi_event_vq;

again:
        VTSCSI_LOCK(sc);

        while ((event = virtqueue_dequeue(vq, NULL)) != NULL)
                vtscsi_handle_event(sc, event);

        if (virtqueue_enable_intr(vq) != 0) {
                virtqueue_disable_intr(vq);
                VTSCSI_UNLOCK(sc);
                goto again;
        }

        VTSCSI_UNLOCK(sc);
}

static void
vtscsi_request_vq_intr(void *xsc)
{
        struct vtscsi_softc *sc;
        struct virtqueue *vq;

        sc = xsc;
        vq = sc->vtscsi_request_vq;

again:
        VTSCSI_LOCK(sc);

        vtscsi_complete_vq(sc, sc->vtscsi_request_vq);

        if (virtqueue_enable_intr(vq) != 0) {
                virtqueue_disable_intr(vq);
                VTSCSI_UNLOCK(sc);
                goto again;
        }

        VTSCSI_UNLOCK(sc);
}

static void
vtscsi_disable_vqs_intr(struct vtscsi_softc *sc)
{

        virtqueue_disable_intr(sc->vtscsi_control_vq);
        virtqueue_disable_intr(sc->vtscsi_event_vq);
        virtqueue_disable_intr(sc->vtscsi_request_vq);
}

static void
vtscsi_enable_vqs_intr(struct vtscsi_softc *sc)
{

        virtqueue_enable_intr(sc->vtscsi_control_vq);
        virtqueue_enable_intr(sc->vtscsi_event_vq);
        virtqueue_enable_intr(sc->vtscsi_request_vq);
}

static void
vtscsi_get_tunables(struct vtscsi_softc *sc)
{
        char tmpstr[64];

        TUNABLE_INT_FETCH("hw.vtscsi.debug_level", &sc->vtscsi_debug);

        snprintf(tmpstr, sizeof(tmpstr), "dev.vtscsi.%d.debug_level",
            device_get_unit(sc->vtscsi_dev));
        TUNABLE_INT_FETCH(tmpstr, &sc->vtscsi_debug);
}

static void
vtscsi_setup_sysctl(struct vtscsi_softc *sc)
{
        device_t dev;
        struct vtscsi_statistics *stats;
        struct sysctl_ctx_list *ctx;
        struct sysctl_oid *tree;
        struct sysctl_oid_list *child;

        dev = sc->vtscsi_dev;
        stats = &sc->vtscsi_stats;
        ctx = device_get_sysctl_ctx(dev);
        tree = device_get_sysctl_tree(dev);
        child = SYSCTL_CHILDREN(tree);

        SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level",
            CTLFLAG_RW, &sc->vtscsi_debug, 0,
            "Debug level");

        SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "scsi_cmd_timeouts",
            CTLFLAG_RD, &stats->scsi_cmd_timeouts,
            "SCSI command timeouts");
        SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dequeue_no_requests",
            CTLFLAG_RD, &stats->dequeue_no_requests,
            "No available requests to dequeue");
}

static void
vtscsi_printf_req(struct vtscsi_request *req, const char *func,
    const char *fmt, ...)
{
        struct vtscsi_softc *sc;
        union ccb *ccb;
        struct sbuf sb;
        va_list ap;
        char str[192];

        if (req == NULL)
                return;

        sc = req->vsr_softc;
        ccb = req->vsr_ccb;

        va_start(ap, fmt);
        sbuf_new(&sb, str, sizeof(str), 0);

        if (ccb == NULL) {
                sbuf_printf(&sb, "(noperiph:%s%d:%u): ",
                    cam_sim_name(sc->vtscsi_sim), cam_sim_unit(sc->vtscsi_sim),
                    cam_sim_bus(sc->vtscsi_sim));
        } else {
                xpt_path_sbuf(ccb->ccb_h.path, &sb);
                if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
                        scsi_command_string(&ccb->csio, &sb);
                        sbuf_printf(&sb, "length %d ", ccb->csio.dxfer_len);
                }
        }

        sbuf_vprintf(&sb, fmt, ap);
        va_end(ap);

        sbuf_finish(&sb);
        printf("%s: %s: %s", device_get_nameunit(sc->vtscsi_dev), func,
            sbuf_data(&sb));
}