/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2016 Nexenta Systems, Inc.
 * Copyright 2022 RackTop Systems, Inc.
 */

#include <sys/atomic.h>
#include <sys/cmn_err.h>
#include <sys/cpuvar.h>
#include <sys/ddi.h>
#include <sys/id32.h>
#include <sys/kmem.h>
#include <sys/list.h>
#include <sys/modctl.h>
#include <sys/pci.h>
#include <sys/scsi/scsi.h>
#include <sys/sunddi.h>
#include <sys/sysmacros.h>
#include <sys/types.h>
#include <sys/note.h>

#include "pvscsi.h"
#include "pvscsi_var.h"

/* we can support any of the interrupt types */
int pvscsi_intr_types = \
        DDI_INTR_TYPE_MSIX|DDI_INTR_TYPE_MSI|DDI_INTR_TYPE_FIXED;
int pvscsi_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_PER_RING;
int pvscsi_msg_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_MSG_RING;
static int pvscsi_hz;

static int pvscsi_abort(struct scsi_address *, struct scsi_pkt *);
static void pvscsi_timeout(void *);
static void pvscsi_setup_rings(pvscsi_softc_t *);
static void pvscsi_complete_cmds(pvscsi_softc_t *, pvscsi_cmd_t *);
static boolean_t pvscsi_cmd_init(pvscsi_softc_t *, pvscsi_cmd_t *, int);
static void pvscsi_cmd_fini(pvscsi_cmd_t *);

/* HBA DMA attributes */
static ddi_dma_attr_t pvscsi_dma_attr = {
        .dma_attr_version =     DMA_ATTR_V0,
        .dma_attr_addr_lo =     0,
        .dma_attr_addr_hi =     0xFFFFFFFFFFFFFFFFull,
        .dma_attr_count_max =   0xFFFFFFFFFFFFFFFFull,
        .dma_attr_align =       PAGE_SIZE,
        .dma_attr_burstsizes =  1,
        .dma_attr_minxfer =     1,
        .dma_attr_maxxfer =     0xFFFFFFFFFFFFFFFFull,
        .dma_attr_seg =         0xFFFFFFFFFFFFFFFFull,
        .dma_attr_sgllen =      1,
        .dma_attr_granular =    1,
        .dma_attr_flags =       0
};

/* DMA attributes for buffer I/O */
static ddi_dma_attr_t pvscsi_io_dma_attr = {
        .dma_attr_version =     DMA_ATTR_V0,
        .dma_attr_addr_lo =     0,
        .dma_attr_addr_hi =     0xFFFFFFFFFFFFFFFFull,
        .dma_attr_count_max =   0x7FFFFFFFll,
        .dma_attr_align =       1,
        .dma_attr_burstsizes =  1,
        .dma_attr_minxfer =     1,
        .dma_attr_maxxfer =     PAGE_SIZE * PVSCSI_MAX_SG_SIZE,
        .dma_attr_seg =         0xFFFFFFFFFFFFFFFFull,
        .dma_attr_sgllen =      PVSCSI_MAX_SG_SIZE,
        .dma_attr_granular =    1,
        .dma_attr_flags =       0
};

/*
 * The structures are always little endian (VMware only runs
 * on little endian CPUs), but we only run on LE processors,
 * and NEVERSWAP avoids needing to use DDI accessor functions.
 * (It would be incredibly bizarre to have a VMware guest running
 * with a different endianness than the hypervisor.)
 */
static ddi_device_acc_attr_t pvscsi_mmio_attr = {
        .devacc_attr_version =          DDI_DEVICE_ATTR_V1,
        .devacc_attr_endian_flags =     DDI_NEVERSWAP_ACC,
        .devacc_attr_dataorder =        DDI_STRICTORDER_ACC,
        .devacc_attr_access =           DDI_DEFAULT_ACC
};

static ddi_device_acc_attr_t pvscsi_dma_attrs = {
        .devacc_attr_version =          DDI_DEVICE_ATTR_V1,
        .devacc_attr_endian_flags =     DDI_NEVERSWAP_ACC,
        .devacc_attr_dataorder =        DDI_STRICTORDER_ACC,
        .devacc_attr_access =           DDI_DEFAULT_ACC,
};

static void
pvscsi_add_to_queue(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
{
        pvscsi_cmd_t    *r;
        list_t          *l;

        /*
         * We insert in order of expiration, with the earliest
         * expirations at the front.  This logic assumes that most
         * commands will have the same timeout, and is optimized
         * to minimize walking the list.  It allows timeouts to
         * run without looking at more than one node that has not
         * yet expired.
         */
        ASSERT(mutex_owned(&pvs->lock));

        l = &pvs->cmd_queue;
        for (r = list_tail(l); r != NULL; r = list_prev(l, r)) {
                /* this subtraction is safe if lbolt wraps */
                if (((cmd->start + cmd->timeout) -
                    (r->start + r->timeout)) >= 0) {
                        list_insert_after(l, r, cmd);
                        return;
                }
        }

        list_insert_head(l, cmd);
}

static uint32_t
pvscsi_reg_read(pvscsi_softc_t *pvs, uint32_t offset)
{
        uint32_t        ret;

        ASSERT((offset & (sizeof (uint32_t) - 1)) == 0);

        ret = ddi_get32(pvs->mmio_handle,
            (uint32_t *)(pvs->mmio_base + offset));

        return (ret);
}

static void
pvscsi_reg_write(pvscsi_softc_t *pvs, uint32_t offset, uint32_t value)
{
        ASSERT((offset & (sizeof (uint32_t) - 1)) == 0);

        ddi_put32(pvs->mmio_handle, (uint32_t *)(pvs->mmio_base + offset),
            value);
}

static void
pvscsi_write_cmd_desc(pvscsi_softc_t *pvs, uint32_t cmd, void *desc, size_t len)
{
        len /= sizeof (uint32_t);
        pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_COMMAND, cmd);
        ddi_rep_put32(pvs->mmio_handle, (uint32_t *)desc,
            (uint32_t *)(pvs->mmio_base + PVSCSI_REG_OFFSET_COMMAND_DATA),
            len, DDI_DEV_NO_AUTOINCR);
}

static uint32_t
pvscsi_read_intr_status(pvscsi_softc_t *pvs)
{
        return (pvscsi_reg_read(pvs, PVSCSI_REG_OFFSET_INTR_STATUS));
}

static void
pvscsi_write_intr_status(pvscsi_softc_t *pvs, uint32_t val)
{
        pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_STATUS, val);
}

static pvscsi_cmd_t *
pvscsi_reclaim_cmds(pvscsi_softc_t *pvs)
{
        pvscsi_cmd_t    *head = NULL;
        pvscsi_cmd_t    **tail = &head;
        pvscsi_cmd_t    *cmd;

        ASSERT(mutex_owned(&pvs->lock));
        while ((cmd = list_remove_head(&pvs->cmd_queue)) != NULL) {
                list_remove(&pvs->cmd_queue, cmd);
                *tail = cmd;
                tail = &cmd->next_cmd;
                *tail = NULL;
                cmd->host_status = BTSTAT_BUSRESET;
        }
        return (head);
}

static void
pvscsi_stop_hba(pvscsi_softc_t *pvs)
{
        pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_ADAPTER_RESET, NULL, 0);
        pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK, 0);
        /* read interrupt status to flush PCI write buffers */
        (void) pvscsi_read_intr_status(pvs);
}

static void
pvscsi_start_hba(pvscsi_softc_t *pvs)
{
        pvscsi_setup_rings(pvs);
        pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK,
            PVSCSI_INTR_CMPL_MASK | PVSCSI_INTR_MSG_MASK);
}

static void
pvscsi_reset_bus(pvscsi_softc_t *pvs)
{
        pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_RESET_BUS, NULL, 0);
}

/*
 * pvscsi_restart_hba resets the HBA, and reconfigures it.  It also
 * completes all commands that have not been already completed with
 * a reset.
 */
static void
pvscsi_restart_hba(pvscsi_softc_t *pvs)
{
        pvscsi_cmd_t    *cmd;

        mutex_enter(&pvs->lock);
        pvscsi_stop_hba(pvs);
        cmd = pvscsi_reclaim_cmds(pvs);
        pvscsi_start_hba(pvs);
        mutex_exit(&pvs->lock);

        /* run the completions from the reclaimed commands */
        pvscsi_complete_cmds(pvs, cmd);
}

static void
pvscsi_submit_nonrw_io(pvscsi_softc_t *pvs)
{
        pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0);
}

static void
pvscsi_submit_rw_io(pvscsi_softc_t *pvs)
{
        pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_KICK_RW_IO, 0);
}

static pvscsi_cmd_t *
pvscsi_process_comp_ring(pvscsi_softc_t *pvs)
{
        pvscsi_cmd_t    **pnext_cmd;
        pvscsi_cmd_t    *cmd;
        pvscsi_cmd_t    *head = NULL;
        struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
        uint32_t        cmp_ne = sdesc->cmpNumEntriesLog2;

        ASSERT(mutex_owned(&pvs->lock));

        pnext_cmd = &head;

        (void) ddi_dma_sync(pvs->state_buf.dmah, 0, 0, DDI_DMA_SYNC_FORKERNEL);

        while (sdesc->cmpConsIdx != sdesc->cmpProdIdx) {
                struct PVSCSIRingCmpDesc *cdesc;

                (void) ddi_dma_sync(pvs->cmp_ring_buf.dmah, 0, 0,
                    DDI_DMA_SYNC_FORKERNEL);

                cdesc = CMP_RING(pvs) + (sdesc->cmpConsIdx & MASK(cmp_ne));

                if ((cmd = id32_lookup((uint32_t)cdesc->context)) != NULL) {
                        cmd->next_cmd = NULL;

                        /* Save command status for further processing */
                        cmd->host_status = cdesc->hostStatus;
                        cmd->scsi_status = cdesc->scsiStatus;
                        cmd->transferred = cdesc->dataLen;

                        *pnext_cmd = cmd;
                        pnext_cmd = &cmd->next_cmd;

                        list_remove(&pvs->cmd_queue, cmd);
                }

                sdesc->cmpConsIdx++;
        }
        (void) ddi_dma_sync(pvs->state_buf.dmah, 0, 0, DDI_DMA_SYNC_FORDEV);

        return (head);
}

static pvscsi_msg_t *
pvscsi_process_msg_ring(pvscsi_softc_t *pvs)
{
        pvscsi_msg_t    *msg;
        struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
        struct PVSCSIRingMsgDesc *mdesc;
        struct PVSCSIMsgDescDevStatusChanged *desc;
        uint32_t        msg_ne = sdesc->msgNumEntriesLog2;

        (void) ddi_dma_sync(pvs->state_buf.dmah, 0, 0, DDI_DMA_SYNC_FORKERNEL);

        if (sdesc->msgProdIdx == sdesc->msgConsIdx) {
                return (NULL);
        }

        (void) ddi_dma_sync(pvs->msg_ring_buf.dmah, 0, 0,
            DDI_DMA_SYNC_FORKERNEL);

        mdesc = MSG_RING(pvs) + (sdesc->msgConsIdx & MASK(msg_ne));

        switch (mdesc->type) {
        case PVSCSI_MSG_DEV_ADDED:
        case PVSCSI_MSG_DEV_REMOVED:
                desc = (struct PVSCSIMsgDescDevStatusChanged *)mdesc;
                msg = kmem_alloc(sizeof (pvscsi_msg_t), KM_NOSLEEP);
                if (msg == NULL)
                        return (NULL);
                msg->pvs = pvs;
                msg->type = mdesc->type;
                msg->target = desc->target;
                msg->lun = desc->lun[1]; /* T10 format */
                break;
        default:
                dev_err(pvs->dip, CE_WARN, "!unknown msg type: %d",
                    mdesc->type);
                return (NULL);
        }

        sdesc->msgConsIdx++;
        (void) ddi_dma_sync(pvs->state_buf.dmah, 0, 0, DDI_DMA_SYNC_FORDEV);
        return (msg);
}

static void
pvscsi_handle_msg(void *arg)
{
        pvscsi_msg_t    *msg = arg;
        pvscsi_softc_t  *pvs = msg->pvs;
        char            addr[8];

        (void) snprintf(addr, sizeof (addr), "%x", msg->target);

        if (msg->lun == 0) {
                switch (msg->type) {
                case PVSCSI_MSG_DEV_ADDED:
                        (void) scsi_hba_tgtmap_tgt_add(pvs->tgtmap,
                            SCSI_TGT_SCSI_DEVICE, addr, NULL);
                        break;
                case PVSCSI_MSG_DEV_REMOVED:
                        (void) scsi_hba_tgtmap_tgt_remove(pvs->tgtmap,
                            SCSI_TGT_SCSI_DEVICE, addr);
                        break;
                }
        } else {
                scsi_hba_tgtmap_scan_luns(pvs->tgtmap, addr);
        }
        kmem_free(msg, sizeof (pvscsi_msg_t));
}

static void
pvscsi_abort_cmd(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
{
        struct PVSCSICmdDescAbortCmd    acmd;

        bzero(&acmd, sizeof (acmd));
        acmd.target = cmd->target;
        acmd.context = cmd->ctx;
        pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_ABORT_CMD, &acmd, sizeof (acmd));
}

static void
pvscsi_map_buffers(pvscsi_cmd_t *cmd, struct PVSCSIRingReqDesc *rdesc)
{
        struct scsi_pkt *pkt = cmd->pkt;

        rdesc->dataLen = 0;
        rdesc->dataAddr = 0;
        if (pkt == NULL || pkt->pkt_numcookies == 0) {
                return;
        }

        pkt->pkt_resid = 0;

        if (pkt->pkt_numcookies > 1) {
                size_t  len = 0;
                struct PVSCSISGElement *sgl = cmd->sgl;

                for (uint_t i = 0; i < pkt->pkt_numcookies; i++) {
                        sgl[i].addr = pkt->pkt_cookies[i].dmac_laddress;
                        sgl[i].length = pkt->pkt_cookies[i].dmac_size;
                        sgl[i].flags = 0;
                        len += pkt->pkt_cookies[i].dmac_size;
                }
                rdesc->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST;
                rdesc->dataAddr = cmd->sgl_pa;
                rdesc->dataLen = len;
                (void) ddi_dma_sync(cmd->sgl_dmah, 0, 0, DDI_DMA_SYNC_FORDEV);
        } else {
                rdesc->flags = 0;
                rdesc->dataAddr = pkt->pkt_cookies[0].dmac_laddress;
                rdesc->dataLen = pkt->pkt_cookies[0].dmac_size;
        }
        pkt->pkt_resid = rdesc->dataLen;
}

static void
pvscsi_comp_cmd(pvscsi_cmd_t *cmd)
{
        struct scsi_pkt *pkt = cmd->pkt;
        uint8_t         status = cmd->scsi_status;

        pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD |
            STATE_GOT_STATUS);
        if (pkt->pkt_numcookies > 0) {
                pkt->pkt_state |= STATE_XFERRED_DATA;
        }
        pkt->pkt_reason = CMD_CMPLT;
        pkt->pkt_resid -= cmd->transferred;
        *(pkt->pkt_scbp) = status;

        if (status == STATUS_CHECK) {
                /*
                 * Our virtual HBA *always* does ARQ, and it never
                 * is more than 20 bytes, so no need to try to handle
                 * extended versions of it.
                 */
                struct scsi_arq_status *ars = (void *)(pkt->pkt_scbp);
                int             len = min(pkt->pkt_scblen, SENSE_LENGTH);

                pkt->pkt_state |= STATE_ARQ_DONE;
                ars->sts_rqpkt_resid = 0;
                bcopy(cmd->arq_sense, &ars->sts_sensedata, len);
                ars->sts_rqpkt_reason = CMD_CMPLT;
                *(uint8_t *)&ars->sts_rqpkt_status = STATUS_GOOD;
                ars->sts_rqpkt_state = STATE_GOT_BUS |
                    STATE_GOT_TARGET | STATE_SENT_CMD |
                    STATE_XFERRED_DATA | STATE_GOT_STATUS;
        }
}

static void
pvscsi_set_status(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
{
        struct scsi_pkt *pkt = cmd->pkt;
        uint32_t        host_status = cmd->host_status;

        switch (host_status) {
        case BTSTAT_SUCCESS:
        case BTSTAT_LINKED_COMMAND_COMPLETED:
        case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG:
                pvscsi_comp_cmd(cmd);
                break;
        case BTSTAT_DATARUN:
                pkt->pkt_reason = CMD_DATA_OVR;
                pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
                    STATE_SENT_CMD | STATE_GOT_STATUS |
                    STATE_XFERRED_DATA);
                pkt->pkt_resid -= cmd->transferred;
                break;
        case BTSTAT_SELTIMEO:
                pkt->pkt_reason = CMD_DEV_GONE;
                pkt->pkt_state |= STATE_GOT_BUS;
                break;
        case BTSTAT_TAGREJECT:
                pkt->pkt_reason = CMD_TAG_REJECT;
                pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
                    STATE_SENT_CMD | STATE_GOT_STATUS);
                break;
        case BTSTAT_BADMSG:
                pkt->pkt_reason = CMD_BADMSG;
                pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
                    STATE_SENT_CMD | STATE_GOT_STATUS);
                break;
        case BTSTAT_SENTRST:
        case BTSTAT_RECVRST:
                pkt->pkt_reason = CMD_RESET;
                pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
                    STATE_SENT_CMD | STATE_GOT_STATUS);
                pkt->pkt_statistics |= STAT_DEV_RESET;
                pkt->pkt_resid -= cmd->transferred;
                break;
        case BTSTAT_BUSRESET:
                pkt->pkt_reason = CMD_RESET;
                pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
                    STATE_SENT_CMD | STATE_GOT_STATUS);
                pkt->pkt_statistics |= STAT_BUS_RESET;
                pkt->pkt_resid -= cmd->transferred;
                break;
        case BTSTAT_ABORTQUEUE:
                if (cmd->expired) {
                        pkt->pkt_reason = CMD_TIMEOUT;
                        pkt->pkt_statistics |= STAT_TIMEOUT;
                } else {
                        pkt->pkt_reason = CMD_ABORTED;
                        pkt->pkt_statistics |= STAT_ABORTED;
                }
                pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
                    STATE_SENT_CMD | STATE_GOT_STATUS);
                pkt->pkt_resid -= cmd->transferred;
                break;
        case BTSTAT_HAHARDWARE:
        case BTSTAT_INVPHASE:
        case BTSTAT_HATIMEOUT:
        case BTSTAT_NORESPONSE:
        case BTSTAT_DISCONNECT:
        case BTSTAT_HASOFTWARE:
        case BTSTAT_BUSFREE:
        case BTSTAT_SENSFAILED:
        case BTSTAT_DATA_UNDERRUN:
                pkt->pkt_reason = CMD_TRAN_ERR;
                pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
                    STATE_SENT_CMD | STATE_GOT_STATUS);
                pkt->pkt_resid -= cmd->transferred;
                break;
        default:
                dev_err(pvs->dip, CE_WARN,
                    "!unknown host status code: %d", host_status);
                pkt->pkt_reason = CMD_TRAN_ERR;
                pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
                    STATE_SENT_CMD | STATE_GOT_STATUS);
                break;
        }
}

/*
 * pvscsi_complete_cmds processes a linked list of
 * commands that have been completed.  This is done
 * without acquiring any locks.
 */
static void
pvscsi_complete_cmds(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
{
        struct scsi_pkt *pkt;

        while (cmd != NULL) {
                pvscsi_cmd_t    *next = cmd->next_cmd;

                cmd->next_cmd = NULL;

                if (((pkt = cmd->pkt) == NULL) || (cmd->poll)) {
                        atomic_or_8(&cmd->done, 1);
                } else {
                        pvscsi_set_status(pvs, cmd);
                        scsi_hba_pkt_comp(pkt);
                }

                cmd = next;
        }
}

static void
pvscsi_dev_reset(pvscsi_softc_t *pvs, int target, int lun)
{
        struct PVSCSICmdDescResetDevice cmd = { 0 };

        cmd.target = target;
        cmd.lun[1] = lun & 0xff;
        pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_RESET_DEVICE, &cmd, sizeof (cmd));
}

static boolean_t
pvscsi_poll_cmd_until(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd, clock_t usec)
{
        while (usec > 0) {
                pvscsi_cmd_t    *done;
                if (cmd->done) {
                        return (B_TRUE);
                }
                mutex_enter(&pvs->lock);
                done = pvscsi_process_comp_ring(pvs);
                mutex_exit(&pvs->lock);

                pvscsi_complete_cmds(pvs, done);
                drv_usecwait(10);
                usec -= 10;
        }

        return (B_FALSE);
}

static void
pvscsi_poll_cmd(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
{
        if (pvscsi_poll_cmd_until(pvs, cmd, drv_hztousec(cmd->timeout))) {
                return;
        }

        /* now we try an abort first */
        pvscsi_abort_cmd(pvs, cmd);
        if (pvscsi_poll_cmd_until(pvs, cmd, 2)) {
                return;
        }
        /* well that failed... try reset */
        pvscsi_dev_reset(pvs, cmd->target, cmd->lun);
        if (pvscsi_poll_cmd_until(pvs, cmd, 2)) {
                return;
        }
        /* still trying... reset the bus */
        pvscsi_reset_bus(pvs);
        if (pvscsi_poll_cmd_until(pvs, cmd, 2)) {
                return;
        }
        /* full up adapter reset -- be brutal */
        pvscsi_restart_hba(pvs);
}

static void
pvscsi_abort_all(pvscsi_softc_t *pvs, pvscsi_device_t *pd)
{
        pvscsi_cmd_t    *cmd;

        mutex_enter(&pvs->lock);
        list_t *l = &pvs->cmd_queue;
        for (cmd = list_head(l); cmd != NULL; cmd = list_next(l, cmd)) {
                if ((pd->target == cmd->target) && (pd->lun == cmd->lun)) {
                        pvscsi_abort_cmd(pvs, cmd);
                }
        }
        mutex_exit(&pvs->lock);
}

static int
pvscsi_transport_command(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
{
        struct PVSCSIRingReqDesc        *rdesc;
        struct PVSCSIRingsState         *sdesc = RINGS_STATE(pvs);
        uint32_t                        req_ne = sdesc->reqNumEntriesLog2;

        cmd->done = 0;
        cmd->expired = 0;

        mutex_enter(&pvs->lock);

        if ((sdesc->reqProdIdx - sdesc->cmpConsIdx) >= (1 << req_ne)) {
                mutex_exit(&pvs->lock);
                return (TRAN_BUSY);
        }

        rdesc = REQ_RING(pvs) + (sdesc->reqProdIdx & MASK(req_ne));

        rdesc->bus = 0;
        rdesc->target = cmd->target;
        bzero(rdesc->lun, sizeof (rdesc->lun));
        /* Matches other implementations; can pvscsi support luns > 255? */
        rdesc->lun[1] = cmd->lun & 0xff;

        bzero(cmd->arq_sense, sizeof (cmd->arq_sense));
        rdesc->context = cmd->ctx;
        rdesc->senseLen = sizeof (cmd->arq_sense);
        rdesc->senseAddr = cmd->arq_pa;
        rdesc->tag = cmd->tag;
        rdesc->vcpuHint = CPU->cpu_id;
        rdesc->cdbLen = cmd->cdblen;
        rdesc->flags = cmd->dma_dir;
        bcopy(cmd->cdb, rdesc->cdb, cmd->cdblen);
        pvscsi_map_buffers(cmd, rdesc);

        (void) ddi_dma_sync(pvs->req_ring_buf.dmah, 0, 0, DDI_DMA_SYNC_FORDEV);

        sdesc->reqProdIdx++;
        (void) ddi_dma_sync(pvs->state_buf.dmah, 0, 0, DDI_DMA_SYNC_FORDEV);

        pvscsi_add_to_queue(pvs, cmd);

        switch (cmd->cdb[0]) {
        case SCMD_READ:
        case SCMD_WRITE:
        case SCMD_READ_G1:
        case SCMD_WRITE_G1:
        case SCMD_READ_G4:
        case SCMD_WRITE_G4:
        case SCMD_READ_G5:
        case SCMD_WRITE_G5:
                pvscsi_submit_rw_io(pvs);
                break;
        default:
                pvscsi_submit_nonrw_io(pvs);
                break;
        }

        if (pvs->timeout == 0) {
                /* drivers above should supply, but give a default */
                pvs->timeout = timeout(pvscsi_timeout, pvs, pvscsi_hz * 8);
        }
        mutex_exit(&pvs->lock);

        return (TRAN_ACCEPT);
}

static int
pvscsi_setup_dma_buffer(pvscsi_softc_t *pvs, size_t length,
    pvscsi_dma_buf_t *buf)
{
        if ((ddi_dma_alloc_handle(pvs->dip, &pvscsi_dma_attr,
            DDI_DMA_SLEEP, NULL, &buf->dmah)) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to alloc DMA handle");
                return (DDI_FAILURE);
        }

        if ((ddi_dma_mem_alloc(buf->dmah, length, &pvscsi_dma_attrs,
            DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &buf->addr,
            &length, &buf->acch)) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to alloc DMA memory");
                return (DDI_FAILURE);
        }

        if ((ddi_dma_addr_bind_handle(buf->dmah, NULL, buf->addr,
            length, DDI_DMA_CONSISTENT | DDI_DMA_RDWR, DDI_DMA_SLEEP,
            NULL, NULL, NULL)) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to bind DMA buffer");
                return (DDI_FAILURE);
        }

        buf->pa = ddi_dma_cookie_one(buf->dmah)->dmac_laddress;

        return (DDI_SUCCESS);
}

static void
pvscsi_free_dma_buffer(pvscsi_dma_buf_t *buf)
{
        if (buf->pa != 0) {
                (void) ddi_dma_unbind_handle(buf->dmah);
        }
        if (buf->acch != NULL) {
                ddi_dma_mem_free(&buf->acch);
        }
        if (buf->dmah != NULL) {
                ddi_dma_free_handle(&buf->dmah);
        }
}

static int
pvscsi_allocate_rings(pvscsi_softc_t *pvs)
{
        /* allocate DMA buffer for rings state */
        if (pvscsi_setup_dma_buffer(pvs, PAGE_SIZE, &pvs->state_buf) !=
            DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        /* allocate DMA buffer for request ring */
        pvs->req_pages = MIN(pvscsi_ring_pages, PVSCSI_MAX_NUM_PAGES_REQ_RING);
        pvs->req_depth = pvs->req_pages * PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
        if (pvscsi_setup_dma_buffer(pvs, pvs->req_pages * PAGE_SIZE,
            &pvs->req_ring_buf) != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        /* allocate completion ring */
        pvs->cmp_pages = MIN(pvscsi_ring_pages, PVSCSI_MAX_NUM_PAGES_CMP_RING);
        if (pvscsi_setup_dma_buffer(pvs, pvs->cmp_pages * PAGE_SIZE,
            &pvs->cmp_ring_buf) != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        /* allocate message ring */
        pvs->msg_pages = MIN(pvscsi_msg_ring_pages,
            PVSCSI_MAX_NUM_PAGES_MSG_RING);
        if (pvscsi_setup_dma_buffer(pvs, pvs->msg_pages * PAGE_SIZE,
            &pvs->msg_ring_buf) != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        return (DDI_SUCCESS);
}

static void
pvscsi_free_rings(pvscsi_softc_t *pvs)
{
        pvscsi_free_dma_buffer(&pvs->msg_ring_buf);
        pvscsi_free_dma_buffer(&pvs->cmp_ring_buf);
        pvscsi_free_dma_buffer(&pvs->req_ring_buf);
        pvscsi_free_dma_buffer(&pvs->state_buf);
}

static void
pvscsi_setup_rings(pvscsi_softc_t *pvs)
{
        int             i;
        struct PVSCSICmdDescSetupMsgRing cmd_msg = { 0 };
        struct PVSCSICmdDescSetupRings cmd = { 0 };
        uint64_t        base;

        cmd.ringsStatePPN = pvs->state_buf.pa >> PAGE_SHIFT;
        cmd.reqRingNumPages = pvs->req_pages;
        cmd.cmpRingNumPages = pvs->cmp_pages;

        /* Setup request ring */
        base = pvs->req_ring_buf.pa;
        for (i = 0; i < pvs->req_pages; i++) {
                cmd.reqRingPPNs[i] = base >> PAGE_SHIFT;
                base += PAGE_SIZE;
        }

        /* Setup completion ring */
        base = pvs->cmp_ring_buf.pa;
        for (i = 0; i < pvs->cmp_pages; i++) {
                cmd.cmpRingPPNs[i] = base >> PAGE_SHIFT;
                base += PAGE_SIZE;
        }

        bzero(RINGS_STATE(pvs), PAGE_SIZE);
        bzero(REQ_RING(pvs), pvs->req_pages * PAGE_SIZE);
        bzero(CMP_RING(pvs), pvs->cmp_pages * PAGE_SIZE);

        /* Issue SETUP command */
        pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_SETUP_RINGS, &cmd, sizeof (cmd));

        /* Setup message ring */
        cmd_msg.numPages = pvs->msg_pages;
        base = pvs->msg_ring_buf.pa;

        for (i = 0; i < pvs->msg_pages; i++) {
                cmd_msg.ringPPNs[i] = base >> PAGE_SHIFT;
                base += PAGE_SIZE;
        }
        bzero(MSG_RING(pvs), pvs->msg_pages * PAGE_SIZE);

        pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_SETUP_MSG_RING, &cmd_msg,
            sizeof (cmd_msg));
}

static int
pvscsi_setup_io(pvscsi_softc_t *pvs)
{
        int             offset, rcount, rn, type;
        int             ret = DDI_FAILURE;
        off_t           regsize;
        pci_regspec_t   *regs;
        uint_t          regs_length;

        if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, pvs->dip,
            DDI_PROP_DONTPASS, "reg", (int **)&regs,
            &regs_length) != DDI_PROP_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to lookup 'reg' property");
                return (DDI_FAILURE);
        }

        rcount = regs_length * sizeof (int) / sizeof (pci_regspec_t);

        for (offset = PCI_CONF_BASE0; offset <= PCI_CONF_BASE5; offset += 4) {
                for (rn = 0; rn < rcount; ++rn) {
                        if (PCI_REG_REG_G(regs[rn].pci_phys_hi) == offset) {
                                type = regs[rn].pci_phys_hi & PCI_ADDR_MASK;
                                break;
                        }
                }

                if (rn >= rcount)
                        continue;

                if (type != PCI_ADDR_IO) {
                        if (ddi_dev_regsize(pvs->dip, rn,
                            &regsize) != DDI_SUCCESS) {
                                dev_err(pvs->dip, CE_WARN,
                                    "!failed to get size of reg %d", rn);
                                goto out;
                        }
                        if (regsize == PVSCSI_MEM_SPACE_SIZE) {
                                if (ddi_regs_map_setup(pvs->dip, rn,
                                    &pvs->mmio_base, 0, 0,
                                    &pvscsi_mmio_attr,
                                    &pvs->mmio_handle) != DDI_SUCCESS) {
                                        dev_err(pvs->dip, CE_WARN,
                                            "!failed to map MMIO BAR");
                                        goto out;
                                }
                                ret = DDI_SUCCESS;
                                break;
                        }
                }
        }

out:
        ddi_prop_free(regs);

        return (ret);
}

static int
pvscsi_enable_intrs(pvscsi_softc_t *pvs)
{
        int     i, rc, intr_caps;

        if ((rc = ddi_intr_get_cap(pvs->intr_handles[0], &intr_caps)) !=
            DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to get interrupt caps");
                return (DDI_FAILURE);
        }

        if ((intr_caps & DDI_INTR_FLAG_BLOCK) != 0) {
                if ((rc = ddi_intr_block_enable(pvs->intr_handles,
                    pvs->intr_cnt)) != DDI_SUCCESS) {
                        dev_err(pvs->dip, CE_WARN,
                            "!failed to enable interrupt block");
                }
        } else {
                for (i = 0; i < pvs->intr_cnt; i++) {
                        if ((rc = ddi_intr_enable(pvs->intr_handles[i])) ==
                            DDI_SUCCESS)
                                continue;
                        dev_err(pvs->dip, CE_WARN,
                            "!failed to enable interrupt");
                        while (--i >= 0)
                                (void) ddi_intr_disable(pvs->intr_handles[i]);
                        break;
                }
        }

        return (rc);
}

static uint32_t
pvscsi_intr(caddr_t arg1, caddr_t arg2)
{
        pvscsi_softc_t  *pvs = (pvscsi_softc_t *)arg1;
        uint32_t        status;
        pvscsi_cmd_t    *cmd;
        pvscsi_msg_t    *msg;
        uint32_t        rv = DDI_INTR_CLAIMED;
        _NOTE(ARGUNUSED(arg2));

        mutex_enter(&pvs->lock);
        status = pvscsi_read_intr_status(pvs);
        if ((status & PVSCSI_INTR_ALL_SUPPORTED) != 0) {
                pvscsi_write_intr_status(pvs, status);
        } else if (pvs->intr_type == DDI_INTR_TYPE_FIXED) {
                rv = DDI_INTR_UNCLAIMED;
        }
        if (pvs->detach) {
                mutex_exit(&pvs->lock);
                return (rv);
        }
        cmd = pvscsi_process_comp_ring(pvs);
        msg = pvscsi_process_msg_ring(pvs);

        /*
         * Do this under the lock, so that we won't dispatch
         * if we are detaching
         */
        if (msg != NULL) {
                if (ddi_taskq_dispatch(pvs->tq, pvscsi_handle_msg, msg,
                    DDI_NOSLEEP) != DDI_SUCCESS) {
                        dev_err(pvs->dip, CE_WARN,
                            "!failed to dispatch discovery");
                }
        }
        mutex_exit(&pvs->lock);

        pvscsi_complete_cmds(pvs, cmd);

        return (rv);
}

static void
pvscsi_free_intrs(pvscsi_softc_t *pvs)
{
        for (int i = 0; i < pvs->intr_cnt; i++) {
                (void) ddi_intr_disable(pvs->intr_handles[i]);
                (void) ddi_intr_remove_handler(pvs->intr_handles[i]);
                (void) ddi_intr_free(pvs->intr_handles[i]);
        }
        pvs->intr_cnt = 0;
}

static int
pvscsi_register_isr(pvscsi_softc_t *pvs, int type)
{
        int     navail, nactual;
        int     i;

        if (ddi_intr_get_navail(pvs->dip, type, &navail) != DDI_SUCCESS ||
            navail == 0) {
                dev_err(pvs->dip, CE_WARN,
                    "!failed to get number of available interrupts of type %d",
                    type);
                return (DDI_FAILURE);
        }
        navail = MIN(navail, PVSCSI_MAX_INTRS);

        if (ddi_intr_alloc(pvs->dip, pvs->intr_handles, type, 0, navail,
            &nactual, DDI_INTR_ALLOC_NORMAL) != DDI_SUCCESS || nactual == 0) {
                dev_err(pvs->dip, CE_WARN, "!failed to allocate %d interrupts",
                    navail);
                return (DDI_FAILURE);
        }

        pvs->intr_cnt = nactual;

        if (ddi_intr_get_pri(pvs->intr_handles[0], (uint_t *)&pvs->intr_pri) !=
            DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to get interrupt priority");
                pvscsi_free_intrs(pvs);
                return (DDI_FAILURE);
        }

        for (i = 0; i < nactual; i++) {
                if (ddi_intr_add_handler(pvs->intr_handles[i], pvscsi_intr,
                    (caddr_t)pvs, NULL) != DDI_SUCCESS) {
                        dev_err(pvs->dip, CE_WARN,
                            "!failed to add intr handler");
                        pvscsi_free_intrs(pvs);
                        return (DDI_FAILURE);
                }
        }

        pvs->intr_type = type;
        return (DDI_SUCCESS);
}

static int
pvscsi_setup_isr(pvscsi_softc_t *pvs)
{
        int     types;

        if (ddi_intr_get_supported_types(pvs->dip, &types) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to get interrupt types");
                return (DDI_FAILURE);
        }

        types &= pvscsi_intr_types;
        if (types == 0) {
                dev_err(pvs->dip, CE_WARN, "!no supported interrupt types");
                return (DDI_FAILURE);
        }


        if (((types & DDI_INTR_TYPE_MSIX) != 0) &&
            (pvscsi_register_isr(pvs, DDI_INTR_TYPE_MSIX) == DDI_SUCCESS)) {
                return (DDI_SUCCESS);
        }
        if (((types & DDI_INTR_TYPE_MSI) != 0) &&
            (pvscsi_register_isr(pvs, DDI_INTR_TYPE_MSI) == DDI_SUCCESS)) {
                return (DDI_SUCCESS);
        }
        if (((types & DDI_INTR_TYPE_FIXED) != 0) &&
            (pvscsi_register_isr(pvs, DDI_INTR_TYPE_FIXED) == DDI_SUCCESS)) {
                return (DDI_SUCCESS);
        }

        dev_err(pvs->dip, CE_WARN, "!failed installing any interrupt handler");
        return (DDI_FAILURE);
}


static void
pvscsi_timeout(void *arg)
{
        pvscsi_softc_t  *pvs;
        pvscsi_cmd_t    *cmd;
        pvscsi_cmd_t    *reclaimed = NULL;
        list_t          *l;
        clock_t         now;

        pvs = arg;
        l = &pvs->cmd_queue;
        now = ddi_get_lbolt();

        mutex_enter(&pvs->lock);
        if (pvs->timeout == 0) {
                mutex_exit(&pvs->lock);
                return;
        }

        for (cmd = list_head(l); cmd != NULL; cmd = list_next(l, cmd)) {
                clock_t overdue;

                /* polling takes care of it's own timeouts */
                if (cmd->poll) {
                        continue;
                }

                overdue = now - (cmd->start + cmd->timeout);

                /*
                 * We keep the list of requests sorted by expiration
                 * time, so we hopefully won't need to walk through
                 * many of these.  This check is safe if lbolt wraps.
                 */
                if (overdue <= 0) {
                        break;
                }

                /* first we try aborting */
                if (!cmd->expired) {
                        atomic_or_8(&cmd->expired, 1);
                        dev_err(pvs->dip, CE_WARN, "!cmd timed out (%lds)",
                            drv_hztousec(cmd->timeout)/1000000);
                        continue;
                }

                /* if we're less than 2 seconds overdue, wait for abort */
                if (overdue <= pvscsi_hz * 2) {
                        continue;
                }

                /* next it's a reset of the device */
                if (overdue <= pvscsi_hz * 8) {
                        pvscsi_dev_reset(pvs, cmd->target, cmd->lun);
                        break;
                }

                /* next it's a reset of the bus */
                if (overdue <= pvscsi_hz * 16) {
                        pvscsi_reset_bus(pvs);
                        break;
                }

                /* finally it's a reset of the entire adapter */
                dev_err(pvs->dip, CE_WARN, "!adapter hung? restarting...");
                mutex_enter(&pvs->lock);
                pvscsi_stop_hba(pvs);
                reclaimed = pvscsi_reclaim_cmds(pvs);
                pvscsi_start_hba(pvs);
                mutex_exit(&pvs->lock);
                break;
        }

        /* see if reset or abort completed anything */
        cmd = pvscsi_process_comp_ring(pvs);

        /* reschedule us if we still have requests pending */
        if (!list_is_empty(l)) {
                pvs->timeout = timeout(pvscsi_timeout, pvs, pvscsi_hz);
        }

        mutex_exit(&pvs->lock);

        /* if we had things that got completed, then do the callbacks */
        pvscsi_complete_cmds(pvs, reclaimed);
        pvscsi_complete_cmds(pvs, cmd);
}

static int
pvscsi_start(struct scsi_address *ap, struct scsi_pkt *pkt)
{
        pvscsi_cmd_t            *cmd = pkt->pkt_ha_private;
        struct scsi_device      *sd;
        pvscsi_device_t         *pd;
        pvscsi_softc_t          *pvs;
        int                     rc;
        boolean_t               poll;

        /* make sure the packet is sane */
        if ((pkt->pkt_numcookies > PVSCSI_MAX_SG_SIZE) ||
            ((pkt->pkt_dma_flags & DDI_DMA_RDWR) == DDI_DMA_RDWR) ||
            (pkt->pkt_cdblen > sizeof (cmd->cdb)) ||
            ((sd = scsi_address_device(ap)) == NULL) ||
            ((pd = scsi_device_hba_private_get(sd)) == NULL) ||
            ((pvs = pd->pvs) == NULL))  {
                return (TRAN_BADPKT);
        }

        ASSERT(cmd->pkt == pkt);

        poll = cmd->poll = ((pkt->pkt_flags & FLAG_NOINTR) != 0);

        if (pkt->pkt_flags & (FLAG_HTAG|FLAG_HEAD)) {
                cmd->tag = MSG_HEAD_QTAG;
        } else if (pkt->pkt_flags & FLAG_OTAG) {
                cmd->tag = MSG_ORDERED_QTAG;
        } else { /* also FLAG_STAG */
                cmd->tag = MSG_SIMPLE_QTAG;
        }

        bcopy(pkt->pkt_cdbp, cmd->cdb, pkt->pkt_cdblen);
        cmd->cdblen = pkt->pkt_cdblen;
        bzero(&cmd->cmd_scb, sizeof (cmd->cmd_scb));

        /*
         * Reinitialize some fields because the packet may
         * have been resubmitted.
         */
        pkt->pkt_reason = CMD_CMPLT;
        pkt->pkt_state = 0;
        pkt->pkt_statistics = 0;

        /* Zero status byte - but only if present */
        if (pkt->pkt_scblen > 0) {
                *(pkt->pkt_scbp) = 0;
        }

        if (pkt->pkt_numcookies > 0) {
                if (pkt->pkt_dma_flags & DDI_DMA_READ) {
                        cmd->dma_dir = PVSCSI_FLAG_CMD_DIR_TOHOST;
                } else if (pkt->pkt_dma_flags & DDI_DMA_WRITE) {
                        cmd->dma_dir = PVSCSI_FLAG_CMD_DIR_TODEVICE;
                } else {
                        cmd->dma_dir = 0;
                }
        }

        cmd->target = pd->target;
        cmd->lun = pd->lun;
        cmd->start = ddi_get_lbolt();
        cmd->timeout = pkt->pkt_time * pvscsi_hz;

        rc = pvscsi_transport_command(pvs, cmd);

        if (poll && rc == TRAN_ACCEPT) {
                pvscsi_poll_cmd(pvs, cmd);
                pvscsi_set_status(pvs, cmd);
        }

        return (rc);
}


static int
pvscsi_parse_ua(const char *ua, int *target, int *lun)
{
        char *end;
        long num;
        if (((ddi_strtol(ua, &end, 16, &num)) != 0) ||
            ((*end != ',') && (*end != 0))) {
                return (DDI_FAILURE);
        }
        *target = (int)num;
        if (*end == 0) {
                *lun = 0;
                return (DDI_SUCCESS);
        }
        end++;
        if ((ddi_strtol(end, &end, 16, &num) != 0) || (*end != 0)) {
                return (DDI_FAILURE);
        }
        *lun = (int)num;
        return (DDI_SUCCESS);
}

static uint32_t
pvscsi_max_targets(pvscsi_softc_t *pvs)
{
        pvscsi_dma_buf_t                        db;
        struct PVSCSIConfigPageController       cpc;
        struct PVSCSICmdDescConfigCmd           cmd;

        bzero(&db, sizeof (db));

        /* NB: config pages fit in a single page */
        if (pvscsi_setup_dma_buffer(pvs, PAGE_SIZE, &db) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN,
                    "!failed to setup config page DMA");
                return (PVSCSI_MAXTGTS);
        }

        bzero(&cmd, sizeof (cmd));
        cmd.configPageAddress = PVSCSI_CONFIG_CONTROLLER_ADDRESS;
        cmd.configPageAddress <<= 32;
        cmd.configPageNum = PVSCSI_CONFIG_PAGE_CONTROLLER;
        cmd.cmpAddr = db.pa;

        pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_CONFIG, &cmd, sizeof (cmd));
        (void) ddi_dma_sync(db.dmah, 0, 0, DDI_DMA_SYNC_FORKERNEL);
        bcopy(db.addr, &cpc, sizeof (cpc));
        pvscsi_free_dma_buffer(&db);


        if ((cpc.header.scsiStatus == STATUS_GOOD) &&
            (cpc.header.hostStatus == BTSTAT_SUCCESS) &&
            (cpc.numPhys > 0)) {
                return (cpc.numPhys);
        }

        dev_err(pvs->dip, CE_WARN, "!failed to determine max targets");
        return (PVSCSI_MAXTGTS);
}

static boolean_t
pvscsi_probe_target(pvscsi_softc_t *pvs, int target)
{
        pvscsi_cmd_t            cmd;

        if (!pvscsi_cmd_init(pvs, &cmd, KM_SLEEP)) {
                pvscsi_cmd_fini(&cmd);
                return (B_FALSE);
        }
        /* NB: CDB 0 is a TUR which is perfect for our needs */
        bzero(cmd.cdb, sizeof (cmd.cdb));
        cmd.poll = B_TRUE;
        cmd.dma_dir = 0;
        cmd.target = target;
        cmd.lun = 0;
        cmd.start = ddi_get_lbolt();
        cmd.timeout = pvscsi_hz;

        if (pvscsi_transport_command(pvs, &cmd) != TRAN_ACCEPT) {
                pvscsi_cmd_fini(&cmd);
                return (B_FALSE);
        }
        pvscsi_poll_cmd(pvs, &cmd);

        switch (cmd.host_status) {
        case BTSTAT_SUCCESS:
        case BTSTAT_LINKED_COMMAND_COMPLETED:
        case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG:
                /* We don't care about the actual SCSI status */
                pvscsi_cmd_fini(&cmd);
                return (B_TRUE);
        }

        pvscsi_cmd_fini(&cmd);
        return (B_FALSE);
}

static int
pvscsi_tgt_init(dev_info_t *dip, dev_info_t *child, scsi_hba_tran_t *tran,
    struct scsi_device *sd)
{
        /*
         * Assumption: the HBA framework only asks us to have a single
         * target initialized per address at any given time.
         */
        pvscsi_device_t *pd;
        pvscsi_softc_t  *pvs;
        const char      *ua;

        if (((scsi_hba_iport_unit_address(dip)) == NULL) ||
            ((pvs = tran->tran_hba_private) == NULL) ||
            ((ua = scsi_device_unit_address(sd)) == NULL)) {
                return (DDI_FAILURE);
        }

        /* parse the unit address */
        pd = kmem_zalloc(sizeof (*pd), KM_SLEEP);
        if (pvscsi_parse_ua(ua, &pd->target, &pd->lun) != DDI_SUCCESS) {
                kmem_free(pd, sizeof (*pd));
                return (DDI_FAILURE);
        }
        pd->pvs = pvs;
        scsi_device_hba_private_set(sd, pd);

        mutex_enter(&pvs->lock);
        list_insert_tail(&pvs->devices, pd);
        mutex_exit(&pvs->lock);
        return (DDI_SUCCESS);
}

static void
pvscsi_tgt_free(dev_info_t *dip, dev_info_t *child, scsi_hba_tran_t *tran,
    struct scsi_device *sd)
{
        pvscsi_device_t *pd;
        pvscsi_softc_t  *pvs;

        if (((scsi_hba_iport_unit_address(dip)) == NULL) ||
            ((pvs = tran->tran_hba_private) == NULL) ||
            ((pd = scsi_device_hba_private_get(sd)) == NULL)) {
                return;
        }
        scsi_device_hba_private_set(sd, NULL);
        mutex_enter(&pvs->lock);
        list_remove(&pvs->devices, pd);
        mutex_exit(&pvs->lock);

        kmem_free(pd, sizeof (*pd));
}

static int
pvscsi_reset(struct scsi_address *ap, int level)
{
        struct scsi_device      *sd;
        pvscsi_device_t         *pd;
        pvscsi_softc_t          *pvs;
        pvscsi_cmd_t            *cmd;

        if (((sd = scsi_address_device(ap)) == NULL) ||
            ((pd = scsi_device_hba_private_get(sd)) == NULL) ||
            ((pvs = pd->pvs) == NULL))  {
                return (0);
        }
        switch (level) {
        case RESET_ALL:
        case RESET_BUS:
                pvscsi_reset_bus(pvs);
                break;
        case RESET_TARGET:
                /* reset both the lun and lun 0 */
                pvscsi_dev_reset(pvs, pd->target, pd->lun);
                pvscsi_dev_reset(pvs, pd->target, 0);
                break;
        case RESET_LUN:
                pvscsi_dev_reset(pvs, pd->target, pd->lun);
                break;
        default:
                return (0);
        }

        /* reset may have caused some completions */
        mutex_enter(&pvs->lock);
        cmd = pvscsi_process_comp_ring(pvs);
        mutex_exit(&pvs->lock);

        pvscsi_complete_cmds(pvs, cmd);
        return (1);
}

static int
pvscsi_abort(struct scsi_address *ap, struct scsi_pkt *pkt)
{
        struct scsi_device      *sd;
        pvscsi_device_t         *pd;
        pvscsi_softc_t          *pvs;
        pvscsi_cmd_t            *cmd;

        if (pkt != NULL) {
                /* abort single command */
                cmd = pkt->pkt_ha_private;
                pvs = cmd->pvs;
                pvscsi_abort_cmd(pvs, cmd);
        } else if ((ap != NULL) &&
            ((sd = scsi_address_device(ap)) != NULL) &&
            ((pd = scsi_device_hba_private_get(sd)) != NULL) &&
            ((pvs = pd->pvs) != NULL)) {
                /* abort all commands on the bus */
                pvscsi_abort_all(pvs, pd);
        } else {
                return (0);
        }

        /* abort may have caused some completions */
        mutex_enter(&pvs->lock);
        cmd = pvscsi_process_comp_ring(pvs);
        mutex_exit(&pvs->lock);

        pvscsi_complete_cmds(pvs, cmd);

        return (1);
}

static int
pvscsi_getcap(struct scsi_address *ap, char *cap, int whom)
{
        _NOTE(ARGUNUSED(ap));
        _NOTE(ARGUNUSED(whom));

        if (cap == NULL) {
                return (-1);
        }

        switch (scsi_hba_lookup_capstr(cap)) {
        case SCSI_CAP_ARQ:
        case SCSI_CAP_UNTAGGED_QING:
        case SCSI_CAP_TAGGED_QING:
                return (1);
        default:
                return (-1);
        }
}

static int
pvscsi_setcap(struct scsi_address *ap, char *cap, int value, int whom)
{
        _NOTE(ARGUNUSED(ap));
        _NOTE(ARGUNUSED(value));
        _NOTE(ARGUNUSED(whom));

        if (cap == NULL) {
                return (-1);
        }

        switch (scsi_hba_lookup_capstr(cap)) {
        case SCSI_CAP_ARQ:
        case SCSI_CAP_UNTAGGED_QING:
        case SCSI_CAP_TAGGED_QING:
                return (0); /* not changeable */
        default:
                return (-1);
        }
}

static void
pvscsi_cmd_fini(pvscsi_cmd_t *cmd)
{
        if (cmd->arq_pa != 0) {
                (void) ddi_dma_unbind_handle(cmd->arq_dmah);
                cmd->arq_dmah = NULL;
        }
        if (cmd->arq_dmah != NULL) {
                ddi_dma_free_handle(&cmd->arq_dmah);
                cmd->arq_dmah = NULL;
        }
        if (cmd->sgl_pa != 0) {
                (void) ddi_dma_unbind_handle(cmd->sgl_dmah);
                cmd->sgl_pa = 0;
        }
        if (cmd->sgl_acch != NULL) {
                ddi_dma_mem_free(&cmd->sgl_acch);
                cmd->sgl_acch = NULL;
                cmd->sgl = NULL;
        }
        if (cmd->sgl_dmah != NULL) {
                ddi_dma_free_handle(&cmd->sgl_dmah);
                cmd->sgl_dmah = NULL;
        }
        if (cmd->ctx != 0) {
                id32_free(cmd->ctx);
                cmd->ctx = 0;
        }
}

static void
pvscsi_pkt_dtor(struct scsi_pkt *pkt, scsi_hba_tran_t *tran)
{
        pvscsi_cmd_t    *cmd = pkt->pkt_ha_private;
        pvscsi_cmd_fini(cmd);
}

static boolean_t
pvscsi_cmd_init(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd, int sleep)
{
        int             (*cb)(caddr_t);
        size_t          len;
        caddr_t         kaddr;

        cb = sleep == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;

        bzero(cmd, sizeof (*cmd));
        cmd->ctx = id32_alloc(cmd, sleep);
        if (cmd->ctx == 0) {
                dev_err(pvs->dip, CE_WARN,
                    "!failed to allocate 32-bit context id");
                return (B_FALSE);
        }

        /* allocate DMA resources for scatter/gather list */
        if (ddi_dma_alloc_handle(pvs->dip, &pvscsi_dma_attr, cb, NULL,
            &cmd->sgl_dmah) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN,
                    "!failed to allocate DMA handle for SG list");
                return (B_FALSE);
        }
        if (ddi_dma_mem_alloc(cmd->sgl_dmah, PAGE_SIZE, &pvscsi_dma_attrs,
            DDI_DMA_CONSISTENT, cb, NULL, &kaddr, &len, &cmd->sgl_acch) !=
            DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN,
                    "!failed to allocate DMA memory for SG list");
                return (B_FALSE);
        }
        cmd->sgl = (void *)kaddr;
        if (ddi_dma_addr_bind_handle(cmd->sgl_dmah, NULL, kaddr,
            PAGE_SIZE, DDI_DMA_WRITE | DDI_DMA_CONSISTENT, cb, NULL,
            NULL, NULL) != DDI_DMA_MAPPED) {
                dev_err(pvs->dip, CE_WARN, "!failed to bind SGL list");
                return (B_FALSE);
        }
        cmd->sgl_pa = ddi_dma_cookie_one(cmd->sgl_dmah)->dmac_laddress;

        /* allocate DMA resource for auto-sense-request */
        if (ddi_dma_alloc_handle(pvs->dip, &pvscsi_dma_attr,
            cb, NULL, &cmd->arq_dmah) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN,
                    "!failed to allocate DMA handle for ARQ buffer");
                return (B_FALSE);
        }

        if (ddi_dma_addr_bind_handle(cmd->arq_dmah, NULL,
            (void *)cmd->arq_sense, SENSE_LENGTH,
            DDI_DMA_READ | DDI_DMA_CONSISTENT, cb, NULL,
            NULL, NULL) != DDI_DMA_MAPPED) {
                dev_err(pvs->dip, CE_WARN, "!failed to bind ARQ buffer");
                return (B_FALSE);
        }
        cmd->arq_pa = ddi_dma_cookie_one(cmd->arq_dmah)->dmac_laddress;
        return (B_TRUE);
}

static int
pvscsi_pkt_ctor(struct scsi_pkt *pkt, scsi_hba_tran_t *tran, int sleep)
{
        pvscsi_cmd_t    *cmd = pkt->pkt_ha_private;
        pvscsi_softc_t  *pvs = tran->tran_hba_private;

        if (!pvscsi_cmd_init(pvs, cmd, sleep)) {
                pvscsi_pkt_dtor(pkt, tran);
                return (-1);
        }
        cmd->pkt = pkt;
        return (0);
}

static void
pvscsi_teardown_pkt(struct scsi_pkt *pkt)
{
        _NOTE(ARGUNUSED(pkt));
        /* nothing to do */
}

static int
pvscsi_setup_pkt(struct scsi_pkt *pkt, int (*cb)(caddr_t), caddr_t arg)
{
        /* all work is done in start */
        return (0);
}

static int
pvscsi_hba_setup(pvscsi_softc_t *pvs)
{
        scsi_hba_tran_t *tran;

        tran = scsi_hba_tran_alloc(pvs->dip, SCSI_HBA_CANSLEEP);
        ASSERT(tran != NULL);

        tran->tran_hba_private = pvs;
        tran->tran_start = pvscsi_start;
        tran->tran_reset = pvscsi_reset;
        tran->tran_abort = pvscsi_abort;
        tran->tran_getcap = pvscsi_getcap;
        tran->tran_setcap = pvscsi_setcap;
        tran->tran_pkt_constructor = pvscsi_pkt_ctor;
        tran->tran_pkt_destructor = pvscsi_pkt_dtor;
        tran->tran_setup_pkt = pvscsi_setup_pkt;
        tran->tran_teardown_pkt = pvscsi_teardown_pkt;
        tran->tran_tgt_init = pvscsi_tgt_init;
        tran->tran_tgt_free = pvscsi_tgt_free;
        tran->tran_hba_len = sizeof (pvscsi_cmd_t);

        tran->tran_interconnect_type = INTERCONNECT_PARALLEL;

        if (scsi_hba_attach_setup(pvs->dip, &pvscsi_io_dma_attr, tran,
            SCSI_HBA_HBA | SCSI_HBA_TRAN_CDB | SCSI_HBA_TRAN_SCB |
            SCSI_HBA_ADDR_COMPLEX) !=
            DDI_SUCCESS) {
                scsi_hba_tran_free(tran);
                dev_err(pvs->dip, CE_WARN, "!failed to attach HBA");
                return (DDI_FAILURE);
        }

        pvs->tran = tran;
        return (DDI_SUCCESS);
}

static void
pvscsi_teardown(pvscsi_softc_t *pvs)
{
        timeout_id_t    tid;

        pvscsi_stop_hba(pvs);

        if (pvs->tq != NULL) {
                ddi_taskq_destroy(pvs->tq);
        }
        mutex_enter(&pvs->lock);
        tid = pvs->timeout;
        pvs->timeout = 0;
        mutex_exit(&pvs->lock);

        if (tid != 0) {
                (void) untimeout(tid);
        }

        pvscsi_free_intrs(pvs);
        pvscsi_free_rings(pvs);

        if (pvs->mmio_handle != NULL) {
                ddi_regs_map_free(&pvs->mmio_handle);
        }

        if (pvs->tran != NULL) {
                scsi_hba_tran_free(pvs->tran);
        }
        mutex_destroy(&pvs->lock);
        list_destroy(&pvs->cmd_queue);
        list_destroy(&pvs->devices);

        kmem_free(pvs, sizeof (*pvs));
}

static int
pvscsi_iport_attach(dev_info_t *dip)
{
        scsi_hba_tran_t *tran;
        dev_info_t      *parent;
        pvscsi_softc_t  *pvs;
        char            *ua;
        uint32_t        max_targets;

        if (((parent = ddi_get_parent(dip)) == NULL) ||
            ((tran = ddi_get_driver_private(parent)) == NULL) ||
            ((pvs = tran->tran_hba_private) == NULL) ||
            ((ua = scsi_hba_iport_unit_address(dip)) == NULL) ||
            (strcmp(ua, "iport0") != 0)) {
                return (DDI_FAILURE);
        }

        /* store our softc on the iport private tran */
        tran = ddi_get_driver_private(dip);
        tran->tran_hba_private = pvs;

        /* setup the target map - allow 100ms for settle / sync times */
        if (scsi_hba_tgtmap_create(dip, SCSI_TM_PERADDR, 100000,
            100000, pvs, NULL, NULL, &pvs->tgtmap) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to create target map");
                return (DDI_FAILURE);
        }

        /* reset hardware and setup the rings */
        mutex_enter(&pvs->lock);
        pvs->detach = B_FALSE; /* in case of reattach */
        pvscsi_start_hba(pvs);

        max_targets = pvs->max_targets = pvscsi_max_targets(pvs);
        mutex_exit(&pvs->lock);

        for (uint32_t i = 0; i < max_targets; i++) {
                char addr[8];
                if (pvscsi_probe_target(pvs, i)) {
                        (void) snprintf(addr, sizeof (addr), "%x", i);
                        (void) scsi_hba_tgtmap_tgt_add(pvs->tgtmap,
                            SCSI_TGT_SCSI_DEVICE, addr, NULL);
                }
        }

        return (DDI_SUCCESS);
}

static int
pvscsi_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
        pvscsi_softc_t  *pvs;

        if (cmd != DDI_ATTACH) {
                return (DDI_FAILURE);
        }

        if (scsi_hba_iport_unit_address(dip) != NULL) {
                return (pvscsi_iport_attach(dip));
        }

        pvs = kmem_zalloc(sizeof (*pvs), KM_SLEEP);

        /* Setup HBA instance */
        pvs->dip = dip;

        /*
         * mutex initialization - note that we always run below
         * lock level, so we can get by without interrupt priorities
         */
        mutex_init(&pvs->lock, NULL, MUTEX_DRIVER, NULL);
        list_create(&pvs->cmd_queue, sizeof (pvscsi_cmd_t),
            offsetof(pvscsi_cmd_t, queue_node));
        list_create(&pvs->devices, sizeof (pvscsi_device_t),
            offsetof(pvscsi_device_t, node));

        if ((pvscsi_setup_io(pvs)) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to setup I/O region");
                pvscsi_teardown(pvs);
                return (DDI_FAILURE);
        }

        pvscsi_stop_hba(pvs);

        if ((pvscsi_allocate_rings(pvs)) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA rings");
                pvscsi_teardown(pvs);
                return (DDI_FAILURE);
        }

        if (pvscsi_setup_isr(pvs) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to setup ISR");
                pvscsi_teardown(pvs);
                return (DDI_FAILURE);
        }

        /* enable interrupts */
        if (pvscsi_enable_intrs(pvs) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to enable interrupts");
                pvscsi_teardown(pvs);
                return (DDI_FAILURE);
        }

        pvs->tq = ddi_taskq_create(dip, "iport", 1, TASKQ_DEFAULTPRI, 0);
        if (pvs->tq == NULL) {
                dev_err(pvs->dip, CE_WARN, "!failed creating tq");
                pvscsi_teardown(pvs);
                return (DDI_FAILURE);
        }
        if (pvscsi_hba_setup(pvs) != DDI_SUCCESS) {
                dev_err(pvs->dip, CE_WARN, "!failed to setup HBA");
                pvscsi_teardown(pvs);
                return (DDI_FAILURE);
        }

        if (scsi_hba_iport_register(dip, "iport0") != 0) {
                dev_err(pvs->dip, CE_WARN, "failed to register iport");
                /* detach cannot fail since we didn't setup the iport */
                (void) scsi_hba_detach(dip);
                pvscsi_teardown(pvs);
                return (DDI_FAILURE);
        }

        return (DDI_SUCCESS);
}

static int
pvscsi_iport_detach(dev_info_t *dip)
{
        pvscsi_softc_t  *pvs;
        scsi_hba_tran_t *tran;
        const char      *ua;
        pvscsi_cmd_t    *reclaimed;

        if (((ua = scsi_hba_iport_unit_address(dip)) == NULL) ||
            (strcmp(ua, "iport0") != 0) ||
            ((tran = ddi_get_driver_private(dip)) == NULL) ||
            ((pvs = tran->tran_hba_private) == NULL)) {
                return (DDI_FAILURE);
        }

        /* stop the HBA */
        mutex_enter(&pvs->lock);
        pvs->detach = B_TRUE;
        pvscsi_stop_hba(pvs);
        mutex_exit(&pvs->lock);

        /* drain the taskq - nothing else will post to it */
        ddi_taskq_wait(pvs->tq);

        /* reset the HBA */
        mutex_enter(&pvs->lock);
        reclaimed = pvscsi_reclaim_cmds(pvs);
        mutex_exit(&pvs->lock);

        /*
         * If we had any commands, complete them so we can
         * reclaim the resources.  There really should not be any.
         */
        pvscsi_complete_cmds(pvs, reclaimed);

        scsi_hba_tgtmap_destroy(pvs->tgtmap);
        pvs->tgtmap = NULL;

        return (DDI_SUCCESS);
}

static int
pvscsi_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
        pvscsi_softc_t  *pvs;
        scsi_hba_tran_t *tran;

        if (cmd != DDI_DETACH) {
                return (DDI_FAILURE);
        }

        if (scsi_hba_iport_unit_address(dip) != NULL) {
                return (pvscsi_iport_detach(dip));
        }

        if (((tran = ddi_get_driver_private(dip)) == NULL) ||
            ((pvs = tran->tran_hba_private) == NULL)) {
                /* this can only mean we aren't attached yet */
                return (DDI_SUCCESS);
        }
        if (scsi_hba_detach(dip) != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        pvscsi_teardown(pvs);

        return (DDI_SUCCESS);
}

static int
pvscsi_quiesce(dev_info_t *dip)
{
        scsi_hba_tran_t *tran;
        pvscsi_softc_t  *pvs;

        if (((tran = ddi_get_driver_private(dip)) == NULL) ||
            ((pvs = tran->tran_hba_private) == NULL)) {
                return (DDI_SUCCESS);
        }

        pvscsi_stop_hba(pvs);

        return (DDI_SUCCESS);
}

static struct dev_ops pvscsi_ops = {
        .devo_rev =     DEVO_REV,
        .devo_refcnt =  0,
        .devo_getinfo = nodev,
        .devo_identify = nulldev,
        .devo_probe =   nulldev,
        .devo_attach =  pvscsi_attach,
        .devo_detach =  pvscsi_detach,
        .devo_reset =   nodev,
        .devo_cb_ops =  NULL,
        .devo_bus_ops = NULL,
        .devo_power =   NULL,
        .devo_quiesce = pvscsi_quiesce
};

#define PVSCSI_IDENT "VMware PVSCSI"

static struct modldrv modldrv = {
        &mod_driverops,
        PVSCSI_IDENT,
        &pvscsi_ops,
};

static struct modlinkage modlinkage = {
        MODREV_1,
        &modldrv,
        NULL
};

int
_init(void)
{
        int     ret;

        /* get HZ - DDI compliant */
        pvscsi_hz = drv_usectohz(1000000);

        if ((ret = scsi_hba_init(&modlinkage)) != 0) {
                cmn_err(CE_WARN, "!scsi_hba_init() failed");
                return (ret);
        }

        if ((ret = mod_install(&modlinkage)) != 0) {
                cmn_err(CE_WARN, "!mod_install() failed");
                scsi_hba_fini(&modlinkage);
        }

        return (ret);
}

int
_info(struct modinfo *modinfop)
{
        return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
        int     ret;

        if ((ret = mod_remove(&modlinkage)) == 0) {
                scsi_hba_fini(&modlinkage);
        }

        return (ret);
}