/*
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2020-2025, Broadcom Inc. All rights reserved.
 * Support: <fbsd-storage-driver.pdl@broadcom.com>
 *
 * Authors: Sumit Saxena <sumit.saxena@broadcom.com>
 *          Chandrakanth Patil <chandrakanth.patil@broadcom.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation and/or other
 *    materials provided with the distribution.
 * 3. Neither the name of the Broadcom Inc. nor the names of its contributors
 *    may be used to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * The views and conclusions contained in the software and documentation are
 * those of the authors and should not be interpreted as representing
 * official policies,either expressed or implied, of the FreeBSD Project.
 *
 * Mail to: Broadcom Inc 1320 Ridder Park Dr, San Jose, CA 95131
 *
 * Broadcom Inc. (Broadcom) MPI3MR Adapter FreeBSD
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/selinfo.h>
#include <sys/module.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/bio.h>
#include <sys/malloc.h>
#include <sys/uio.h>
#include <sys/sysctl.h>
#include <sys/endian.h>
#include <sys/queue.h>
#include <sys/kthread.h>
#include <sys/taskqueue.h>
#include <sys/sbuf.h>
#include <sys/stdarg.h>

#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/rman.h>

#include <cam/cam.h>
#include <cam/cam_ccb.h>
#include <cam/cam_debug.h>
#include <cam/cam_sim.h>
#include <cam/cam_xpt_sim.h>
#include <cam/cam_xpt_periph.h>
#include <cam/cam_periph.h>
#include <cam/scsi/scsi_all.h>
#include <cam/scsi/scsi_message.h>
#include <cam/scsi/smp_all.h>

#include <dev/nvme/nvme.h>
#include "mpi/mpi30_api.h"
#include "mpi3mr_cam.h"
#include "mpi3mr.h"
#include <sys/time.h>                   /* XXX for pcpu.h */
#include <sys/pcpu.h>                   /* XXX for PCPU_GET */
#include <asm/unaligned.h>

#define smp_processor_id()  PCPU_GET(cpuid)

static void
mpi3mr_enqueue_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cm);
static void
mpi3mr_map_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cm);
void
mpi3mr_release_simq_reinit(struct mpi3mr_cam_softc *cam_sc);
static void
mpi3mr_freeup_events(struct mpi3mr_softc *sc);

extern int
mpi3mr_register_events(struct mpi3mr_softc *sc);
extern void mpi3mr_add_sg_single(void *paddr, U8 flags, U32 length,
    bus_addr_t dma_addr);

static U32 event_count;

static
inline void mpi3mr_divert_ws(Mpi3SCSIIORequest_t *req,
                             struct ccb_scsiio *csio,
                             U16 ws_len)
{
        U8 unmap = 0, ndob = 0;
        U32 num_blocks = 0;
        U8 opcode = scsiio_cdb_ptr(csio)[0];
        U16 service_action = ((scsiio_cdb_ptr(csio)[8] << 8) | scsiio_cdb_ptr(csio)[9]);


        if (opcode == WRITE_SAME_16 ||
           (opcode == VARIABLE_LEN_CDB &&
            service_action == WRITE_SAME_32)) {

                int unmap_ndob_index = (opcode == WRITE_SAME_16) ? 1 : 10;

                unmap = scsiio_cdb_ptr(csio)[unmap_ndob_index] & 0x08;
                ndob = scsiio_cdb_ptr(csio)[unmap_ndob_index] & 0x01;
                num_blocks = get_unaligned_be32(scsiio_cdb_ptr(csio) +
                                                ((opcode == WRITE_SAME_16) ? 10 : 28));

                /* Check conditions for diversion to firmware */
                if (unmap && ndob && num_blocks > ws_len) {
                        req->MsgFlags |= MPI3_SCSIIO_MSGFLAGS_DIVERT_TO_FIRMWARE;
                        req->Flags = htole32(le32toh(req->Flags) |
                                             MPI3_SCSIIO_FLAGS_DIVERT_REASON_WRITE_SAME_TOO_LARGE);
                }
        }
}

static void mpi3mr_prepare_sgls(void *arg,
        bus_dma_segment_t *segs, int nsegs, int error)
{
        struct mpi3mr_softc *sc;
        struct mpi3mr_cmd *cm;
        u_int i;
        bus_addr_t chain_dma;
        void *chain;
        U8 *sg_local;
        U32 chain_length;
        int sges_left;
        U32 sges_in_segment;
        U8 simple_sgl_flags;
        U8 simple_sgl_flags_last;
        U8 last_chain_sgl_flags;
        struct mpi3mr_chain *chain_req;
        Mpi3SCSIIORequest_t *scsiio_req;
        union ccb *ccb;
        
        cm = (struct mpi3mr_cmd *)arg;
        sc = cm->sc;
        scsiio_req = (Mpi3SCSIIORequest_t *) &cm->io_request;
        ccb = cm->ccb;

        if (error) {
                device_printf(sc->mpi3mr_dev, "%s: error=%d\n",__func__, error);
                if (error == EFBIG) {
                        mpi3mr_set_ccbstatus(ccb, CAM_REQ_TOO_BIG);
                } else {
                        mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
                }
                mpi3mr_release_command(cm);
                xpt_done(ccb);
                return;
        }
        
        if (cm->data_dir == MPI3MR_READ)
                bus_dmamap_sync(sc->buffer_dmat, cm->dmamap,
                    BUS_DMASYNC_PREREAD);
        if (cm->data_dir == MPI3MR_WRITE)
                bus_dmamap_sync(sc->buffer_dmat, cm->dmamap,
                    BUS_DMASYNC_PREWRITE);

        KASSERT(nsegs <= sc->max_sgl_entries && nsegs > 0,
            ("%s: bad SGE count: %d\n", device_get_nameunit(sc->mpi3mr_dev), nsegs));
        KASSERT(scsiio_req->DataLength != 0,
            ("%s: Data segments (%d), but DataLength == 0\n",
                device_get_nameunit(sc->mpi3mr_dev), nsegs));

        simple_sgl_flags = MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE |
            MPI3_SGE_FLAGS_DLAS_SYSTEM;
        simple_sgl_flags_last = simple_sgl_flags |
            MPI3_SGE_FLAGS_END_OF_LIST;
        last_chain_sgl_flags = MPI3_SGE_FLAGS_ELEMENT_TYPE_LAST_CHAIN |
            MPI3_SGE_FLAGS_DLAS_SYSTEM;

        sg_local = (U8 *)&scsiio_req->SGL;

        sges_left = nsegs;

        sges_in_segment = (sc->facts.op_req_sz -
            offsetof(Mpi3SCSIIORequest_t, SGL))/sizeof(Mpi3SGESimple_t);

        i = 0;

        mpi3mr_dprint(sc, MPI3MR_TRACE, "SGE count: %d IO size: %d\n",
                nsegs, scsiio_req->DataLength);

        if (sges_left <= sges_in_segment)
                goto fill_in_last_segment;

        /* fill in main message segment when there is a chain following */
        while (sges_in_segment > 1) {
                mpi3mr_add_sg_single(sg_local, simple_sgl_flags,
                    segs[i].ds_len, segs[i].ds_addr);
                sg_local += sizeof(Mpi3SGESimple_t);
                sges_left--;
                sges_in_segment--;
                i++;
        }

        chain_req = &sc->chain_sgl_list[cm->hosttag];
        
        chain = chain_req->buf;
        chain_dma = chain_req->buf_phys;
        memset(chain_req->buf, 0, sc->max_sgl_entries * sizeof(Mpi3SGESimple_t));
        sges_in_segment = sges_left;
        chain_length = sges_in_segment * sizeof(Mpi3SGESimple_t);

        mpi3mr_add_sg_single(sg_local, last_chain_sgl_flags,
            chain_length, chain_dma);

        sg_local = chain;

fill_in_last_segment:
        while (sges_left > 0) {
                if (sges_left == 1)
                        mpi3mr_add_sg_single(sg_local,
                            simple_sgl_flags_last, segs[i].ds_len,
                            segs[i].ds_addr);
                else
                        mpi3mr_add_sg_single(sg_local, simple_sgl_flags,
                            segs[i].ds_len, segs[i].ds_addr);
                sg_local += sizeof(Mpi3SGESimple_t);
                sges_left--;
                i++;
        }

        /*
         * Now that we've created the sgls, we send the request to the device.
         * Unlike in Linux, dmaload isn't guaranteed to load every time, but
         * this function is always called when the resources are available, so
         * we can send the request to hardware here always. mpi3mr_map_request
         * knows about this quirk and will only take evasive action when an
         * error other than EINPROGRESS is returned from dmaload.
         */
        mpi3mr_enqueue_request(sc, cm);

        return;
}

static void
mpi3mr_map_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cm)
{
        u_int32_t retcode = 0;
        union ccb *ccb;

        ccb = cm->ccb;
        if (cm->data != NULL) {
                mtx_lock(&sc->io_lock);
                /* Map data buffer into bus space */
                retcode = bus_dmamap_load_ccb(sc->buffer_dmat, cm->dmamap,
                    ccb, mpi3mr_prepare_sgls, cm, 0);
                mtx_unlock(&sc->io_lock);
                if (retcode != 0 && retcode != EINPROGRESS) {
                        device_printf(sc->mpi3mr_dev,
                            "bus_dmamap_load(): retcode = %d\n", retcode);
                        /*
                         * Any other error means prepare_sgls wasn't called, and
                         * will never be called, so we have to mop up. This error
                         * should never happen, though.
                         */
                        mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
                        mpi3mr_release_command(cm);
                        xpt_done(ccb);
                }
        } else {
                /*
                 * No data, we enqueue it directly here.
                 */
                mpi3mr_enqueue_request(sc, cm);
        }
}

void
mpi3mr_unmap_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd)
{
        if (cmd->data != NULL) {
                if (cmd->data_dir == MPI3MR_READ)
                        bus_dmamap_sync(sc->buffer_dmat, cmd->dmamap, BUS_DMASYNC_POSTREAD);
                if (cmd->data_dir == MPI3MR_WRITE)
                        bus_dmamap_sync(sc->buffer_dmat, cmd->dmamap, BUS_DMASYNC_POSTWRITE);
                mtx_lock(&sc->io_lock);
                bus_dmamap_unload(sc->buffer_dmat, cmd->dmamap);
                mtx_unlock(&sc->io_lock);
        }
}

/**
 * mpi3mr_allow_unmap_to_fw - Whether an unmap is allowed to fw
 * @sc: Adapter instance reference
 * @ccb: SCSI Command reference
 *
 * The controller hardware cannot handle certain unmap commands
 * for NVMe drives, this routine checks those and return true
 * and completes the SCSI command with proper status and sense
 * data.
 *
 * Return: TRUE for allowed unmap, FALSE otherwise.
 */
static bool mpi3mr_allow_unmap_to_fw(struct mpi3mr_softc *sc,
        union ccb *ccb)
{
        struct ccb_scsiio *csio;
        uint16_t param_list_len, block_desc_len, trunc_param_len = 0;

        csio = &ccb->csio;
        param_list_len = (uint16_t) ((scsiio_cdb_ptr(csio)[7] << 8) | scsiio_cdb_ptr(csio)[8]);

        switch(pci_get_revid(sc->mpi3mr_dev)) {
        case SAS4116_CHIP_REV_A0:
                if (!param_list_len) {
                        mpi3mr_dprint(sc, MPI3MR_ERROR,
                            "%s: CDB received with zero parameter length\n",
                            __func__);
                        mpi3mr_print_cdb(ccb);
                        mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP);
                        xpt_done(ccb);
                        return false;
                }

                if (param_list_len < 24) {
                        mpi3mr_dprint(sc, MPI3MR_ERROR,
                            "%s: CDB received with invalid param_list_len: %d\n",
                            __func__, param_list_len);
                        mpi3mr_print_cdb(ccb);
                        scsi_set_sense_data(&ccb->csio.sense_data,
                                /*sense_format*/ SSD_TYPE_FIXED,
                                /*current_error*/ 1,
                                /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
                                /*asc*/ 0x1A,
                                /*ascq*/ 0x00,
                                /*extra args*/ SSD_ELEM_NONE);
                        ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
                        ccb->ccb_h.status =
                            CAM_SCSI_STATUS_ERROR |
                            CAM_AUTOSNS_VALID;
                        return false;
                }

                if (param_list_len != csio->dxfer_len) {
                        mpi3mr_dprint(sc, MPI3MR_ERROR,
                            "%s: CDB received with param_list_len: %d bufflen: %d\n",
                            __func__, param_list_len, csio->dxfer_len);
                        mpi3mr_print_cdb(ccb);
                        scsi_set_sense_data(&ccb->csio.sense_data,
                                /*sense_format*/ SSD_TYPE_FIXED,
                                /*current_error*/ 1,
                                /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
                                /*asc*/ 0x1A,
                                /*ascq*/ 0x00,
                                /*extra args*/ SSD_ELEM_NONE);
                        ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
                        ccb->ccb_h.status =
                            CAM_SCSI_STATUS_ERROR |
                            CAM_AUTOSNS_VALID;
                        xpt_done(ccb);
                        return false;
                }
                
                block_desc_len = (uint16_t) (csio->data_ptr[2] << 8 | csio->data_ptr[3]);

                if (block_desc_len < 16) {
                        mpi3mr_dprint(sc, MPI3MR_ERROR,
                            "%s: Invalid descriptor length in param list: %d\n",
                            __func__, block_desc_len);
                        mpi3mr_print_cdb(ccb);
                        scsi_set_sense_data(&ccb->csio.sense_data,
                                /*sense_format*/ SSD_TYPE_FIXED,
                                /*current_error*/ 1,
                                /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
                                /*asc*/ 0x26,
                                /*ascq*/ 0x00,
                                /*extra args*/ SSD_ELEM_NONE);
                        ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
                        ccb->ccb_h.status =
                            CAM_SCSI_STATUS_ERROR |
                            CAM_AUTOSNS_VALID;
                        xpt_done(ccb);
                        return false;
                }

                if (param_list_len > (block_desc_len + 8)) {
                        mpi3mr_print_cdb(ccb);
                        mpi3mr_dprint(sc, MPI3MR_INFO,
                            "%s: Truncating param_list_len(%d) to block_desc_len+8(%d)\n",
                            __func__, param_list_len, (block_desc_len + 8));
                        param_list_len = block_desc_len + 8;
                        scsiio_cdb_ptr(csio)[7] = (param_list_len >> 8) | 0xff;
                        scsiio_cdb_ptr(csio)[8] = param_list_len | 0xff;
                        mpi3mr_print_cdb(ccb);
                }
                break;

        case SAS4116_CHIP_REV_B0:
                if ((param_list_len > 24) && ((param_list_len - 8) & 0xF)) {
                        trunc_param_len -= (param_list_len - 8) & 0xF;
                        mpi3mr_print_cdb(ccb);
                        mpi3mr_dprint(sc, MPI3MR_INFO,
                            "%s: Truncating param_list_len from (%d) to (%d)\n",
                            __func__, param_list_len, trunc_param_len);
                        scsiio_cdb_ptr(csio)[7] = (param_list_len >> 8) | 0xff;
                        scsiio_cdb_ptr(csio)[8] = param_list_len | 0xff;
                        mpi3mr_print_cdb(ccb);
                }
                break;
        }

        return true;
}

/**
 * mpi3mr_tm_response_name -  get TM response as a string
 * @resp_code: TM response code
 *
 * Convert known task management response code as a readable
 * string.
 *
 * Return: response code string.
 */
static const char* mpi3mr_tm_response_name(U8 resp_code)
{
        char *desc;

        switch (resp_code) {
        case MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE:
                desc = "task management request completed";
                break;
        case MPI3_SCSITASKMGMT_RSPCODE_INVALID_FRAME:
                desc = "invalid frame";
                break;
        case MPI3_SCSITASKMGMT_RSPCODE_TM_FUNCTION_NOT_SUPPORTED:
                desc = "task management request not supported";
                break;
        case MPI3_SCSITASKMGMT_RSPCODE_TM_FAILED:
                desc = "task management request failed";
                break;
        case MPI3_SCSITASKMGMT_RSPCODE_TM_SUCCEEDED:
                desc = "task management request succeeded";
                break;
        case MPI3_SCSITASKMGMT_RSPCODE_TM_INVALID_LUN:
                desc = "invalid LUN";
                break;
        case MPI3_SCSITASKMGMT_RSPCODE_TM_OVERLAPPED_TAG:
                desc = "overlapped tag attempted";
                break;
        case MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC:
                desc = "task queued, however not sent to target";
                break;
        case MPI3_SCSITASKMGMT_RSPCODE_TM_NVME_DENIED:
                desc = "task management request denied by NVMe device";
                break;
        default:
                desc = "unknown";
                break;
        }

        return desc;
}

void mpi3mr_poll_pend_io_completions(struct mpi3mr_softc *sc)
{
        int i;
        int num_of_reply_queues = sc->num_queues;
        struct mpi3mr_irq_context *irq_ctx;

        for (i = 0; i < num_of_reply_queues; i++) {
                irq_ctx = &sc->irq_ctx[i];
                mpi3mr_complete_io_cmd(sc, irq_ctx);
        }
}

void
trigger_reset_from_watchdog(struct mpi3mr_softc *sc, U8 reset_type, U16 reset_reason)
{
        if (sc->reset_in_progress) {
                mpi3mr_dprint(sc, MPI3MR_INFO, "Another reset is in progress, no need to trigger the reset\n");
                return;
        }
        sc->reset.type = reset_type;
        sc->reset.reason = reset_reason;

        return;
}

/**
 * mpi3mr_issue_tm - Issue Task Management request
 * @sc: Adapter instance reference
 * @tm_type: Task Management type
 * @handle: Device handle
 * @lun: lun ID
 * @htag: Host tag of the TM request
 * @timeout: TM timeout value
 * @drv_cmd: Internal command tracker
 * @resp_code: Response code place holder
 * @cmd: Timed out command reference
 *
 * Issues a Task Management Request to the controller for a
 * specified target, lun and command and wait for its completion
 * and check TM response. Recover the TM if it timed out by
 * issuing controller reset.
 *
 * Return: 0 on success, non-zero on errors
 */
static int
mpi3mr_issue_tm(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd,
                U8 tm_type, unsigned long timeout)
{
        int retval = 0;
        MPI3_SCSI_TASK_MGMT_REQUEST tm_req;
        MPI3_SCSI_TASK_MGMT_REPLY *tm_reply = NULL;
        struct mpi3mr_drvr_cmd *drv_cmd = NULL;
        struct mpi3mr_target *tgtdev = NULL;
        struct mpi3mr_op_req_queue *op_req_q = NULL;
        union ccb *ccb;
        U8 resp_code;

        
        if (sc->unrecoverable) {
                mpi3mr_dprint(sc, MPI3MR_INFO,
                        "Controller is in unrecoverable state!! TM not required\n");
                return retval;
        }
        if (sc->reset_in_progress) {
                mpi3mr_dprint(sc, MPI3MR_INFO,
                        "controller reset in progress!! TM not required\n");
                return retval;
        }
        
        if (!cmd->ccb) {
                mpi3mr_dprint(sc, MPI3MR_ERROR, "SCSIIO command timed-out with NULL ccb\n");
                return retval;
        }
        ccb = cmd->ccb;

        tgtdev = cmd->targ;
        if (tgtdev == NULL)  {
                mpi3mr_dprint(sc, MPI3MR_ERROR, "Device does not exist target ID:0x%x,"
                              "TM is not required\n", ccb->ccb_h.target_id);
                return retval;
        }
        if (tgtdev->dev_removed == 1)  {
                mpi3mr_dprint(sc, MPI3MR_ERROR, "Device(0x%x) is removed, TM is not required\n",
                              ccb->ccb_h.target_id);
                return retval;
        }
        
        drv_cmd = &sc->host_tm_cmds;
        mtx_lock(&drv_cmd->lock);

        memset(&tm_req, 0, sizeof(tm_req));
        tm_req.DevHandle = htole16(tgtdev->dev_handle);
        tm_req.TaskType = tm_type;
        tm_req.HostTag = htole16(MPI3MR_HOSTTAG_TMS);
        int_to_lun(ccb->ccb_h.target_lun, tm_req.LUN);
        tm_req.Function = MPI3_FUNCTION_SCSI_TASK_MGMT;
        drv_cmd->state = MPI3MR_CMD_PENDING;
        drv_cmd->is_waiting = 1;
        drv_cmd->callback = NULL;

        if (ccb) {
                if (tm_type == MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK) {
                        op_req_q = &sc->op_req_q[cmd->req_qidx];
                        tm_req.TaskHostTag = htole16(cmd->hosttag);
                        tm_req.TaskRequestQueueID = htole16(op_req_q->qid);
                }
        }
        
        if (tgtdev)
                mpi3mr_atomic_inc(&tgtdev->block_io);

        if (tgtdev && (tgtdev->dev_type == MPI3_DEVICE_DEVFORM_PCIE)) {
                if ((tm_type == MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK)
                     && tgtdev->dev_spec.pcie_inf.abort_to)
                        timeout = tgtdev->dev_spec.pcie_inf.abort_to;
                else if ((tm_type == MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET)
                         && tgtdev->dev_spec.pcie_inf.reset_to)
                         timeout = tgtdev->dev_spec.pcie_inf.reset_to;
        }
        
        sc->tm_chan = (void *)&drv_cmd;
        
        mpi3mr_dprint(sc, MPI3MR_DEBUG_TM,
                      "posting task management request: type(%d), handle(0x%04x)\n",
                       tm_type, tgtdev->dev_handle);

        init_completion(&drv_cmd->completion);
        retval = mpi3mr_submit_admin_cmd(sc, &tm_req, sizeof(tm_req));
        if (retval) {
                mpi3mr_dprint(sc, MPI3MR_ERROR,
                              "posting task management request is failed\n");
                retval = -1;
                goto out_unlock;
        }
        wait_for_completion_timeout_tm(&drv_cmd->completion, timeout, sc);

        if (!(drv_cmd->state & MPI3MR_CMD_COMPLETE)) {
                drv_cmd->is_waiting = 0;
                retval = -1;
                if (!(drv_cmd->state & MPI3MR_CMD_RESET)) {
                        mpi3mr_dprint(sc, MPI3MR_ERROR,
                                      "task management request timed out after %ld seconds\n", timeout);
                        if (sc->mpi3mr_debug & MPI3MR_DEBUG_TM) {
                                mpi3mr_dprint(sc, MPI3MR_INFO, "tm_request dump\n");
                                mpi3mr_hexdump(&tm_req, sizeof(tm_req), 8);
                        }
                        trigger_reset_from_watchdog(sc, MPI3MR_TRIGGER_SOFT_RESET, MPI3MR_RESET_FROM_TM_TIMEOUT);
                        retval = ETIMEDOUT;
                }
                goto out_unlock;
        }

        if (!(drv_cmd->state & MPI3MR_CMD_REPLYVALID)) {
                mpi3mr_dprint(sc, MPI3MR_ERROR,
                              "invalid task management reply message\n");
                retval = -1;
                goto out_unlock;
        }
        tm_reply = (MPI3_SCSI_TASK_MGMT_REPLY *)drv_cmd->reply;

        switch (drv_cmd->ioc_status) {
        case MPI3_IOCSTATUS_SUCCESS:
                resp_code = tm_reply->ResponseData & MPI3MR_RI_MASK_RESPCODE;
                break;
        case MPI3_IOCSTATUS_SCSI_IOC_TERMINATED:
                resp_code = MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE;
                break;
        default:
                mpi3mr_dprint(sc, MPI3MR_ERROR,
                              "task management request to handle(0x%04x) is failed with ioc_status(0x%04x) log_info(0x%08x)\n",
                               tgtdev->dev_handle, drv_cmd->ioc_status, drv_cmd->ioc_loginfo);
                retval = -1;
                goto out_unlock;
        }

        switch (resp_code) {
        case MPI3_SCSITASKMGMT_RSPCODE_TM_SUCCEEDED:
        case MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE:
                break;
        case MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC:
                if (tm_type != MPI3_SCSITASKMGMT_TASKTYPE_QUERY_TASK)
                        retval = -1;
                break;
        default:
                retval = -1;
                break;
        }
        
        mpi3mr_dprint(sc, MPI3MR_DEBUG_TM,
                      "task management request type(%d) completed for handle(0x%04x) with ioc_status(0x%04x), log_info(0x%08x)"
                      "termination_count(%u), response:%s(0x%x)\n", tm_type, tgtdev->dev_handle, drv_cmd->ioc_status, drv_cmd->ioc_loginfo,
                      tm_reply->TerminationCount, mpi3mr_tm_response_name(resp_code), resp_code);
        
        if (retval)
                goto out_unlock;

        mpi3mr_disable_interrupts(sc);
        mpi3mr_poll_pend_io_completions(sc);
        mpi3mr_enable_interrupts(sc);
        mpi3mr_poll_pend_io_completions(sc);

        switch (tm_type) {
        case MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
                if (cmd->state == MPI3MR_CMD_STATE_IN_TM) {
                        mpi3mr_dprint(sc, MPI3MR_ERROR,
                                      "%s: task abort returned success from firmware but corresponding CCB (%p) was not terminated"
                                      "marking task abort failed!\n", sc->name, cmd->ccb);
                        retval = -1;
                }
                break;
        case MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
                if (mpi3mr_atomic_read(&tgtdev->outstanding)) {
                        mpi3mr_dprint(sc, MPI3MR_ERROR,
                                      "%s: target reset returned success from firmware but IOs are still pending on the target (%p)"
                                      "marking target reset failed!\n",
                                      sc->name, tgtdev);
                        retval = -1;
                }
                break;
        default:
                break;
        }
        
out_unlock:
        drv_cmd->state = MPI3MR_CMD_NOTUSED;
        mtx_unlock(&drv_cmd->lock);
        if (tgtdev && mpi3mr_atomic_read(&tgtdev->block_io) > 0)
                mpi3mr_atomic_dec(&tgtdev->block_io);
        
        return retval;
}

/**
 * mpi3mr_task_abort- Abort error handling callback
 * @cmd: Timed out command reference
 *
 * Issue Abort Task Management if the command is in LLD scope
 * and verify if it is aborted successfully and return status
 * accordingly.
 *
 * Return: SUCCESS of successful abort the SCSI command else FAILED
 */
static int mpi3mr_task_abort(struct mpi3mr_cmd *cmd)
{
        int retval = 0;
        struct mpi3mr_softc *sc;
        union ccb *ccb;

        sc = cmd->sc;

        if (!cmd->ccb) {
                mpi3mr_dprint(sc, MPI3MR_ERROR, "SCSIIO command timed-out with NULL ccb\n");
                return retval;
        }
        ccb = cmd->ccb;
        
        mpi3mr_dprint(sc, MPI3MR_INFO,
                      "attempting abort task for ccb(%p)\n", ccb);
        
        mpi3mr_print_cdb(ccb);

        if (cmd->state != MPI3MR_CMD_STATE_BUSY) {
                mpi3mr_dprint(sc, MPI3MR_INFO,
                              "%s: ccb is not in driver scope, abort task is not required\n",
                              sc->name);
                return retval;
        }
        cmd->state = MPI3MR_CMD_STATE_IN_TM;

        retval = mpi3mr_issue_tm(sc, cmd, MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK, MPI3MR_ABORTTM_TIMEOUT);
        
        mpi3mr_dprint(sc, MPI3MR_INFO,
                      "abort task is %s for ccb(%p)\n", ((retval == 0) ? "SUCCESS" : "FAILED"), ccb);
        
        return retval;
}

/**
 * mpi3mr_target_reset - Target reset error handling callback
 * @cmd: Timed out command reference
 *
 * Issue Target reset Task Management and verify the SCSI commands are
 * terminated successfully and return status accordingly.
 *
 * Return: SUCCESS of successful termination of the SCSI commands else
 *         FAILED
 */
static int mpi3mr_target_reset(struct mpi3mr_cmd *cmd)
{
        int retval = 0;
        struct mpi3mr_softc *sc;
        struct mpi3mr_target *target;

        sc = cmd->sc;
        
        target = cmd->targ;
        if (target == NULL)  {
                mpi3mr_dprint(sc, MPI3MR_XINFO, "Device does not exist for target:0x%p,"
                              "target reset is not required\n", target);
                return retval;
        }
        
        mpi3mr_dprint(sc, MPI3MR_INFO,
                      "attempting target reset on target(%d)\n", target->per_id);

        
        if (mpi3mr_atomic_read(&target->outstanding)) {
                mpi3mr_dprint(sc, MPI3MR_INFO,
                              "no outstanding IOs on the target(%d),"
                              " target reset not required.\n", target->per_id);
                return retval;
        }
        
        retval = mpi3mr_issue_tm(sc, cmd, MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET, MPI3MR_RESETTM_TIMEOUT);

        mpi3mr_dprint(sc, MPI3MR_INFO,
                      "target reset is %s for target(%d)\n", ((retval == 0) ? "SUCCESS" : "FAILED"),
                      target->per_id);

        return retval;
}

/**
 * mpi3mr_get_fw_pending_ios - Calculate pending I/O count
 * @sc: Adapter instance reference
 *
 * Calculate the pending I/Os for the controller and return.
 *
 * Return: Number of pending I/Os
 */
static inline int mpi3mr_get_fw_pending_ios(struct mpi3mr_softc *sc)
{
        U16 i, pend_ios = 0;

        for (i = 0; i < sc->num_queues; i++)
                pend_ios += mpi3mr_atomic_read(&sc->op_reply_q[i].pend_ios);
        return pend_ios;
}

/**
 * mpi3mr_wait_for_host_io - block for I/Os to complete
 * @sc: Adapter instance reference
 * @timeout: time out in seconds
 *
 * Waits for pending I/Os for the given adapter to complete or
 * to hit the timeout.
 *
 * Return: Nothing
 */
static int mpi3mr_wait_for_host_io(struct mpi3mr_softc *sc, U32 timeout)
{
        enum mpi3mr_iocstate iocstate;

        iocstate = mpi3mr_get_iocstate(sc);
        if (iocstate != MRIOC_STATE_READY) {
                mpi3mr_dprint(sc, MPI3MR_XINFO, "%s :Controller is in NON-READY state! Proceed with Reset\n", __func__);
                return -1;
        }

        if (!mpi3mr_get_fw_pending_ios(sc))
                return 0;
        
        mpi3mr_dprint(sc, MPI3MR_INFO,
                      "%s :Waiting for %d seconds prior to reset for %d pending I/Os to complete\n",
                      __func__, timeout, mpi3mr_get_fw_pending_ios(sc));

        int i;
        for (i = 0; i < timeout; i++) {
                if (!mpi3mr_get_fw_pending_ios(sc)) {
                        mpi3mr_dprint(sc, MPI3MR_INFO, "%s :All pending I/Os got completed while waiting! Reset not required\n", __func__);
                        return 0;
                        
                }
                iocstate = mpi3mr_get_iocstate(sc);
                if (iocstate != MRIOC_STATE_READY) {
                        mpi3mr_dprint(sc, MPI3MR_XINFO, "%s :Controller state becomes NON-READY while waiting! dont wait further"
                                      "Proceed with Reset\n", __func__);
                        return -1;
                }
                DELAY(1000 * 1000);
        }

        mpi3mr_dprint(sc, MPI3MR_INFO, "%s :Pending I/Os after wait exaust is %d! Proceed with Reset\n", __func__,
                      mpi3mr_get_fw_pending_ios(sc));

        return -1;
}

static void
mpi3mr_scsiio_timeout(void *data)
{
        int retval = 0;
        struct mpi3mr_softc *sc;
        struct mpi3mr_cmd *cmd;
        struct mpi3mr_target *targ_dev = NULL;

        if (!data)
                return;

        cmd = (struct mpi3mr_cmd *)data;
        sc = cmd->sc;

        if (cmd->ccb == NULL) {
                mpi3mr_dprint(sc, MPI3MR_ERROR, "SCSIIO command timed-out with NULL ccb\n");
                return;
        }

        /*
         * TMs are not supported for IO timeouts on VD/LD, so directly issue controller reset
         * with max timeout for outstanding IOs to complete is 180sec.
         */
        targ_dev = cmd->targ;
        if (targ_dev && (targ_dev->dev_type == MPI3_DEVICE_DEVFORM_VD)) {
                if (mpi3mr_wait_for_host_io(sc, MPI3MR_RAID_ERRREC_RESET_TIMEOUT))
                        trigger_reset_from_watchdog(sc, MPI3MR_TRIGGER_SOFT_RESET, MPI3MR_RESET_FROM_SCSIIO_TIMEOUT);
                return;
        }
        
        /* Issue task abort to recover the timed out IO */
        retval = mpi3mr_task_abort(cmd);
        if (!retval || (retval == ETIMEDOUT))
                return;
        
        /*
         * task abort has failed to recover the timed out IO,
         * try with the target reset
         */
        retval = mpi3mr_target_reset(cmd);
        if (!retval || (retval == ETIMEDOUT))
                return;

        /*
         * task abort and target reset has failed. So issue Controller reset(soft reset)
         * through OCR thread context
         */
        trigger_reset_from_watchdog(sc, MPI3MR_TRIGGER_SOFT_RESET, MPI3MR_RESET_FROM_SCSIIO_TIMEOUT);

        return;
}

void int_to_lun(unsigned int lun, U8 *req_lun)
{
        int i;
        
        memset(req_lun, 0, sizeof(*req_lun));

        for (i = 0; i < sizeof(lun); i += 2) {
                req_lun[i] = (lun >> 8) & 0xFF;
                req_lun[i+1] = lun & 0xFF;
                lun = lun >> 16;
        }

}

static U16 get_req_queue_index(struct mpi3mr_softc *sc)
{
        U16 i = 0, reply_q_index = 0, reply_q_pend_ios = 0;
        
        reply_q_pend_ios = mpi3mr_atomic_read(&sc->op_reply_q[0].pend_ios);
        for (i = 0; i < sc->num_queues; i++) {
                if (reply_q_pend_ios > mpi3mr_atomic_read(&sc->op_reply_q[i].pend_ios)) {
                        reply_q_pend_ios = mpi3mr_atomic_read(&sc->op_reply_q[i].pend_ios);
                        reply_q_index = i;
                }
        }
        
        return reply_q_index;
}

static void
mpi3mr_action_scsiio(struct mpi3mr_cam_softc *cam_sc, union ccb *ccb)
{
        Mpi3SCSIIORequest_t *req = NULL;
        struct ccb_scsiio *csio;
        struct mpi3mr_softc *sc;
        struct mpi3mr_target *targ;
        struct mpi3mr_cmd *cm;
        uint8_t scsi_opcode, queue_idx;
        uint32_t mpi_control;

        sc = cam_sc->sc;
        mtx_assert(&sc->mpi3mr_mtx, MA_OWNED);
        
        if (sc->unrecoverable) {
                mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
                xpt_done(ccb);
                return;
        }

        csio = &ccb->csio;
        KASSERT(csio->ccb_h.target_id < cam_sc->maxtargets,
            ("Target %d out of bounds in XPT_SCSI_IO\n",
             csio->ccb_h.target_id));
        
        scsi_opcode = scsiio_cdb_ptr(csio)[0];
        
        if ((sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN) &&
            !((scsi_opcode == SYNCHRONIZE_CACHE) ||
              (scsi_opcode == START_STOP_UNIT))) {
                mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP);
                xpt_done(ccb);
                return;
        }

        targ = mpi3mr_find_target_by_per_id(cam_sc, csio->ccb_h.target_id);
        if (targ == NULL)  {
                mpi3mr_dprint(sc, MPI3MR_XINFO, "Device with target ID: 0x%x does not exist\n",
                              csio->ccb_h.target_id);
                mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
                xpt_done(ccb);
                return;
        }
        
        if (targ && targ->is_hidden)  {
                mpi3mr_dprint(sc, MPI3MR_XINFO, "Device with target ID: 0x%x is hidden\n",
                              csio->ccb_h.target_id);
                mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
                xpt_done(ccb);
                return;
        }

        if (targ->dev_removed == 1)  {
                mpi3mr_dprint(sc, MPI3MR_XINFO, "Device with target ID: 0x%x is removed\n", csio->ccb_h.target_id);
                mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
                xpt_done(ccb);
                return;
        }

        if (targ->dev_handle == 0x0) {
                mpi3mr_dprint(sc, MPI3MR_ERROR, "%s NULL handle for target 0x%x\n",
                    __func__, csio->ccb_h.target_id);
                mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
                xpt_done(ccb);
                return;
        }

        if (mpi3mr_atomic_read(&targ->block_io) ||
                (sc->reset_in_progress == 1) || (sc->prepare_for_reset == 1)) {
                mpi3mr_dprint(sc, MPI3MR_TRACE, "%s target is busy target_id: 0x%x\n",
                    __func__, csio->ccb_h.target_id);
                mpi3mr_set_ccbstatus(ccb, CAM_REQUEUE_REQ);
                xpt_done(ccb);
                return;
        }

        /*
         * Sometimes, it is possible to get a command that is not "In
         * Progress" and was actually aborted by the upper layer.  Check for
         * this here and complete the command without error.
         */
        if (mpi3mr_get_ccbstatus(ccb) != CAM_REQ_INPROG) {
                mpi3mr_dprint(sc, MPI3MR_TRACE, "%s Command is not in progress for "
                    "target %u\n", __func__, csio->ccb_h.target_id);
                xpt_done(ccb);
                return;
        }
        /*
         * If devinfo is 0 this will be a volume.  In that case don't tell CAM
         * that the volume has timed out.  We want volumes to be enumerated
         * until they are deleted/removed, not just failed.
         */
        if (targ->flags & MPI3MRSAS_TARGET_INREMOVAL) {
                if (targ->devinfo == 0)
                        mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP);
                else
                        mpi3mr_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
                xpt_done(ccb);
                return;
        }

        if ((scsi_opcode == UNMAP) &&
                (pci_get_device(sc->mpi3mr_dev) == MPI3_MFGPAGE_DEVID_SAS4116) &&
                (targ->dev_type == MPI3_DEVICE_DEVFORM_PCIE) &&
                (mpi3mr_allow_unmap_to_fw(sc, ccb) == false))
                return;

        cm = mpi3mr_get_command(sc);
        if (cm == NULL || (sc->mpi3mr_flags & MPI3MR_FLAGS_DIAGRESET)) {
                if (cm != NULL) {
                        mpi3mr_release_command(cm);
                }
                if ((cam_sc->flags & MPI3MRSAS_QUEUE_FROZEN) == 0) {
                        xpt_freeze_simq(cam_sc->sim, 1);
                        cam_sc->flags |= MPI3MRSAS_QUEUE_FROZEN;
                }
                ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
                mpi3mr_set_ccbstatus(ccb, CAM_REQUEUE_REQ);
                xpt_done(ccb);
                return;
        }

        switch (csio->ccb_h.flags & CAM_DIR_MASK) {
        case CAM_DIR_IN:
                mpi_control = MPI3_SCSIIO_FLAGS_DATADIRECTION_READ;
                cm->data_dir = MPI3MR_READ;
                break;
        case CAM_DIR_OUT:
                mpi_control = MPI3_SCSIIO_FLAGS_DATADIRECTION_WRITE;
                cm->data_dir = MPI3MR_WRITE;
                break;
        case CAM_DIR_NONE:
        default:
                mpi_control = MPI3_SCSIIO_FLAGS_DATADIRECTION_NO_DATA_TRANSFER;
                break;
        }

        if (csio->cdb_len > 16)
                mpi_control |= MPI3_SCSIIO_FLAGS_CDB_GREATER_THAN_16;

        req = (Mpi3SCSIIORequest_t *)&cm->io_request;
        bzero(req, sizeof(*req));
        req->Function = MPI3_FUNCTION_SCSI_IO;
        req->HostTag = cm->hosttag;
        req->DataLength = htole32(csio->dxfer_len);
        req->DevHandle = htole16(targ->dev_handle);

        /*
         * It looks like the hardware doesn't require an explicit tag
         * number for each transaction.  SAM Task Management not supported
         * at the moment.
         */
        switch (csio->tag_action) {
        case MSG_HEAD_OF_Q_TAG:
                mpi_control |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_HEADOFQ;
                break;
        case MSG_ORDERED_Q_TAG:
                mpi_control |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_ORDEREDQ;
                break;
        case MSG_ACA_TASK:
                mpi_control |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_ACAQ;
                break;
        case CAM_TAG_ACTION_NONE:
        case MSG_SIMPLE_Q_TAG:
        default:
                mpi_control |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_SIMPLEQ;
                break;
        }

        if (targ->ws_len)
                mpi3mr_divert_ws(req, csio, targ->ws_len);

        req->Flags = htole32(mpi_control);

        if (csio->ccb_h.flags & CAM_CDB_POINTER)
                bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
        else {
                KASSERT(csio->cdb_len <= IOCDBLEN,
                    ("cdb_len %d is greater than IOCDBLEN but CAM_CDB_POINTER "
                    "is not set", csio->cdb_len));
                bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
        }

        cm->length = csio->dxfer_len;
        cm->targ = targ;
        int_to_lun(csio->ccb_h.target_lun, req->LUN);
        cm->ccb = ccb;
        csio->ccb_h.qos.sim_data = sbinuptime();
        queue_idx = get_req_queue_index(sc);
        cm->req_qidx = queue_idx;
        
        mpi3mr_dprint(sc, MPI3MR_TRACE, "[QID:%d]: func: %s line:%d CDB: 0x%x targetid: %x SMID: 0x%x\n",
                (queue_idx + 1), __func__, __LINE__, scsi_opcode, csio->ccb_h.target_id, cm->hosttag);

        switch ((ccb->ccb_h.flags & CAM_DATA_MASK)) {
        case CAM_DATA_PADDR:
        case CAM_DATA_SG_PADDR:
                device_printf(sc->mpi3mr_dev, "%s: physical addresses not supported\n",
                    __func__);
                mpi3mr_set_ccbstatus(ccb, CAM_REQ_INVALID);
                mpi3mr_release_command(cm);
                xpt_done(ccb);
                return;
        case CAM_DATA_SG:
                device_printf(sc->mpi3mr_dev, "%s: scatter gather is not supported\n",
                    __func__);
                mpi3mr_set_ccbstatus(ccb, CAM_REQ_INVALID);
                mpi3mr_release_command(cm);
                xpt_done(ccb);
                return;
        case CAM_DATA_VADDR:
        case CAM_DATA_BIO:
                if (csio->dxfer_len > (sc->max_sgl_entries * PAGE_SIZE)) {
                        mpi3mr_set_ccbstatus(ccb, CAM_REQ_TOO_BIG);
                        mpi3mr_release_command(cm);
                        xpt_done(ccb);
                        return;
                }
                ccb->ccb_h.status |= CAM_SIM_QUEUED;
                cm->length = csio->dxfer_len;
                if (cm->length)
                        cm->data = csio->data_ptr;
                break;
        default:
                mpi3mr_set_ccbstatus(ccb, CAM_REQ_INVALID);
                mpi3mr_release_command(cm);
                xpt_done(ccb);
                return;
        }

        /* Prepare SGEs and queue to hardware */
        mpi3mr_map_request(sc, cm);
}

static void
mpi3mr_enqueue_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cm)
{
        static int ratelimit;
        struct mpi3mr_op_req_queue *opreqq = &sc->op_req_q[cm->req_qidx];
        struct mpi3mr_throttle_group_info *tg = NULL;
        uint32_t data_len_blks = 0;
        uint32_t tracked_io_sz = 0;
        uint32_t ioc_pend_data_len = 0, tg_pend_data_len = 0;
        struct mpi3mr_target *targ = cm->targ;
        union ccb *ccb = cm->ccb;
        Mpi3SCSIIORequest_t *req = (Mpi3SCSIIORequest_t *)&cm->io_request;

        if (sc->iot_enable) {
                data_len_blks = ccb->csio.dxfer_len >> 9;

                if ((data_len_blks >= sc->io_throttle_data_length) &&
                    targ->io_throttle_enabled) {

                        tracked_io_sz = data_len_blks;
                        tg = targ->throttle_group;
                        if (tg) {
                                mpi3mr_atomic_add(&sc->pend_large_data_sz, data_len_blks);
                                mpi3mr_atomic_add(&tg->pend_large_data_sz, data_len_blks);
                                
                                ioc_pend_data_len = mpi3mr_atomic_read(&sc->pend_large_data_sz);
                                tg_pend_data_len = mpi3mr_atomic_read(&tg->pend_large_data_sz);

                                if (ratelimit % 1000) {
                                        mpi3mr_dprint(sc, MPI3MR_IOT,
                                                "large vd_io persist_id(%d), handle(0x%04x), data_len(%d),"
                                                "ioc_pending(%d), tg_pending(%d), ioc_high(%d), tg_high(%d)\n",
                                                targ->per_id, targ->dev_handle,
                                                data_len_blks, ioc_pend_data_len,
                                                tg_pend_data_len, sc->io_throttle_high,
                                                tg->high);
                                        ratelimit++;
                                }

                                if (!tg->io_divert  && ((ioc_pend_data_len >=
                                    sc->io_throttle_high) ||
                                    (tg_pend_data_len >= tg->high))) {
                                        tg->io_divert = 1;
                                        mpi3mr_dprint(sc, MPI3MR_IOT,
                                                "VD: Setting divert flag for tg_id(%d), persist_id(%d)\n",
                                                tg->id, targ->per_id);
                                        if (sc->mpi3mr_debug & MPI3MR_IOT)
                                                mpi3mr_print_cdb(ccb);
                                        mpi3mr_set_io_divert_for_all_vd_in_tg(sc,
                                            tg, 1);
                                }
                        } else {
                                mpi3mr_atomic_add(&sc->pend_large_data_sz, data_len_blks);
                                ioc_pend_data_len = mpi3mr_atomic_read(&sc->pend_large_data_sz);
                                if (ratelimit % 1000) {
                                        mpi3mr_dprint(sc, MPI3MR_IOT,
                                            "large pd_io persist_id(%d), handle(0x%04x), data_len(%d), ioc_pending(%d), ioc_high(%d)\n",
                                            targ->per_id, targ->dev_handle,
                                            data_len_blks, ioc_pend_data_len,
                                            sc->io_throttle_high);
                                        ratelimit++;
                                }

                                if (ioc_pend_data_len >= sc->io_throttle_high) {
                                        targ->io_divert = 1;
                                        mpi3mr_dprint(sc, MPI3MR_IOT,
                                                "PD: Setting divert flag for persist_id(%d)\n",
                                                targ->per_id);
                                        if (sc->mpi3mr_debug & MPI3MR_IOT)
                                                mpi3mr_print_cdb(ccb);
                                }
                        }
                }

                if (targ->io_divert) {
                        req->MsgFlags |= MPI3_SCSIIO_MSGFLAGS_DIVERT_TO_FIRMWARE;
                        req->Flags = htole32(le32toh(req->Flags) | MPI3_SCSIIO_FLAGS_DIVERT_REASON_IO_THROTTLING);
                }
        }

        if (mpi3mr_submit_io(sc, opreqq, (U8 *)&cm->io_request)) {
                if (tracked_io_sz) {
                        mpi3mr_atomic_sub(&sc->pend_large_data_sz, tracked_io_sz);
                        if (tg)
                                mpi3mr_atomic_sub(&tg->pend_large_data_sz, tracked_io_sz);
                }
                mpi3mr_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
                mpi3mr_release_command(cm);
                xpt_done(ccb);
        } else {
                callout_reset_sbt(&cm->callout, mstosbt(ccb->ccb_h.timeout), 0,
                    mpi3mr_scsiio_timeout, cm, 0);
                cm->callout_owner = true;
                mpi3mr_atomic_inc(&sc->fw_outstanding);
                mpi3mr_atomic_inc(&targ->outstanding);
                if (mpi3mr_atomic_read(&sc->fw_outstanding) > sc->io_cmds_highwater)
                        sc->io_cmds_highwater++;
        }

        return;
}

static void
mpi3mr_cam_poll(struct cam_sim *sim)
{
        struct mpi3mr_cam_softc *cam_sc;
        struct mpi3mr_irq_context *irq_ctx;
        struct mpi3mr_softc *sc;
        int i;

        cam_sc = cam_sim_softc(sim);
        sc = cam_sc->sc;

        mpi3mr_dprint(cam_sc->sc, MPI3MR_TRACE, "func: %s line: %d is called\n",
                __func__, __LINE__);

        for (i = 0; i < sc->num_queues; i++) {
                irq_ctx = sc->irq_ctx + i;
                if (irq_ctx->op_reply_q->qid) {
                        mpi3mr_complete_io_cmd(sc, irq_ctx);
                }
        }
}

static void
mpi3mr_cam_action(struct cam_sim *sim, union ccb *ccb)
{
        struct mpi3mr_cam_softc *cam_sc;
        struct mpi3mr_target *targ;
        struct mpi3mr_softc *sc;

        cam_sc = cam_sim_softc(sim);
        sc = cam_sc->sc;

        mpi3mr_dprint(cam_sc->sc, MPI3MR_TRACE, "ccb func_code 0x%x target id: 0x%x\n",
            ccb->ccb_h.func_code, ccb->ccb_h.target_id);

        mtx_assert(&cam_sc->sc->mpi3mr_mtx, MA_OWNED);

        switch (ccb->ccb_h.func_code) {
        case XPT_PATH_INQ:
        {
                struct ccb_pathinq *cpi = &ccb->cpi;

                cpi->version_num = 1;
                cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16;
                cpi->target_sprt = 0;
                cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED | PIM_NOSCAN;
                cpi->hba_eng_cnt = 0;
                cpi->max_target = cam_sc->maxtargets - 1;
                cpi->max_lun = 0;

                /*
                 * initiator_id is set here to an ID outside the set of valid
                 * target IDs (including volumes).
                 */
                cpi->initiator_id = cam_sc->maxtargets;
                strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
                strlcpy(cpi->hba_vid, "Broadcom", HBA_IDLEN);
                strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
                cpi->unit_number = cam_sim_unit(sim);
                cpi->bus_id = cam_sim_bus(sim);
                /*
                 * XXXSLM-I think this needs to change based on config page or
                 * something instead of hardcoded to 150000.
                 */
                cpi->base_transfer_speed = 150000;
                cpi->transport = XPORT_SAS;
                cpi->transport_version = 0;
                cpi->protocol = PROTO_SCSI;
                cpi->protocol_version = SCSI_REV_SPC;

                targ = mpi3mr_find_target_by_per_id(cam_sc, ccb->ccb_h.target_id);

                if (targ && (targ->dev_type == MPI3_DEVICE_DEVFORM_PCIE) &&
                    ((targ->dev_spec.pcie_inf.dev_info &
                    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) ==
                    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_NVME_DEVICE)) {
                        cpi->maxio = targ->dev_spec.pcie_inf.mdts;
                        mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO,
                                "PCI device target_id: %u max io size: %u\n",
                                ccb->ccb_h.target_id, cpi->maxio);
                } else {
                        cpi->maxio = PAGE_SIZE * (sc->max_sgl_entries - 1);
                }
                mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP);
                break;
        }
        case XPT_GET_TRAN_SETTINGS:
        {
                struct ccb_trans_settings       *cts;
                struct ccb_trans_settings_sas   *sas;
                struct ccb_trans_settings_scsi  *scsi;

                cts = &ccb->cts;
                sas = &cts->xport_specific.sas;
                scsi = &cts->proto_specific.scsi;

                KASSERT(cts->ccb_h.target_id < cam_sc->maxtargets,
                    ("Target %d out of bounds in XPT_GET_TRAN_SETTINGS\n",
                    cts->ccb_h.target_id));
                targ = mpi3mr_find_target_by_per_id(cam_sc, cts->ccb_h.target_id);
                
                if (targ == NULL) {
                        mpi3mr_dprint(cam_sc->sc, MPI3MR_TRACE, "Device with target ID: 0x%x does not exist\n",
                        cts->ccb_h.target_id);
                        mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
                        break;
                }

                if ((targ->dev_handle == 0x0) || (targ->dev_removed == 1))  {
                        mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
                        break;
                }

                cts->protocol_version = SCSI_REV_SPC2;
                cts->transport = XPORT_SAS;
                cts->transport_version = 0;

                sas->valid = CTS_SAS_VALID_SPEED;
                
                switch (targ->link_rate) {
                case 0x08:
                        sas->bitrate = 150000;
                        break;
                case 0x09:
                        sas->bitrate = 300000;
                        break;
                case 0x0a:
                        sas->bitrate = 600000;
                        break;
                case 0x0b:
                        sas->bitrate = 1200000;
                        break;
                default:
                        sas->valid = 0;
                }

                cts->protocol = PROTO_SCSI;
                scsi->valid = CTS_SCSI_VALID_TQ;
                scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;

                mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP);
                break;
        }
        case XPT_CALC_GEOMETRY:
                cam_calc_geometry(&ccb->ccg, /*extended*/1);
                mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP);
                break;
        case XPT_RESET_DEV:
                mpi3mr_dprint(cam_sc->sc, MPI3MR_INFO, "mpi3mr_action "
                    "XPT_RESET_DEV\n");
                return;
        case XPT_RESET_BUS:
        case XPT_ABORT:
        case XPT_TERM_IO:
                mpi3mr_dprint(cam_sc->sc, MPI3MR_INFO, "mpi3mr_action faking success "
                    "for abort or reset\n");
                mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP);
                break;
        case XPT_SCSI_IO:
                mpi3mr_action_scsiio(cam_sc, ccb);
                return;
        default:
                mpi3mr_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL);
                break;
        }
        xpt_done(ccb);
}

void
mpi3mr_startup_increment(struct mpi3mr_cam_softc *cam_sc)
{
        if ((cam_sc->flags & MPI3MRSAS_IN_STARTUP) != 0) {
                if (cam_sc->startup_refcount++ == 0) {
                        /* just starting, freeze the simq */
                        mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO,
                            "%s freezing simq\n", __func__);
                        xpt_hold_boot();
                }
                mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO, "%s refcount %u\n", __func__,
                    cam_sc->startup_refcount);
        }
}

void
mpi3mr_release_simq_reinit(struct mpi3mr_cam_softc *cam_sc)
{
        if (cam_sc->flags & MPI3MRSAS_QUEUE_FROZEN) {
                cam_sc->flags &= ~MPI3MRSAS_QUEUE_FROZEN;
                xpt_release_simq(cam_sc->sim, 1);
                mpi3mr_dprint(cam_sc->sc, MPI3MR_INFO, "Unfreezing SIM queue\n");
        }
}

void
mpi3mr_rescan_target(struct mpi3mr_softc *sc, struct mpi3mr_target *targ)
{
        struct mpi3mr_cam_softc *cam_sc = sc->cam_sc;
        path_id_t pathid;
        target_id_t targetid;
        union ccb *ccb;

        pathid = cam_sim_path(cam_sc->sim);
        if (targ == NULL)
                targetid = CAM_TARGET_WILDCARD;
        else
                targetid = targ->per_id;

        /*
         * Allocate a CCB and schedule a rescan.
         */
        ccb = xpt_alloc_ccb_nowait();
        if (ccb == NULL) {
                mpi3mr_dprint(sc, MPI3MR_ERROR, "unable to alloc CCB for rescan\n");
                return;
        }

        if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, targetid,
            CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
                mpi3mr_dprint(sc, MPI3MR_ERROR, "unable to create path for rescan\n");
                xpt_free_ccb(ccb);
                return;
        }

        if (targetid == CAM_TARGET_WILDCARD)
                ccb->ccb_h.func_code = XPT_SCAN_BUS;
        else
                ccb->ccb_h.func_code = XPT_SCAN_TGT;

        mpi3mr_dprint(sc, MPI3MR_EVENT, "%s target id 0x%x\n", __func__, targetid);
        xpt_rescan(ccb);
}

void
mpi3mr_startup_decrement(struct mpi3mr_cam_softc *cam_sc)
{
        if ((cam_sc->flags & MPI3MRSAS_IN_STARTUP) != 0) {
                if (--cam_sc->startup_refcount == 0) {
                        /* finished all discovery-related actions, release
                         * the simq and rescan for the latest topology.
                         */
                        mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO,
                            "%s releasing simq\n", __func__);
                        cam_sc->flags &= ~MPI3MRSAS_IN_STARTUP;
                        xpt_release_simq(cam_sc->sim, 1);
                        xpt_release_boot();
                }
                mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO, "%s refcount %u\n", __func__,
                    cam_sc->startup_refcount);
        }
}

static void
mpi3mr_fw_event_free(struct mpi3mr_softc *sc, struct mpi3mr_fw_event_work *fw_event)
{
        if (!fw_event)
                return;

        if (fw_event->event_data != NULL) {
                free(fw_event->event_data, M_MPI3MR);
                fw_event->event_data = NULL;
        }

        free(fw_event, M_MPI3MR);
        fw_event = NULL;
}

static void
mpi3mr_freeup_events(struct mpi3mr_softc *sc)
{
        struct mpi3mr_fw_event_work *fw_event = NULL;
        mtx_lock(&sc->mpi3mr_mtx);
        while ((fw_event = TAILQ_FIRST(&sc->cam_sc->ev_queue)) != NULL) {
                TAILQ_REMOVE(&sc->cam_sc->ev_queue, fw_event, ev_link);
                mpi3mr_fw_event_free(sc, fw_event);
        }
        mtx_unlock(&sc->mpi3mr_mtx);
}

static void
mpi3mr_sastopochg_evt_debug(struct mpi3mr_softc *sc,
        Mpi3EventDataSasTopologyChangeList_t *event_data)
{
        int i;
        U16 handle;
        U8 reason_code, phy_number;
        char *status_str = NULL;
        U8 link_rate, prev_link_rate;

        switch (event_data->ExpStatus) {
        case MPI3_EVENT_SAS_TOPO_ES_NOT_RESPONDING:
                status_str = "remove";
                break;
        case MPI3_EVENT_SAS_TOPO_ES_RESPONDING:
                status_str =  "responding";
                break;
        case MPI3_EVENT_SAS_TOPO_ES_DELAY_NOT_RESPONDING:
                status_str = "remove delay";
                break;
        case MPI3_EVENT_SAS_TOPO_ES_NO_EXPANDER:
                status_str = "direct attached";
                break;
        default:
                status_str = "unknown status";
                break;
        }

        mpi3mr_dprint(sc, MPI3MR_INFO, "%s :sas topology change: (%s)\n",
            __func__, status_str);
        mpi3mr_dprint(sc, MPI3MR_INFO,
                "%s :\texpander_handle(0x%04x), enclosure_handle(0x%04x) "
            "start_phy(%02d), num_entries(%d)\n", __func__,
            (event_data->ExpanderDevHandle),
            (event_data->EnclosureHandle),
            event_data->StartPhyNum, event_data->NumEntries);
        for (i = 0; i < event_data->NumEntries; i++) {
                handle = (event_data->PhyEntry[i].AttachedDevHandle);
                if (!handle)
                        continue;
                phy_number = event_data->StartPhyNum + i;
                reason_code = event_data->PhyEntry[i].PhyStatus &
                    MPI3_EVENT_SAS_TOPO_PHY_RC_MASK;
                switch (reason_code) {
                case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING:
                        status_str = "target remove";
                        break;
                case MPI3_EVENT_SAS_TOPO_PHY_RC_DELAY_NOT_RESPONDING:
                        status_str = "delay target remove";
                        break;
                case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED:
                        status_str = "link rate change";
                        break;
                case MPI3_EVENT_SAS_TOPO_PHY_RC_NO_CHANGE:
                        status_str = "target responding";
                        break;
                default:
                        status_str = "unknown";
                        break;
                }
                link_rate = event_data->PhyEntry[i].LinkRate >> 4;
                prev_link_rate = event_data->PhyEntry[i].LinkRate & 0xF;
                mpi3mr_dprint(sc, MPI3MR_INFO, "%s :\tphy(%02d), attached_handle(0x%04x): %s:"
                    " link rate: new(0x%02x), old(0x%02x)\n", __func__,
                    phy_number, handle, status_str, link_rate, prev_link_rate);
        }
}

static void
mpi3mr_process_sastopochg_evt(struct mpi3mr_softc *sc, struct mpi3mr_fw_event_work *fwevt)
{

        Mpi3EventDataSasTopologyChangeList_t *event_data =
                    (Mpi3EventDataSasTopologyChangeList_t *)fwevt->event_data;
        int i;
        U16 handle;
        U8 reason_code, link_rate;
        struct mpi3mr_target *target = NULL;
        
        
        mpi3mr_sastopochg_evt_debug(sc, event_data);

        for (i = 0; i < event_data->NumEntries; i++) {
                handle = le16toh(event_data->PhyEntry[i].AttachedDevHandle);
                link_rate = event_data->PhyEntry[i].LinkRate >> 4;
                
                if (!handle)
                        continue;
                target = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle);

                if (!target)
                        continue;
                
                target->link_rate = link_rate;
                reason_code = event_data->PhyEntry[i].PhyStatus &
                        MPI3_EVENT_SAS_TOPO_PHY_RC_MASK;

                switch (reason_code) {
                case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING:
                        if (target->exposed_to_os)
                                mpi3mr_remove_device_from_os(sc, target->dev_handle);
                        mpi3mr_remove_device_from_list(sc, target, false);
                        break;
                case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED:
                        break;
                default:
                        break;
                }
        }

        /*
         * refcount was incremented for this event in
         * mpi3mr_evt_handler. Decrement it here because the event has
         * been processed.
         */
        mpi3mr_startup_decrement(sc->cam_sc);
        return;
}

static void
mpi3mr_pcietopochg_evt_debug(struct mpi3mr_softc *sc,
        Mpi3EventDataPcieTopologyChangeList_t *event_data)
{
        int i;
        U16 handle;
        U16 reason_code;
        U8 port_number;
        char *status_str = NULL;
        U8 link_rate, prev_link_rate;

        switch (event_data->SwitchStatus) {
        case MPI3_EVENT_PCIE_TOPO_SS_NOT_RESPONDING:
                status_str = "remove";
                break;
        case MPI3_EVENT_PCIE_TOPO_SS_RESPONDING:
                status_str =  "responding";
                break;
        case MPI3_EVENT_PCIE_TOPO_SS_DELAY_NOT_RESPONDING:
                status_str = "remove delay";
                break;
        case MPI3_EVENT_PCIE_TOPO_SS_NO_PCIE_SWITCH:
                status_str = "direct attached";
                break;
        default:
                status_str = "unknown status";
                break;
        }
        mpi3mr_dprint(sc, MPI3MR_INFO, "%s :pcie topology change: (%s)\n",
                __func__, status_str);
        mpi3mr_dprint(sc, MPI3MR_INFO,
                "%s :\tswitch_handle(0x%04x), enclosure_handle(0x%04x)"
                "start_port(%02d), num_entries(%d)\n", __func__,
                le16toh(event_data->SwitchDevHandle),
                le16toh(event_data->EnclosureHandle),
                event_data->StartPortNum, event_data->NumEntries);
        for (i = 0; i < event_data->NumEntries; i++) {
                handle =
                        le16toh(event_data->PortEntry[i].AttachedDevHandle);
                if (!handle)
                        continue;
                port_number = event_data->StartPortNum + i;
                reason_code = event_data->PortEntry[i].PortStatus;
                switch (reason_code) {
                case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
                        status_str = "target remove";
                        break;
                case MPI3_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING:
                        status_str = "delay target remove";
                        break;
                case MPI3_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
                        status_str = "link rate change";
                        break;
                case MPI3_EVENT_PCIE_TOPO_PS_NO_CHANGE:
                        status_str = "target responding";
                        break;
                default:
                        status_str = "unknown";
                        break;
                }
                link_rate = event_data->PortEntry[i].CurrentPortInfo &
                        MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK;
                prev_link_rate = event_data->PortEntry[i].PreviousPortInfo &
                        MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK;
                mpi3mr_dprint(sc, MPI3MR_INFO, "%s :\tport(%02d), attached_handle(0x%04x): %s:"
                    " link rate: new(0x%02x), old(0x%02x)\n", __func__,
                    port_number, handle, status_str, link_rate, prev_link_rate);
        }
}

static void mpi3mr_process_pcietopochg_evt(struct mpi3mr_softc *sc,
    struct mpi3mr_fw_event_work *fwevt)
{
        Mpi3EventDataPcieTopologyChangeList_t *event_data =
                    (Mpi3EventDataPcieTopologyChangeList_t *)fwevt->event_data;
        int i;
        U16 handle;
        U8 reason_code, link_rate;
        struct mpi3mr_target *target = NULL;
        

        mpi3mr_pcietopochg_evt_debug(sc, event_data);

        for (i = 0; i < event_data->NumEntries; i++) {
                handle =
                        le16toh(event_data->PortEntry[i].AttachedDevHandle);
                if (!handle)
                        continue;
                target = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle);
                if (!target)
                        continue;
                
                link_rate = event_data->PortEntry[i].CurrentPortInfo &
                        MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK;
                target->link_rate = link_rate;

                reason_code = event_data->PortEntry[i].PortStatus;

                switch (reason_code) {
                case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
                        if (target->exposed_to_os)
                                mpi3mr_remove_device_from_os(sc, target->dev_handle);
                        mpi3mr_remove_device_from_list(sc, target, false);
                        break;
                case MPI3_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
                        break;
                default:
                        break;
                }
        }

        /*
         * refcount was incremented for this event in
         * mpi3mr_evt_handler. Decrement it here because the event has
         * been processed.
         */
        mpi3mr_startup_decrement(sc->cam_sc);
        return;
}

void mpi3mr_add_device(struct mpi3mr_softc *sc, U16 per_id)
{
        struct mpi3mr_target *target;

        mpi3mr_dprint(sc, MPI3MR_EVENT,
                "Adding device(persistent id: 0x%x)\n", per_id);

        mpi3mr_startup_increment(sc->cam_sc);
        target = mpi3mr_find_target_by_per_id(sc->cam_sc, per_id);
        
        if (!target) {
                mpi3mr_dprint(sc, MPI3MR_INFO, "Not available in driver's"
                    "internal target list, persistent_id: %d\n",
                    per_id);
                goto out;
        }

        if (target->is_hidden) {
                mpi3mr_dprint(sc, MPI3MR_EVENT, "Target is hidden, persistent_id: %d\n",
                        per_id);
                goto out;
        }

        if (!target->exposed_to_os && !sc->reset_in_progress) {
                mpi3mr_rescan_target(sc, target);
                mpi3mr_dprint(sc, MPI3MR_INFO,
                        "Added device persistent_id: %d dev_handle: %d\n", per_id, target->dev_handle);
                target->exposed_to_os = 1;
        }

out:
        mpi3mr_startup_decrement(sc->cam_sc);
}

int mpi3mr_remove_device_from_os(struct mpi3mr_softc *sc, U16 handle)
{
        int retval = 0;
        struct mpi3mr_target *target;
        unsigned int target_outstanding;

        mpi3mr_dprint(sc, MPI3MR_EVENT,
                "Removing Device (dev_handle: %d)\n", handle);
                        
        target = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle);
        
        if (!target) {
                mpi3mr_dprint(sc, MPI3MR_INFO,
                        "Device (persistent_id: %d dev_handle: %d) is already removed from driver's list\n",
                        target->per_id, handle);
                mpi3mr_rescan_target(sc, NULL);
                retval = -1;
                goto out;
        }

        target->flags |= MPI3MRSAS_TARGET_INREMOVAL;

        target_outstanding = mpi3mr_atomic_read(&target->outstanding);
        if (target_outstanding) {
                mpi3mr_dprint(sc, MPI3MR_ERROR, "there are [%2d] outstanding IOs on target: %d "
                              "Poll reply queue once\n", target_outstanding, target->per_id);
                mpi3mr_poll_pend_io_completions(sc);
                target_outstanding = mpi3mr_atomic_read(&target->outstanding);
                if (target_outstanding) {
                        target_outstanding = mpi3mr_atomic_read(&target->outstanding);
                        mpi3mr_dprint(sc, MPI3MR_ERROR, "[%2d] outstanding IOs present on target: %d "
                            "despite poll\n", target_outstanding, target->per_id);
                }
        }

        if (target->exposed_to_os && !sc->reset_in_progress) {
                mpi3mr_rescan_target(sc, target);
                mpi3mr_dprint(sc, MPI3MR_INFO,
                        "Removed device(persistent_id: %d dev_handle: %d)\n", target->per_id, handle);
                target->exposed_to_os = 0;
        }

        target->flags &= ~MPI3MRSAS_TARGET_INREMOVAL;
out:
        return retval;
}

void mpi3mr_remove_device_from_list(struct mpi3mr_softc *sc,
        struct mpi3mr_target *target, bool must_delete)
{
        if ((must_delete == false) &&
            (target->state != MPI3MR_DEV_REMOVE_HS_COMPLETED))
                return;

        mtx_lock_spin(&sc->target_lock);
        TAILQ_REMOVE(&sc->cam_sc->tgt_list, target, tgt_next);
        mtx_unlock_spin(&sc->target_lock);

        free(target, M_MPI3MR);
        target = NULL;
        
        return;
}

/**
 * mpi3mr_devstatuschg_evt_bh - DevStatusChange evt bottomhalf
 * @sc: Adapter instance reference
 * @fwevt: Firmware event
 *
 * Process Device Status Change event and based on device's new
 * information, either expose the device to the upper layers, or
 * remove the device from upper layers.
 *
 * Return: Nothing.
 */
static void mpi3mr_devstatuschg_evt_bh(struct mpi3mr_softc *sc,
        struct mpi3mr_fw_event_work *fwevt)
{
        U16 dev_handle = 0;
        U8 uhide = 0, delete = 0, cleanup = 0;
        struct mpi3mr_target *tgtdev = NULL;
        Mpi3EventDataDeviceStatusChange_t *evtdata =
            (Mpi3EventDataDeviceStatusChange_t *)fwevt->event_data;
        


        dev_handle = le16toh(evtdata->DevHandle);
        mpi3mr_dprint(sc, MPI3MR_INFO,
            "%s :device status change: handle(0x%04x): reason code(0x%x)\n",
            __func__, dev_handle, evtdata->ReasonCode);
        switch (evtdata->ReasonCode) {
        case MPI3_EVENT_DEV_STAT_RC_HIDDEN:
                delete = 1;
                break;
        case MPI3_EVENT_DEV_STAT_RC_NOT_HIDDEN:
                uhide = 1;
                break;
        case MPI3_EVENT_DEV_STAT_RC_VD_NOT_RESPONDING:
                delete = 1;
                cleanup = 1;
                break;
        default:
                mpi3mr_dprint(sc, MPI3MR_INFO, "%s :Unhandled reason code(0x%x)\n", __func__,
                    evtdata->ReasonCode);
                break;
        }

        tgtdev = mpi3mr_find_target_by_dev_handle(sc->cam_sc, dev_handle);
        if (!tgtdev)
                return;

        if (uhide) {
                if (!tgtdev->exposed_to_os)
                        mpi3mr_add_device(sc, tgtdev->per_id);
        }

        if (delete)
                mpi3mr_remove_device_from_os(sc, dev_handle);

        if (cleanup)
                mpi3mr_remove_device_from_list(sc, tgtdev, false);
}

/**
 * mpi3mr_devinfochg_evt_bh - DeviceInfoChange evt bottomhalf
 * @sc: Adapter instance reference
 * @dev_pg0: New device page0
 *
 * Process Device Info Change event and based on device's new
 * information, either expose the device to the upper layers, or
 * remove the device from upper layers or update the details of
 * the device.
 *
 * Return: Nothing.
 */
static void mpi3mr_devinfochg_evt_bh(struct mpi3mr_softc *sc,
        Mpi3DevicePage0_t *dev_pg0)
{
        struct mpi3mr_target *tgtdev = NULL;
        U16 dev_handle = 0, perst_id = 0;

        perst_id = le16toh(dev_pg0->PersistentID);
        dev_handle = le16toh(dev_pg0->DevHandle);
        mpi3mr_dprint(sc, MPI3MR_INFO,
            "%s :Device info change: handle(0x%04x): persist_id(0x%x)\n",
            __func__, dev_handle, perst_id);
        tgtdev = mpi3mr_find_target_by_dev_handle(sc->cam_sc, dev_handle);
        if (!tgtdev)
                return;
        
        mpi3mr_update_device(sc, tgtdev, dev_pg0, false);
        if (!tgtdev->is_hidden && !tgtdev->exposed_to_os)
                mpi3mr_add_device(sc, perst_id);

        if (tgtdev->is_hidden && tgtdev->exposed_to_os)
                mpi3mr_remove_device_from_os(sc, tgtdev->dev_handle);
}

static void
mpi3mr_fw_work(struct mpi3mr_softc *sc, struct mpi3mr_fw_event_work *fw_event)
{
        if (sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN)
                goto out;

        if (!fw_event->process_event)
                goto evt_ack;

        mpi3mr_dprint(sc, MPI3MR_EVENT, "(%d)->(%s) Working on  Event: [%x]\n",
            event_count++, __func__, fw_event->event);

        switch (fw_event->event) {
        case MPI3_EVENT_DEVICE_ADDED:
        {
                Mpi3DevicePage0_t *dev_pg0 =
                        (Mpi3DevicePage0_t *) fw_event->event_data;
                mpi3mr_add_device(sc, dev_pg0->PersistentID);
                break;
        }
        case MPI3_EVENT_DEVICE_INFO_CHANGED:
        {
                mpi3mr_devinfochg_evt_bh(sc,
                    (Mpi3DevicePage0_t *) fw_event->event_data);
                break;
        }
        case MPI3_EVENT_DEVICE_STATUS_CHANGE:
        {
                mpi3mr_devstatuschg_evt_bh(sc, fw_event);
                break;
        }
        case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
        {
                mpi3mr_process_sastopochg_evt(sc, fw_event);
                break;
        }
        case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
        {
                mpi3mr_process_pcietopochg_evt(sc, fw_event);
                break;
        }
        default:
                mpi3mr_dprint(sc, MPI3MR_TRACE,"Unhandled event 0x%0X\n",
                    fw_event->event);
                break;

        }

evt_ack:
        if (fw_event->send_ack) {
                mpi3mr_dprint(sc, MPI3MR_EVENT,"Process event ACK for event 0x%0X\n",
                    fw_event->event);
                mpi3mr_process_event_ack(sc, fw_event->event,
                    fw_event->event_context);
        }

out:
        mpi3mr_dprint(sc, MPI3MR_EVENT, "(%d)->(%s) Event Free: [%x]\n", event_count,
            __func__, fw_event->event);
        
        mpi3mr_fw_event_free(sc, fw_event);
}

void
mpi3mr_firmware_event_work(void *arg, int pending)
{
        struct mpi3mr_fw_event_work *fw_event;
        struct mpi3mr_softc *sc;

        sc = (struct mpi3mr_softc *)arg;

        mtx_lock(&sc->fwevt_lock);
        while ((fw_event = TAILQ_FIRST(&sc->cam_sc->ev_queue)) != NULL) {
                TAILQ_REMOVE(&sc->cam_sc->ev_queue, fw_event, ev_link);
                mtx_unlock(&sc->fwevt_lock);
                mpi3mr_fw_work(sc, fw_event);
                mtx_lock(&sc->fwevt_lock);
        }
        mtx_unlock(&sc->fwevt_lock);
}


/*
 * mpi3mr_cam_attach - CAM layer registration
 * @sc: Adapter reference
 *
 * This function does simq allocation, cam registration, xpt_bus registration,
 * event taskqueue initialization and async event handler registration.
 *
 * Return: 0 on success and proper error codes on failure
 */
int
mpi3mr_cam_attach(struct mpi3mr_softc *sc)
{
        struct mpi3mr_cam_softc *cam_sc;
        cam_status status;
        int unit, error = 0, reqs;

        mpi3mr_dprint(sc, MPI3MR_XINFO, "Starting CAM Attach\n");

        cam_sc = malloc(sizeof(struct mpi3mr_cam_softc), M_MPI3MR, M_WAITOK|M_ZERO);
        cam_sc->maxtargets = sc->facts.max_perids + 1;

        TAILQ_INIT(&cam_sc->tgt_list);
        
        sc->cam_sc = cam_sc;
        cam_sc->sc = sc;

        reqs = sc->max_host_ios;

        if ((cam_sc->devq = cam_simq_alloc(reqs)) == NULL) {
                mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate SIMQ\n");
                error = ENOMEM;
                goto out;
        }

        unit = device_get_unit(sc->mpi3mr_dev);
        cam_sc->sim = cam_sim_alloc(mpi3mr_cam_action, mpi3mr_cam_poll, "mpi3mr", cam_sc,
            unit, &sc->mpi3mr_mtx, reqs, reqs, cam_sc->devq);
        if (cam_sc->sim == NULL) {
                mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate SIM\n");
                error = EINVAL;
                goto out;
        }

        TAILQ_INIT(&cam_sc->ev_queue);

        /* Initialize taskqueue for Event Handling */
        TASK_INIT(&cam_sc->ev_task, 0, mpi3mr_firmware_event_work, sc);
        cam_sc->ev_tq = taskqueue_create("mpi3mr_taskq", M_NOWAIT | M_ZERO,
            taskqueue_thread_enqueue, &cam_sc->ev_tq);
        taskqueue_start_threads(&cam_sc->ev_tq, 1, PRIBIO, "%s taskq",
            device_get_nameunit(sc->mpi3mr_dev));

        mtx_lock(&sc->mpi3mr_mtx);

        /*
         * XXX There should be a bus for every port on the adapter, but since
         * we're just going to fake the topology for now, we'll pretend that
         * everything is just a target on a single bus.
         */
        if ((error = xpt_bus_register(cam_sc->sim, sc->mpi3mr_dev, 0)) != 0) {
                mpi3mr_dprint(sc, MPI3MR_ERROR,
                    "Error 0x%x registering SCSI bus\n", error);
                mtx_unlock(&sc->mpi3mr_mtx);
                goto out;
        }

        /*
         * Assume that discovery events will start right away.
         *
         * Hold off boot until discovery is complete.
         */
        cam_sc->flags |= MPI3MRSAS_IN_STARTUP | MPI3MRSAS_IN_DISCOVERY;
        sc->cam_sc->startup_refcount = 0;
        mpi3mr_startup_increment(cam_sc);

        callout_init(&cam_sc->discovery_callout, 1 /*mpsafe*/);

        /*
         * Register for async events so we can determine the EEDP
         * capabilities of devices.
         */
        status = xpt_create_path(&cam_sc->path, /*periph*/NULL,
            cam_sim_path(sc->cam_sc->sim), CAM_TARGET_WILDCARD,
            CAM_LUN_WILDCARD);
        if (status != CAM_REQ_CMP) {
                mpi3mr_dprint(sc, MPI3MR_ERROR,
                    "Error 0x%x creating sim path\n", status);
                cam_sc->path = NULL;
        }

        if (status != CAM_REQ_CMP) {
                /*
                 * EEDP use is the exception, not the rule.
                 * Warn the user, but do not fail to attach.
                 */
                mpi3mr_dprint(sc, MPI3MR_INFO, "EEDP capabilities disabled.\n");
        }

        mtx_unlock(&sc->mpi3mr_mtx);

        error = mpi3mr_register_events(sc);

out:
        mpi3mr_dprint(sc, MPI3MR_XINFO, "%s Exiting CAM attach, error: 0x%x n", __func__, error);
        return (error);
}

int
mpi3mr_cam_detach(struct mpi3mr_softc *sc)
{
        struct mpi3mr_cam_softc *cam_sc;
        struct mpi3mr_target *target;

        mpi3mr_dprint(sc, MPI3MR_XINFO, "%s, Starting CAM detach\n", __func__);
        if (sc->cam_sc == NULL)
                return (0);

        cam_sc = sc->cam_sc;
        
        mpi3mr_freeup_events(sc);
        
        /*
         * Drain and free the event handling taskqueue with the lock
         * unheld so that any parallel processing tasks drain properly
         * without deadlocking.
         */
        if (cam_sc->ev_tq != NULL)
                taskqueue_free(cam_sc->ev_tq);

        mtx_lock(&sc->mpi3mr_mtx);

        while (cam_sc->startup_refcount != 0)
                mpi3mr_startup_decrement(cam_sc);

        /* Deregister our async handler */
        if (cam_sc->path != NULL) {
                xpt_free_path(cam_sc->path);
                cam_sc->path = NULL;
        }

        if (cam_sc->flags & MPI3MRSAS_IN_STARTUP)
                xpt_release_simq(cam_sc->sim, 1);

        if (cam_sc->sim != NULL) {
                xpt_bus_deregister(cam_sim_path(cam_sc->sim));
                cam_sim_free(cam_sc->sim, FALSE);
        }

        mtx_unlock(&sc->mpi3mr_mtx);

        if (cam_sc->devq != NULL)
                cam_simq_free(cam_sc->devq);

get_target:
        mtx_lock_spin(&sc->target_lock);
        TAILQ_FOREACH(target, &cam_sc->tgt_list, tgt_next) {
                TAILQ_REMOVE(&sc->cam_sc->tgt_list, target, tgt_next);
                mtx_unlock_spin(&sc->target_lock);
                goto out_tgt_free;
        }
        mtx_unlock_spin(&sc->target_lock);
out_tgt_free:
        if (target) {
                free(target, M_MPI3MR);
                target = NULL;
                goto get_target;
        }

        free(cam_sc, M_MPI3MR);
        sc->cam_sc = NULL;

        mpi3mr_dprint(sc, MPI3MR_XINFO, "%s, Exiting CAM detach\n", __func__);
        return (0);
}