/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * hermon_cq.c
 *    Hermon Completion Queue Processing Routines
 *
 *    Implements all the routines necessary for allocating, freeing, resizing,
 *    and handling the completion type events that the Hermon hardware can
 *    generate.
 */

#include <sys/types.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>
#include <sys/bitmap.h>
#include <sys/sysmacros.h>

#include <sys/ib/adapters/hermon/hermon.h>

int hermon_should_panic = 0;    /* debugging aid */

#define hermon_cq_update_ci_doorbell(cq)                                \
        /* Build the doorbell record data (low 24 bits only) */         \
        HERMON_UAR_DB_RECORD_WRITE(cq->cq_arm_ci_vdbr,                  \
            cq->cq_consindx & 0x00FFFFFF)

static int hermon_cq_arm_doorbell(hermon_state_t *state, hermon_cqhdl_t cq,
    uint_t cmd);
static void hermon_arm_cq_dbr_init(hermon_dbr_t *cq_arm_dbr);
static void hermon_cq_cqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
    hermon_hw_cqe_t *cqe, ibt_wc_t *wc);
static void hermon_cq_errcqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
    hermon_hw_cqe_t *cqe, ibt_wc_t *wc);


/*
 * hermon_cq_alloc()
 *    Context: Can be called only from user or kernel context.
 */
int
hermon_cq_alloc(hermon_state_t *state, ibt_cq_hdl_t ibt_cqhdl,
    ibt_cq_attr_t *cq_attr, uint_t *actual_size, hermon_cqhdl_t *cqhdl,
    uint_t sleepflag)
{
        hermon_rsrc_t           *cqc, *rsrc;
        hermon_umap_db_entry_t  *umapdb;
        hermon_hw_cqc_t         cqc_entry;
        hermon_cqhdl_t          cq;
        ibt_mr_attr_t           mr_attr;
        hermon_mr_options_t     op;
        hermon_pdhdl_t          pd;
        hermon_mrhdl_t          mr;
        hermon_hw_cqe_t         *buf;
        uint64_t                value;
        uint32_t                log_cq_size, uarpg;
        uint_t                  cq_is_umap;
        uint32_t                status, flag;
        hermon_cq_sched_t       *cq_schedp;

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq_attr))

        /*
         * Determine whether CQ is being allocated for userland access or
         * whether it is being allocated for kernel access.  If the CQ is
         * being allocated for userland access, then lookup the UAR
         * page number for the current process.  Note:  If this is not found
         * (e.g. if the process has not previously open()'d the Hermon driver),
         * then an error is returned.
         */
        cq_is_umap = (cq_attr->cq_flags & IBT_CQ_USER_MAP) ? 1 : 0;
        if (cq_is_umap) {
                status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(),
                    MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
                if (status != DDI_SUCCESS) {
                        status = IBT_INVALID_PARAM;
                        goto cqalloc_fail;
                }
                uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx;
        } else {
                uarpg = state->hs_kernel_uar_index;
        }

        /* Use the internal protection domain (PD) for setting up CQs */
        pd = state->hs_pdhdl_internal;

        /* Increment the reference count on the protection domain (PD) */
        hermon_pd_refcnt_inc(pd);

        /*
         * Allocate an CQ context entry.  This will be filled in with all
         * the necessary parameters to define the Completion Queue.  And then
         * ownership will be passed to the hardware in the final step
         * below.  If we fail here, we must undo the protection domain
         * reference count.
         */
        status = hermon_rsrc_alloc(state, HERMON_CQC, 1, sleepflag, &cqc);
        if (status != DDI_SUCCESS) {
                status = IBT_INSUFF_RESOURCE;
                goto cqalloc_fail1;
        }

        /*
         * Allocate the software structure for tracking the completion queue
         * (i.e. the Hermon Completion Queue handle).  If we fail here, we must
         * undo the protection domain reference count and the previous
         * resource allocation.
         */
        status = hermon_rsrc_alloc(state, HERMON_CQHDL, 1, sleepflag, &rsrc);
        if (status != DDI_SUCCESS) {
                status = IBT_INSUFF_RESOURCE;
                goto cqalloc_fail2;
        }
        cq = (hermon_cqhdl_t)rsrc->hr_addr;
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq))
        cq->cq_is_umap = cq_is_umap;
        cq->cq_cqnum = cqc->hr_indx;    /* just use index, implicit in Hermon */
        cq->cq_intmod_count = 0;
        cq->cq_intmod_usec = 0;

        /*
         * If this will be a user-mappable CQ, then allocate an entry for
         * the "userland resources database".  This will later be added to
         * the database (after all further CQ operations are successful).
         * If we fail here, we must undo the reference counts and the
         * previous resource allocation.
         */
        if (cq->cq_is_umap) {
                umapdb = hermon_umap_db_alloc(state->hs_instance, cq->cq_cqnum,
                    MLNX_UMAP_CQMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
                if (umapdb == NULL) {
                        status = IBT_INSUFF_RESOURCE;
                        goto cqalloc_fail3;
                }
        }


        /*
         * Allocate the doorbell record.  We'll need one for the CQ, handling
         * both consumer index (SET CI) and the CQ state (CQ ARM).
         */

        status = hermon_dbr_alloc(state, uarpg, &cq->cq_arm_ci_dbr_acchdl,
            &cq->cq_arm_ci_vdbr, &cq->cq_arm_ci_pdbr, &cq->cq_dbr_mapoffset);
        if (status != DDI_SUCCESS) {
                status = IBT_INSUFF_RESOURCE;
                goto cqalloc_fail4;
        }

        /*
         * Calculate the appropriate size for the completion queue.
         * Note:  All Hermon CQs must be a power-of-2 minus 1 in size.  Also
         * they may not be any smaller than HERMON_CQ_MIN_SIZE.  This step is
         * to round the requested size up to the next highest power-of-2
         */
        cq_attr->cq_size = max(cq_attr->cq_size, HERMON_CQ_MIN_SIZE);
        log_cq_size = highbit(cq_attr->cq_size);

        /*
         * Next we verify that the rounded-up size is valid (i.e. consistent
         * with the device limits and/or software-configured limits)
         */
        if (log_cq_size > state->hs_cfg_profile->cp_log_max_cq_sz) {
                status = IBT_HCA_CQ_EXCEEDED;
                goto cqalloc_fail4a;
        }

        /*
         * Allocate the memory for Completion Queue.
         *
         * Note: Although we use the common queue allocation routine, we
         * always specify HERMON_QUEUE_LOCATION_NORMAL (i.e. CQ located in
         * kernel system memory) for kernel CQs because it would be
         * inefficient to have CQs located in DDR memory.  This is primarily
         * because CQs are read from (by software) more than they are written
         * to. (We always specify HERMON_QUEUE_LOCATION_USERLAND for all
         * user-mappable CQs for a similar reason.)
         * It is also worth noting that, unlike Hermon QP work queues,
         * completion queues do not have the same strict alignment
         * requirements.  It is sufficient for the CQ memory to be both
         * aligned to and bound to addresses which are a multiple of CQE size.
         */
        cq->cq_cqinfo.qa_size = (1 << log_cq_size) * sizeof (hermon_hw_cqe_t);

        cq->cq_cqinfo.qa_alloc_align = PAGESIZE;
        cq->cq_cqinfo.qa_bind_align  = PAGESIZE;
        if (cq->cq_is_umap) {
                cq->cq_cqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
        } else {
                cq->cq_cqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
                hermon_arm_cq_dbr_init(cq->cq_arm_ci_vdbr);
        }
        status = hermon_queue_alloc(state, &cq->cq_cqinfo, sleepflag);
        if (status != DDI_SUCCESS) {
                status = IBT_INSUFF_RESOURCE;
                goto cqalloc_fail4;
        }
        buf = (hermon_hw_cqe_t *)cq->cq_cqinfo.qa_buf_aligned;
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))

        /*
         * The ownership bit of the CQE's is set by the HW during the process
         * of transferrring ownership of the CQ (PRM 09.35c, 14.2.1, note D1
         *
         */

        /*
         * Register the memory for the CQ.  The memory for the CQ must
         * be registered in the Hermon TPT tables.  This gives us the LKey
         * to specify in the CQ context below.  Note: If this is a user-
         * mappable CQ, then we will force DDI_DMA_CONSISTENT mapping.
         */
        flag = (sleepflag == HERMON_SLEEP) ?  IBT_MR_SLEEP : IBT_MR_NOSLEEP;
        mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
        mr_attr.mr_len   = cq->cq_cqinfo.qa_size;
        mr_attr.mr_as    = NULL;
        mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
        op.mro_bind_type   = state->hs_cfg_profile->cp_iommu_bypass;
        op.mro_bind_dmahdl = cq->cq_cqinfo.qa_dmahdl;
        op.mro_bind_override_addr = 0;
        status = hermon_mr_register(state, pd, &mr_attr, &mr, &op,
            HERMON_CQ_CMPT);
        if (status != DDI_SUCCESS) {
                status = IBT_INSUFF_RESOURCE;
                goto cqalloc_fail5;
        }
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))

        cq->cq_erreqnum = HERMON_CQ_ERREQNUM_GET(state);
        if (cq_attr->cq_flags & IBT_CQ_HID) {
                if (!HERMON_HID_VALID(state, cq_attr->cq_hid)) {
                        IBTF_DPRINTF_L2("CQalloc", "bad handler id 0x%x",
                            cq_attr->cq_hid);
                        status = IBT_INVALID_PARAM;
                        goto cqalloc_fail5;
                }
                cq->cq_eqnum = HERMON_HID_TO_EQNUM(state, cq_attr->cq_hid);
                IBTF_DPRINTF_L2("cqalloc", "hid: eqn %d", cq->cq_eqnum);
        } else {
                cq_schedp = (hermon_cq_sched_t *)cq_attr->cq_sched;
                if (cq_schedp == NULL) {
                        cq_schedp = &state->hs_cq_sched_default;
                } else if (cq_schedp != &state->hs_cq_sched_default) {
                        int i;
                        hermon_cq_sched_t *tmp;

                        tmp = state->hs_cq_sched_array;
                        for (i = 0; i < state->hs_cq_sched_array_size; i++)
                                if (cq_schedp == &tmp[i])
                                        break;  /* found it */
                        if (i >= state->hs_cq_sched_array_size) {
                                cmn_err(CE_CONT, "!Invalid cq_sched argument: "
                                    "ignored\n");
                                cq_schedp = &state->hs_cq_sched_default;
                        }
                }
                cq->cq_eqnum = HERMON_HID_TO_EQNUM(state,
                    HERMON_CQSCHED_NEXT_HID(cq_schedp));
                IBTF_DPRINTF_L2("cqalloc", "sched: first-1 %d, len %d, "
                    "eqn %d", cq_schedp->cqs_start_hid - 1,
                    cq_schedp->cqs_len, cq->cq_eqnum);
        }

        /*
         * Fill in the CQC entry.  This is the final step before passing
         * ownership of the CQC entry to the Hermon hardware.  We use all of
         * the information collected/calculated above to fill in the
         * requisite portions of the CQC.  Note: If this CQ is going to be
         * used for userland access, then we need to set the UAR page number
         * appropriately (otherwise it's a "don't care")
         */
        bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));

        cqc_entry.state         = HERMON_CQ_DISARMED;
        cqc_entry.pg_offs       = cq->cq_cqinfo.qa_pgoffs >> 5;
        cqc_entry.log_cq_sz     = log_cq_size;
        cqc_entry.usr_page      = uarpg;
        cqc_entry.c_eqn         = cq->cq_eqnum;
        cqc_entry.log2_pgsz     = mr->mr_log2_pgsz;
        cqc_entry.mtt_base_addh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
        cqc_entry.mtt_base_addl = mr->mr_mttaddr >> 3;
        cqc_entry.dbr_addrh = (uint32_t)((uint64_t)cq->cq_arm_ci_pdbr >> 32);
        cqc_entry.dbr_addrl = (uint32_t)((uint64_t)cq->cq_arm_ci_pdbr >> 3);

        /*
         * Write the CQC entry to hardware - we pass ownership of
         * the entry to the hardware (using the Hermon SW2HW_CQ firmware
         * command).  Note: In general, this operation shouldn't fail.  But
         * if it does, we have to undo everything we've done above before
         * returning error.
         */
        status = hermon_cmn_ownership_cmd_post(state, SW2HW_CQ, &cqc_entry,
            sizeof (hermon_hw_cqc_t), cq->cq_cqnum, sleepflag);
        if (status != HERMON_CMD_SUCCESS) {
                cmn_err(CE_CONT, "Hermon: SW2HW_CQ command failed: %08x\n",
                    status);
                if (status == HERMON_CMD_INVALID_STATUS) {
                        hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
                }
                status = ibc_get_ci_failure(0);
                goto cqalloc_fail6;
        }

        /*
         * Fill in the rest of the Hermon Completion Queue handle.  Having
         * successfully transferred ownership of the CQC, we can update the
         * following fields for use in further operations on the CQ.
         */
        cq->cq_resize_hdl = 0;
        cq->cq_cqcrsrcp   = cqc;
        cq->cq_rsrcp      = rsrc;
        cq->cq_consindx   = 0;
                /* least restrictive */
        cq->cq_buf        = buf;
        cq->cq_bufsz      = (1 << log_cq_size);
        cq->cq_log_cqsz   = log_cq_size;
        cq->cq_mrhdl      = mr;
        cq->cq_refcnt     = 0;
        cq->cq_is_special = 0;
        cq->cq_uarpg      = uarpg;
        cq->cq_umap_dhp   = (devmap_cookie_t)NULL;
        avl_create(&cq->cq_wrid_wqhdr_avl_tree, hermon_wrid_workq_compare,
            sizeof (struct hermon_workq_avl_s),
            offsetof(struct hermon_workq_avl_s, wqa_link));

        cq->cq_hdlrarg    = (void *)ibt_cqhdl;

        /*
         * Put CQ handle in Hermon CQNum-to-CQHdl list.  Then fill in the
         * "actual_size" and "cqhdl" and return success
         */
        hermon_icm_set_num_to_hdl(state, HERMON_CQC, cqc->hr_indx, cq);

        /*
         * If this is a user-mappable CQ, then we need to insert the previously
         * allocated entry into the "userland resources database".  This will
         * allow for later lookup during devmap() (i.e. mmap()) calls.
         */
        if (cq->cq_is_umap) {
                hermon_umap_db_add(umapdb);
        }

        /*
         * Fill in the return arguments (if necessary).  This includes the
         * real completion queue size.
         */
        if (actual_size != NULL) {
                *actual_size = (1 << log_cq_size) - 1;
        }
        *cqhdl = cq;

        return (DDI_SUCCESS);

/*
 * The following is cleanup for all possible failure cases in this routine
 */
cqalloc_fail6:
        if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
            sleepflag) != DDI_SUCCESS) {
                HERMON_WARNING(state, "failed to deregister CQ memory");
        }
cqalloc_fail5:
        hermon_queue_free(&cq->cq_cqinfo);
cqalloc_fail4a:
        hermon_dbr_free(state, uarpg, cq->cq_arm_ci_vdbr);
cqalloc_fail4:
        if (cq_is_umap) {
                hermon_umap_db_free(umapdb);
        }
cqalloc_fail3:
        hermon_rsrc_free(state, &rsrc);
cqalloc_fail2:
        hermon_rsrc_free(state, &cqc);
cqalloc_fail1:
        hermon_pd_refcnt_dec(pd);
cqalloc_fail:
        return (status);
}


/*
 * hermon_cq_free()
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
int
hermon_cq_free(hermon_state_t *state, hermon_cqhdl_t *cqhdl, uint_t sleepflag)
{
        hermon_rsrc_t           *cqc, *rsrc;
        hermon_umap_db_entry_t  *umapdb;
        hermon_hw_cqc_t         cqc_entry;
        hermon_pdhdl_t          pd;
        hermon_mrhdl_t          mr;
        hermon_cqhdl_t          cq, resize;
        uint32_t                cqnum;
        uint64_t                value;
        uint_t                  maxprot;
        int                     status;

        /*
         * Pull all the necessary information from the Hermon Completion Queue
         * handle.  This is necessary here because the resource for the
         * CQ handle is going to be freed up as part of this operation.
         */
        cq      = *cqhdl;
        mutex_enter(&cq->cq_lock);
        cqc     = cq->cq_cqcrsrcp;
        rsrc    = cq->cq_rsrcp;
        pd      = state->hs_pdhdl_internal;
        mr      = cq->cq_mrhdl;
        cqnum   = cq->cq_cqnum;

        resize = cq->cq_resize_hdl;             /* save the handle for later */

        /*
         * If there are work queues still associated with the CQ, then return
         * an error.  Otherwise, we will be holding the CQ lock.
         */
        if (cq->cq_refcnt != 0) {
                mutex_exit(&cq->cq_lock);
                return (IBT_CQ_BUSY);
        }

        /*
         * If this was a user-mappable CQ, then we need to remove its entry
         * from the "userland resources database".  If it is also currently
         * mmap()'d out to a user process, then we need to call
         * devmap_devmem_remap() to remap the CQ memory to an invalid mapping.
         * We also need to invalidate the CQ tracking information for the
         * user mapping.
         */
        if (cq->cq_is_umap) {
                status = hermon_umap_db_find(state->hs_instance, cqnum,
                    MLNX_UMAP_CQMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE,
                    &umapdb);
                if (status != DDI_SUCCESS) {
                        mutex_exit(&cq->cq_lock);
                        HERMON_WARNING(state, "failed to find in database");
                        return (ibc_get_ci_failure(0));
                }
                hermon_umap_db_free(umapdb);
                if (cq->cq_umap_dhp != NULL) {
                        maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
                        status = devmap_devmem_remap(cq->cq_umap_dhp,
                            state->hs_dip, 0, 0, cq->cq_cqinfo.qa_size,
                            maxprot, DEVMAP_MAPPING_INVALID, NULL);
                        if (status != DDI_SUCCESS) {
                                mutex_exit(&cq->cq_lock);
                                HERMON_WARNING(state, "failed in CQ memory "
                                    "devmap_devmem_remap()");
                                return (ibc_get_ci_failure(0));
                        }
                        cq->cq_umap_dhp = (devmap_cookie_t)NULL;
                }
        }

        /*
         * Put NULL into the Arbel CQNum-to-CQHdl list.  This will allow any
         * in-progress events to detect that the CQ corresponding to this
         * number has been freed.
         */
        hermon_icm_set_num_to_hdl(state, HERMON_CQC, cqc->hr_indx, NULL);

        mutex_exit(&cq->cq_lock);
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq))

        /*
         * Reclaim CQC entry from hardware (using the Hermon HW2SW_CQ
         * firmware command).  If the ownership transfer fails for any reason,
         * then it is an indication that something (either in HW or SW) has
         * gone seriously wrong.
         */
        status = hermon_cmn_ownership_cmd_post(state, HW2SW_CQ, &cqc_entry,
            sizeof (hermon_hw_cqc_t), cqnum, sleepflag);
        if (status != HERMON_CMD_SUCCESS) {
                HERMON_WARNING(state, "failed to reclaim CQC ownership");
                cmn_err(CE_CONT, "Hermon: HW2SW_CQ command failed: %08x\n",
                    status);
                if (status == HERMON_CMD_INVALID_STATUS) {
                        hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
                }
                return (ibc_get_ci_failure(0));
        }

        /*
         * From here on, we start reliquishing resources - but check to see
         * if a resize was in progress - if so, we need to relinquish those
         * resources as well
         */


        /*
         * Deregister the memory for the Completion Queue.  If this fails
         * for any reason, then it is an indication that something (either
         * in HW or SW) has gone seriously wrong.  So we print a warning
         * message and return.
         */
        status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
            sleepflag);
        if (status != DDI_SUCCESS) {
                HERMON_WARNING(state, "failed to deregister CQ memory");
                return (ibc_get_ci_failure(0));
        }

        if (resize)     {       /* there was a pointer to a handle */
                mr = resize->cq_mrhdl;  /* reuse the pointer to the region */
                status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
                    sleepflag);
                if (status != DDI_SUCCESS) {
                        HERMON_WARNING(state, "failed to deregister resize CQ "
                            "memory");
                        return (ibc_get_ci_failure(0));
                }
        }

        /* Free the memory for the CQ */
        hermon_queue_free(&cq->cq_cqinfo);
        if (resize)     {
                hermon_queue_free(&resize->cq_cqinfo);
                /* and the temporary handle */
                kmem_free(resize, sizeof (struct hermon_sw_cq_s));
        }

        /* everything else does not matter for the resize in progress */

        /* Free the dbr */
        hermon_dbr_free(state, cq->cq_uarpg, cq->cq_arm_ci_vdbr);

        /* Free the Hermon Completion Queue handle */
        hermon_rsrc_free(state, &rsrc);

        /* Free up the CQC entry resource */
        hermon_rsrc_free(state, &cqc);

        /* Decrement the reference count on the protection domain (PD) */
        hermon_pd_refcnt_dec(pd);

        /* Set the cqhdl pointer to NULL and return success */
        *cqhdl = NULL;

        return (DDI_SUCCESS);
}


/*
 * hermon_cq_resize()
 *    Context: Can be called only from user or kernel context.
 */
int
hermon_cq_resize(hermon_state_t *state, hermon_cqhdl_t cq, uint_t req_size,
    uint_t *actual_size, uint_t sleepflag)
{
        hermon_hw_cqc_t         cqc_entry;
        hermon_cqhdl_t          resize_hdl;
        hermon_qalloc_info_t    new_cqinfo;
        ibt_mr_attr_t           mr_attr;
        hermon_mr_options_t     op;
        hermon_pdhdl_t          pd;
        hermon_mrhdl_t          mr;
        hermon_hw_cqe_t         *buf;
        uint32_t                new_prod_indx;
        uint_t                  log_cq_size;
        int                     status, flag;

        if (cq->cq_resize_hdl != 0) {   /* already in process */
                status = IBT_CQ_BUSY;
                goto cqresize_fail;
        }


        /* Use the internal protection domain (PD) for CQs */
        pd = state->hs_pdhdl_internal;

        /*
         * Calculate the appropriate size for the new resized completion queue.
         * Note:  All Hermon CQs must be a power-of-2 minus 1 in size.  Also
         * they may not be any smaller than HERMON_CQ_MIN_SIZE.  This step is
         * to round the requested size up to the next highest power-of-2
         */
        req_size = max(req_size, HERMON_CQ_MIN_SIZE);
        log_cq_size = highbit(req_size);

        /*
         * Next we verify that the rounded-up size is valid (i.e. consistent
         * with the device limits and/or software-configured limits)
         */
        if (log_cq_size > state->hs_cfg_profile->cp_log_max_cq_sz) {
                status = IBT_HCA_CQ_EXCEEDED;
                goto cqresize_fail;
        }

        /*
         * Allocate the memory for newly resized Completion Queue.
         *
         * Note: Although we use the common queue allocation routine, we
         * always specify HERMON_QUEUE_LOCATION_NORMAL (i.e. CQ located in
         * kernel system memory) for kernel CQs because it would be
         * inefficient to have CQs located in DDR memory.  This is the same
         * as we do when we first allocate completion queues primarily
         * because CQs are read from (by software) more than they are written
         * to. (We always specify HERMON_QUEUE_LOCATION_USERLAND for all
         * user-mappable CQs for a similar reason.)
         * It is also worth noting that, unlike Hermon QP work queues,
         * completion queues do not have the same strict alignment
         * requirements.  It is sufficient for the CQ memory to be both
         * aligned to and bound to addresses which are a multiple of CQE size.
         */

        /* first, alloc the resize_handle */
        resize_hdl = kmem_zalloc(sizeof (struct hermon_sw_cq_s), KM_SLEEP);

        new_cqinfo.qa_size = (1 << log_cq_size) * sizeof (hermon_hw_cqe_t);
        new_cqinfo.qa_alloc_align = PAGESIZE;
        new_cqinfo.qa_bind_align  = PAGESIZE;
        if (cq->cq_is_umap) {
                new_cqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
        } else {
                new_cqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
        }
        status = hermon_queue_alloc(state, &new_cqinfo, sleepflag);
        if (status != DDI_SUCCESS) {
                /* free the resize handle */
                kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
                status = IBT_INSUFF_RESOURCE;
                goto cqresize_fail;
        }
        buf = (hermon_hw_cqe_t *)new_cqinfo.qa_buf_aligned;
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))

        /*
         * No initialization of the cq is needed - the command will do it
         */

        /*
         * Register the memory for the CQ.  The memory for the CQ must
         * be registered in the Hermon TPT tables.  This gives us the LKey
         * to specify in the CQ context below.
         */
        flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
        mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
        mr_attr.mr_len   = new_cqinfo.qa_size;
        mr_attr.mr_as    = NULL;
        mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
        op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
        op.mro_bind_dmahdl = new_cqinfo.qa_dmahdl;
        op.mro_bind_override_addr = 0;
        status = hermon_mr_register(state, pd, &mr_attr, &mr, &op,
            HERMON_CQ_CMPT);
        if (status != DDI_SUCCESS) {
                hermon_queue_free(&new_cqinfo);
                /* free the resize handle */
                kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
                status = IBT_INSUFF_RESOURCE;
                goto cqresize_fail;
        }
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))

        /*
         * Now we grab the CQ lock.  Since we will be updating the actual
         * CQ location and the producer/consumer indexes, we should hold
         * the lock.
         *
         * We do a ARBEL_NOSLEEP here (and below), though, because we are
         * holding the "cq_lock" and if we got raised to interrupt level
         * by priority inversion, we would not want to block in this routine
         * waiting for success.
         */
        mutex_enter(&cq->cq_lock);

        /*
         * Fill in the CQC entry.  For the resize operation this is the
         * final step before attempting the resize operation on the CQC entry.
         * We use all of the information collected/calculated above to fill
         * in the requisite portions of the CQC.
         */
        bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
        cqc_entry.log_cq_sz     = log_cq_size;
        cqc_entry.pg_offs       = new_cqinfo.qa_pgoffs >> 5;
        cqc_entry.log2_pgsz     = mr->mr_log2_pgsz;
        cqc_entry.mtt_base_addh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
        cqc_entry.mtt_base_addl = mr->mr_mttaddr >> 3;

        /*
         * Write the CQC entry to hardware.  Lastly, we pass ownership of
         * the entry to the hardware (using the Hermon RESIZE_CQ firmware
         * command).  Note: In general, this operation shouldn't fail.  But
         * if it does, we have to undo everything we've done above before
         * returning error.  Also note that the status returned may indicate
         * the code to return to the IBTF.
         */
        status = hermon_resize_cq_cmd_post(state, &cqc_entry, cq->cq_cqnum,
            &new_prod_indx, HERMON_CMD_NOSLEEP_SPIN);
        if (status != HERMON_CMD_SUCCESS) {
                /* Resize attempt has failed, drop CQ lock and cleanup */
                mutex_exit(&cq->cq_lock);
                if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
                    sleepflag) != DDI_SUCCESS) {
                        HERMON_WARNING(state, "failed to deregister CQ memory");
                }
                kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
                hermon_queue_free(&new_cqinfo);
                if (status == HERMON_CMD_BAD_SIZE) {
                        return (IBT_CQ_SZ_INSUFFICIENT);
                } else {
                        cmn_err(CE_CONT, "Hermon: RESIZE_CQ command failed: "
                            "%08x\n", status);
                        if (status == HERMON_CMD_INVALID_STATUS) {
                                hermon_fm_ereport(state, HCA_SYS_ERR,
                                    HCA_ERR_SRV_LOST);
                        }
                        return (ibc_get_ci_failure(0));
                }
        }

        /*
         * For Hermon, we've alloc'd another handle structure and save off the
         * important things in it. Then, in polling we check to see if there's
         * a "resizing handle" and if so we look for the "special CQE", opcode
         * 0x16, that indicates the transition to the new buffer.
         *
         * At that point, we'll adjust everything - including dereg and
         * freeing of the original buffer, updating all the necessary fields
         * in the cq_hdl, and setting up for the next cqe polling
         */

        resize_hdl->cq_buf      = buf;
        resize_hdl->cq_bufsz    = (1 << log_cq_size);
        resize_hdl->cq_mrhdl    = mr;
        resize_hdl->cq_log_cqsz = log_cq_size;

        bcopy(&new_cqinfo, &(resize_hdl->cq_cqinfo),
            sizeof (struct hermon_qalloc_info_s));

        /* now, save the address in the cq_handle */
        cq->cq_resize_hdl = resize_hdl;

        /*
         * Drop the CQ lock now.
         */

        mutex_exit(&cq->cq_lock);
        /*
         * Fill in the return arguments (if necessary).  This includes the
         * real new completion queue size.
         */
        if (actual_size != NULL) {
                *actual_size = (1 << log_cq_size) - 1;
        }

        return (DDI_SUCCESS);

cqresize_fail:
        return (status);
}


/*
 * hermon_cq_modify()
 *    Context: Can be called base context.
 */
/* ARGSUSED */
int
hermon_cq_modify(hermon_state_t *state, hermon_cqhdl_t cq,
    uint_t count, uint_t usec, ibt_cq_handler_id_t hid, uint_t sleepflag)
{
        int     status;
        hermon_hw_cqc_t         cqc_entry;

        mutex_enter(&cq->cq_lock);
        if (count != cq->cq_intmod_count ||
            usec != cq->cq_intmod_usec) {
                bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
                cqc_entry.cq_max_cnt = count;
                cqc_entry.cq_period = usec;
                status = hermon_modify_cq_cmd_post(state, &cqc_entry,
                    cq->cq_cqnum, MODIFY_MODERATION_CQ, sleepflag);
                if (status != HERMON_CMD_SUCCESS) {
                        mutex_exit(&cq->cq_lock);
                        cmn_err(CE_CONT, "Hermon: MODIFY_MODERATION_CQ "
                            "command failed: %08x\n", status);
                        if (status == HERMON_CMD_INVALID_STATUS) {
                                hermon_fm_ereport(state, HCA_SYS_ERR,
                                    HCA_ERR_SRV_LOST);
                        }
                        return (ibc_get_ci_failure(0));
                }
                cq->cq_intmod_count = count;
                cq->cq_intmod_usec = usec;
        }
        if (hid && (hid - 1 != cq->cq_eqnum)) {
                bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
                cqc_entry.c_eqn = HERMON_HID_TO_EQNUM(state, hid);
                status = hermon_modify_cq_cmd_post(state, &cqc_entry,
                    cq->cq_cqnum, MODIFY_EQN, sleepflag);
                if (status != HERMON_CMD_SUCCESS) {
                        mutex_exit(&cq->cq_lock);
                        cmn_err(CE_CONT, "Hermon: MODIFY_EQN command failed: "
                            "%08x\n", status);
                        if (status == HERMON_CMD_INVALID_STATUS) {
                                hermon_fm_ereport(state, HCA_SYS_ERR,
                                    HCA_ERR_SRV_LOST);
                        }
                        return (ibc_get_ci_failure(0));
                }
                cq->cq_eqnum = hid - 1;
        }
        mutex_exit(&cq->cq_lock);
        return (DDI_SUCCESS);
}

/*
 * hermon_cq_notify()
 *    Context: Can be called from interrupt or base context.
 */
int
hermon_cq_notify(hermon_state_t *state, hermon_cqhdl_t cq,
    ibt_cq_notify_flags_t flags)
{
        uint_t  cmd;
        ibt_status_t status;

        /* Validate IBT flags and call doorbell routine. */
        if (flags == IBT_NEXT_COMPLETION) {
                cmd = HERMON_CQDB_NOTIFY_CQ;
        } else if (flags == IBT_NEXT_SOLICITED) {
                cmd = HERMON_CQDB_NOTIFY_CQ_SOLICIT;
        } else {
                return (IBT_CQ_NOTIFY_TYPE_INVALID);
        }

        status = hermon_cq_arm_doorbell(state, cq, cmd);
        return (status);
}


/*
 * hermon_cq_poll()
 *    Context: Can be called from interrupt or base context.
 */
int
hermon_cq_poll(hermon_state_t *state, hermon_cqhdl_t cq, ibt_wc_t *wc_p,
    uint_t num_wc, uint_t *num_polled)
{
        hermon_hw_cqe_t *cqe;
        uint_t          opcode;
        uint32_t        cons_indx, wrap_around_mask, shift, mask;
        uint32_t        polled_cnt, spec_op = 0;
        int             status;

        /*
         * Check for user-mappable CQ memory.  Note:  We do not allow kernel
         * clients to poll CQ memory that is accessible directly by the user.
         * If the CQ memory is user accessible, then return an error.
         */
        if (cq->cq_is_umap) {
                return (IBT_CQ_HDL_INVALID);
        }

        mutex_enter(&cq->cq_lock);

        /* Get the consumer index */
        cons_indx = cq->cq_consindx;
        shift = cq->cq_log_cqsz;
        mask = cq->cq_bufsz;

        /*
         * Calculate the wrap around mask.  Note: This operation only works
         * because all Hermon completion queues have power-of-2 sizes
         */
        wrap_around_mask = (cq->cq_bufsz - 1);

        /* Calculate the pointer to the first CQ entry */
        cqe = &cq->cq_buf[cons_indx & wrap_around_mask];

        /*
         * Keep pulling entries from the CQ until we find an entry owned by
         * the hardware.  As long as there the CQE's owned by SW, process
         * each entry by calling hermon_cq_cqe_consume() and updating the CQ
         * consumer index.  Note:  We only update the consumer index if
         * hermon_cq_cqe_consume() returns HERMON_CQ_SYNC_AND_DB.  Otherwise,
         * it indicates that we are going to "recycle" the CQE (probably
         * because it is a error CQE and corresponds to more than one
         * completion).
         */
        polled_cnt = 0;
        while (HERMON_CQE_OWNER_IS_SW(cq, cqe, cons_indx, shift, mask)) {
                if (cq->cq_resize_hdl != 0) {   /* in midst of resize */
                        /* peek at the opcode */
                        opcode = HERMON_CQE_OPCODE_GET(cq, cqe);
                        if (opcode == HERMON_CQE_RCV_RESIZE_CODE) {
                                hermon_cq_resize_helper(state, cq);

                                /* Increment the consumer index */
                                cons_indx = (cons_indx + 1);
                                spec_op = 1; /* plus one for the limiting CQE */

                                wrap_around_mask = (cq->cq_bufsz - 1);

                                /* Update the pointer to the next CQ entry */
                                cqe = &cq->cq_buf[cons_indx & wrap_around_mask];

                                continue;
                        }
                }       /* in resizing CQ */

                /*
                 * either resizing and not the special opcode, or
                 * not resizing at all
                 */
                hermon_cq_cqe_consume(state, cq, cqe, &wc_p[polled_cnt++]);

                /* Increment the consumer index */
                cons_indx = (cons_indx + 1);

                /* Update the pointer to the next CQ entry */
                cqe = &cq->cq_buf[cons_indx & wrap_around_mask];

                /*
                 * If we have run out of space to store work completions,
                 * then stop and return the ones we have pulled of the CQ.
                 */
                if (polled_cnt >= num_wc) {
                        break;
                }
        }

        /*
         * Now we only ring the doorbell (to update the consumer index) if
         * we've actually consumed a CQ entry.
         */
        if ((polled_cnt != 0) && (cq->cq_consindx != cons_indx)) {
                /*
                 * Update the consumer index in both the CQ handle and the
                 * doorbell record.
                 */
                cq->cq_consindx = cons_indx;
                hermon_cq_update_ci_doorbell(cq);

        } else if (polled_cnt == 0) {
                if (spec_op != 0) {
                        /* if we got the special opcode, update the consindx */
                        cq->cq_consindx = cons_indx;
                        hermon_cq_update_ci_doorbell(cq);
                }
        }

        mutex_exit(&cq->cq_lock);

        /* Set "num_polled" (if necessary) */
        if (num_polled != NULL) {
                *num_polled = polled_cnt;
        }

        /* Set CQ_EMPTY condition if needed, otherwise return success */
        if (polled_cnt == 0) {
                status = IBT_CQ_EMPTY;
        } else {
                status = DDI_SUCCESS;
        }

        /*
         * Check if the system is currently panicking.  If it is, then call
         * the Hermon interrupt service routine.  This step is necessary here
         * because we might be in a polled I/O mode and without the call to
         * hermon_isr() - and its subsequent calls to poll and rearm each
         * event queue - we might overflow our EQs and render the system
         * unable to sync/dump.
         */
        if (ddi_in_panic() != 0) {
                (void) hermon_isr((caddr_t)state, (caddr_t)NULL);
        }
        return (status);
}

/*
 *      cmd_sn must be initialized to 1 to enable proper reenabling
 *      by hermon_arm_cq_dbr_update().
 */
static void
hermon_arm_cq_dbr_init(hermon_dbr_t *cq_arm_dbr)
{
        uint32_t *target;

        target = (uint32_t *)cq_arm_dbr + 1;
        *target = htonl(1 << HERMON_CQDB_CMDSN_SHIFT);
}


/*
 *      User cmd_sn needs help from this kernel function to know
 *      when it should be incremented (modulo 4).  We do an atomic
 *      update of the arm_cq dbr to communicate this fact.  We retry
 *      in the case that user library is racing with us.  We zero
 *      out the cmd field so that the user library can use the cmd
 *      field to track the last command it issued (solicited verses any).
 */
static void
hermon_arm_cq_dbr_update(hermon_dbr_t *cq_arm_dbr)
{
        uint32_t tmp, cmp, new;
        uint32_t old_cmd_sn, new_cmd_sn;
        uint32_t *target;
        int retries = 0;

        target = (uint32_t *)cq_arm_dbr + 1;
retry:
        cmp = *target;
        tmp = htonl(cmp);
        old_cmd_sn = tmp & (0x3 << HERMON_CQDB_CMDSN_SHIFT);
        new_cmd_sn = (old_cmd_sn + (0x1 << HERMON_CQDB_CMDSN_SHIFT)) &
            (0x3 << HERMON_CQDB_CMDSN_SHIFT);
        new = htonl((tmp & ~(0x37 << HERMON_CQDB_CMD_SHIFT)) | new_cmd_sn);
        tmp = atomic_cas_32(target, cmp, new);
        if (tmp != cmp) {       /* cas failed, so need to retry */
                drv_usecwait(retries & 0xff);   /* avoid race */
                if (++retries > 100000) {
                        cmn_err(CE_CONT, "cas failed in hermon\n");
                        retries = 0;
                }
                goto retry;
        }
}


/*
 * hermon_cq_handler()
 *    Context: Only called from interrupt context
 */
/* ARGSUSED */
int
hermon_cq_handler(hermon_state_t *state, hermon_eqhdl_t eq,
    hermon_hw_eqe_t *eqe)
{
        hermon_cqhdl_t          cq;
        uint_t                  cqnum;

        /* Get the CQ handle from CQ number in event descriptor */
        cqnum = HERMON_EQE_CQNUM_GET(eq, eqe);
        cq = hermon_cqhdl_from_cqnum(state, cqnum);

        /*
         * If the CQ handle is NULL, this is probably an indication
         * that the CQ has been freed already.  In which case, we
         * should not deliver this event.
         *
         * We also check that the CQ number in the handle is the
         * same as the CQ number in the event queue entry.  This
         * extra check allows us to handle the case where a CQ was
         * freed and then allocated again in the time it took to
         * handle the event queue processing.  By constantly incrementing
         * the non-constrained portion of the CQ number every time
         * a new CQ is allocated, we mitigate (somewhat) the chance
         * that a stale event could be passed to the client's CQ
         * handler.
         *
         * Lastly, we check if "hs_ibtfpriv" is NULL.  If it is then it
         * means that we've have either received this event before we
         * finished attaching to the IBTF or we've received it while we
         * are in the process of detaching.
         */
        if ((cq != NULL) && (cq->cq_cqnum == cqnum) &&
            (state->hs_ibtfpriv != NULL)) {
                hermon_arm_cq_dbr_update(cq->cq_arm_ci_vdbr);
                HERMON_DO_IBTF_CQ_CALLB(state, cq);
        }

        return (DDI_SUCCESS);
}


/*
 * hermon_cq_err_handler()
 *    Context: Only called from interrupt context
 */
/* ARGSUSED */
int
hermon_cq_err_handler(hermon_state_t *state, hermon_eqhdl_t eq,
    hermon_hw_eqe_t *eqe)
{
        hermon_cqhdl_t          cq;
        uint_t                  cqnum;
        ibc_async_event_t       event;
        ibt_async_code_t        type;

        HERMON_FMANOTE(state, HERMON_FMA_OVERRUN);
        /* Get the CQ handle from CQ number in event descriptor */
        cqnum = HERMON_EQE_CQNUM_GET(eq, eqe);
        cq = hermon_cqhdl_from_cqnum(state, cqnum);

        /*
         * If the CQ handle is NULL, this is probably an indication
         * that the CQ has been freed already.  In which case, we
         * should not deliver this event.
         *
         * We also check that the CQ number in the handle is the
         * same as the CQ number in the event queue entry.  This
         * extra check allows us to handle the case where a CQ was
         * freed and then allocated again in the time it took to
         * handle the event queue processing.  By constantly incrementing
         * the non-constrained portion of the CQ number every time
         * a new CQ is allocated, we mitigate (somewhat) the chance
         * that a stale event could be passed to the client's CQ
         * handler.
         *
         * And then we check if "hs_ibtfpriv" is NULL.  If it is then it
         * means that we've have either received this event before we
         * finished attaching to the IBTF or we've received it while we
         * are in the process of detaching.
         */
        if ((cq != NULL) && (cq->cq_cqnum == cqnum) &&
            (state->hs_ibtfpriv != NULL)) {
                event.ev_cq_hdl = (ibt_cq_hdl_t)cq->cq_hdlrarg;
                type            = IBT_ERROR_CQ;
                HERMON_DO_IBTF_ASYNC_CALLB(state, type, &event);
        }

        return (DDI_SUCCESS);
}


/*
 * hermon_cq_refcnt_inc()
 *    Context: Can be called from interrupt or base context.
 */
int
hermon_cq_refcnt_inc(hermon_cqhdl_t cq, uint_t is_special)
{
        /*
         * Increment the completion queue's reference count.  Note: In order
         * to ensure compliance with IBA C11-15, we must ensure that a given
         * CQ is not used for both special (SMI/GSI) QP and non-special QP.
         * This is accomplished here by keeping track of how the referenced
         * CQ is being used.
         */
        mutex_enter(&cq->cq_lock);
        if (cq->cq_refcnt == 0) {
                cq->cq_is_special = is_special;
        } else {
                if (cq->cq_is_special != is_special) {
                        mutex_exit(&cq->cq_lock);
                        return (DDI_FAILURE);
                }
        }
        cq->cq_refcnt++;
        mutex_exit(&cq->cq_lock);
        return (DDI_SUCCESS);
}


/*
 * hermon_cq_refcnt_dec()
 *    Context: Can be called from interrupt or base context.
 */
void
hermon_cq_refcnt_dec(hermon_cqhdl_t cq)
{
        /* Decrement the completion queue's reference count */
        mutex_enter(&cq->cq_lock);
        cq->cq_refcnt--;
        mutex_exit(&cq->cq_lock);
}


/*
 * hermon_cq_arm_doorbell()
 *    Context: Can be called from interrupt or base context.
 */
static int
hermon_cq_arm_doorbell(hermon_state_t *state, hermon_cqhdl_t cq, uint_t cq_cmd)
{
        uint32_t        cq_num;
        uint32_t        *target;
        uint32_t        old_cmd, cmp, new, tmp, cmd_sn;
        ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state);

        /* initialize the FMA retry loop */
        hermon_pio_init(fm_loop_cnt, fm_status, fm_test_num);

        cq_num = cq->cq_cqnum;
        target = (uint32_t *)cq->cq_arm_ci_vdbr + 1;

        /* the FMA retry loop starts for Hermon doorbell register. */
        hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt, fm_status,
            fm_test_num);
retry:
        cmp = *target;
        tmp = htonl(cmp);
        old_cmd = tmp & (0x7 << HERMON_CQDB_CMD_SHIFT);
        cmd_sn = tmp & (0x3 << HERMON_CQDB_CMDSN_SHIFT);
        if (cq_cmd == HERMON_CQDB_NOTIFY_CQ) {
                if (old_cmd != HERMON_CQDB_NOTIFY_CQ) {
                        cmd_sn |= (HERMON_CQDB_NOTIFY_CQ <<
                            HERMON_CQDB_CMD_SHIFT);
                        new = htonl(cmd_sn | (cq->cq_consindx & 0xFFFFFF));
                        tmp = atomic_cas_32(target, cmp, new);
                        if (tmp != cmp)
                                goto retry;
                        HERMON_UAR_DOORBELL(state, uarhdl, (uint64_t *)(void *)
                            &state->hs_uar->cq, (((uint64_t)cmd_sn | cq_num) <<
                            32) | (cq->cq_consindx & 0xFFFFFF));
                } /* else it's already armed */
        } else {
                ASSERT(cq_cmd == HERMON_CQDB_NOTIFY_CQ_SOLICIT);
                if (old_cmd != HERMON_CQDB_NOTIFY_CQ &&
                    old_cmd != HERMON_CQDB_NOTIFY_CQ_SOLICIT) {
                        cmd_sn |= (HERMON_CQDB_NOTIFY_CQ_SOLICIT <<
                            HERMON_CQDB_CMD_SHIFT);
                        new = htonl(cmd_sn | (cq->cq_consindx & 0xFFFFFF));
                        tmp = atomic_cas_32(target, cmp, new);
                        if (tmp != cmp)
                                goto retry;
                        HERMON_UAR_DOORBELL(state, uarhdl, (uint64_t *)(void *)
                            &state->hs_uar->cq, (((uint64_t)cmd_sn | cq_num) <<
                            32) | (cq->cq_consindx & 0xFFFFFF));
                } /* else it's already armed */
        }

        /* the FMA retry loop ends. */
        hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt, fm_status,
            fm_test_num);

        return (IBT_SUCCESS);

pio_error:
        hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
        return (ibc_get_ci_failure(0));
}


/*
 * hermon_cqhdl_from_cqnum()
 *    Context: Can be called from interrupt or base context.
 *
 *    This routine is important because changing the unconstrained
 *    portion of the CQ number is critical to the detection of a
 *    potential race condition in the CQ handler code (i.e. the case
 *    where a CQ is freed and alloc'd again before an event for the
 *    "old" CQ can be handled).
 *
 *    While this is not a perfect solution (not sure that one exists)
 *    it does help to mitigate the chance that this race condition will
 *    cause us to deliver a "stale" event to the new CQ owner.  Note:
 *    this solution does not scale well because the number of constrained
 *    bits increases (and, hence, the number of unconstrained bits
 *    decreases) as the number of supported CQs grows.  For small and
 *    intermediate values, it should hopefully provide sufficient
 *    protection.
 */
hermon_cqhdl_t
hermon_cqhdl_from_cqnum(hermon_state_t *state, uint_t cqnum)
{
        uint_t  cqindx, cqmask;

        /* Calculate the CQ table index from the cqnum */
        cqmask = (1 << state->hs_cfg_profile->cp_log_num_cq) - 1;
        cqindx = cqnum & cqmask;
        return (hermon_icm_num_to_hdl(state, HERMON_CQC, cqindx));
}

/*
 * hermon_cq_cqe_consume()
 *    Context: Can be called from interrupt or base context.
 */
static void
hermon_cq_cqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
    hermon_hw_cqe_t *cqe, ibt_wc_t *wc)
{
        uint_t          opcode, qpnum, qp1_indx;
        ibt_wc_flags_t  flags;
        ibt_wrc_opcode_t type;

        /*
         * Determine if this is an "error" CQE by examining "opcode".  If it
         * is an error CQE, then call hermon_cq_errcqe_consume() and return
         * whatever status it returns.  Otherwise, this is a successful
         * completion.
         */
        opcode = HERMON_CQE_OPCODE_GET(cq, cqe);
        if ((opcode == HERMON_CQE_SEND_ERR_OPCODE) ||
            (opcode == HERMON_CQE_RECV_ERR_OPCODE)) {
                hermon_cq_errcqe_consume(state, cq, cqe, wc);
                return;
        }

        /*
         * Fetch the Work Request ID using the information in the CQE.
         * See hermon_wr.c for more details.
         */
        wc->wc_id = hermon_wrid_get_entry(cq, cqe);

        /*
         * Parse the CQE opcode to determine completion type.  This will set
         * not only the type of the completion, but also any flags that might
         * be associated with it (e.g. whether immediate data is present).
         */
        flags = IBT_WC_NO_FLAGS;
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->hs_fcoib_may_be_running))
        if (HERMON_CQE_SENDRECV_GET(cq, cqe) != HERMON_COMPLETION_RECV) {

                /* Send CQE */
                switch (opcode) {
                case HERMON_CQE_SND_RDMAWR_IMM:
                case HERMON_CQE_SND_RDMAWR:
                        type = IBT_WRC_RDMAW;
                        break;

                case HERMON_CQE_SND_SEND_INV:
                case HERMON_CQE_SND_SEND_IMM:
                case HERMON_CQE_SND_SEND:
                        type = IBT_WRC_SEND;
                        break;

                case HERMON_CQE_SND_LSO:
                        type = IBT_WRC_SEND_LSO;
                        break;

                case HERMON_CQE_SND_RDMARD:
                        type = IBT_WRC_RDMAR;
                        break;

                case HERMON_CQE_SND_ATOMIC_CS:
                        type = IBT_WRC_CSWAP;
                        break;

                case HERMON_CQE_SND_ATOMIC_FA:
                        type = IBT_WRC_FADD;
                        break;

                case HERMON_CQE_SND_BIND_MW:
                        type = IBT_WRC_BIND;
                        break;

                case HERMON_CQE_SND_FRWR:
                        type = IBT_WRC_FAST_REG_PMR;
                        break;

                case HERMON_CQE_SND_LCL_INV:
                        type = IBT_WRC_LOCAL_INVALIDATE;
                        break;

                default:
                        HERMON_WARNING(state, "unknown send CQE type");
                        wc->wc_status = IBT_WC_LOCAL_QP_OP_ERR;
                        return;
                }
        } else if ((state->hs_fcoib_may_be_running == B_TRUE) &&
            hermon_fcoib_is_fexch_qpn(state, HERMON_CQE_QPNUM_GET(cq, cqe))) {
                type = IBT_WRC_RECV;
                if (HERMON_CQE_FEXCH_DIFE(cq, cqe))
                        flags |= IBT_WC_DIF_ERROR;
                wc->wc_bytes_xfer = HERMON_CQE_BYTECNT_GET(cq, cqe);
                wc->wc_fexch_seq_cnt = HERMON_CQE_FEXCH_SEQ_CNT(cq, cqe);
                wc->wc_fexch_tx_bytes_xfer = HERMON_CQE_FEXCH_TX_BYTES(cq, cqe);
                wc->wc_fexch_rx_bytes_xfer = HERMON_CQE_FEXCH_RX_BYTES(cq, cqe);
                wc->wc_fexch_seq_id = HERMON_CQE_FEXCH_SEQ_ID(cq, cqe);
                wc->wc_detail = HERMON_CQE_FEXCH_DETAIL(cq, cqe) &
                    IBT_WC_DETAIL_FC_MATCH_MASK;
                wc->wc_rkey = HERMON_CQE_IMM_ETH_PKEY_CRED_GET(cq, cqe);
                flags |= IBT_WC_FEXCH_FMT | IBT_WC_RKEY_INVALIDATED;
        } else {
                /*
                 * Parse the remaining contents of the CQE into the work
                 * completion.  This means filling in SL, QP number, SLID,
                 * immediate data, etc.
                 *
                 * Note: Not all of these fields are valid in a given
                 * completion.  Many of them depend on the actual type of
                 * completion.  So we fill in all of the fields and leave
                 * it up to the IBTF and consumer to sort out which are
                 * valid based on their context.
                 */
                wc->wc_sl         = HERMON_CQE_SL_GET(cq, cqe);
                wc->wc_qpn        = HERMON_CQE_DQPN_GET(cq, cqe);
                wc->wc_slid       = HERMON_CQE_DLID_GET(cq, cqe);
                wc->wc_immed_data =
                    HERMON_CQE_IMM_ETH_PKEY_CRED_GET(cq, cqe);
                wc->wc_ethertype  = (wc->wc_immed_data & 0xFFFF);
                wc->wc_pkey_ix    = (wc->wc_immed_data &
                    ((1 << state->hs_queryport.log_max_pkey) - 1));
                /*
                 * Fill in "bytes transferred" as appropriate.  Also,
                 * if necessary, fill in the "path bits" field.
                 */
                wc->wc_path_bits = HERMON_CQE_PATHBITS_GET(cq, cqe);
                wc->wc_bytes_xfer = HERMON_CQE_BYTECNT_GET(cq, cqe);

                /*
                 * Check for GRH, update the flags, then fill in "wc_flags"
                 * field in the work completion
                 */
                if (HERMON_CQE_GRH_GET(cq, cqe) != 0) {
                        flags |= IBT_WC_GRH_PRESENT;
                }

                /* Receive CQE */
                switch (opcode) {
                case HERMON_CQE_RCV_SEND_IMM:
                        /*
                         * Note:  According to the PRM, all QP1 recv
                         * completions look like the result of a Send with
                         * Immediate.  They are not, however, (MADs are Send
                         * Only) so we need to check the QP number and set
                         * the flag only if it is non-QP1.
                         */
                        qpnum    = HERMON_CQE_QPNUM_GET(cq, cqe);
                        qp1_indx = state->hs_spec_qp1->hr_indx;
                        if ((qpnum < qp1_indx) || (qpnum > qp1_indx + 1)) {
                                flags |= IBT_WC_IMMED_DATA_PRESENT;
                        }
                        /* FALLTHROUGH */

                case HERMON_CQE_RCV_SEND:
                        type = IBT_WRC_RECV;
                        if (HERMON_CQE_IS_IPOK(cq, cqe)) {
                                wc->wc_cksum = HERMON_CQE_CKSUM(cq, cqe);
                                flags |= IBT_WC_CKSUM_OK;
                                wc->wc_detail = IBT_WC_DETAIL_ALL_FLAGS_MASK &
                                    HERMON_CQE_IPOIB_STATUS(cq, cqe);
                        }
                        break;

                case HERMON_CQE_RCV_SEND_INV:
                        type = IBT_WRC_RECV;
                        flags |= IBT_WC_RKEY_INVALIDATED;
                        wc->wc_rkey = wc->wc_immed_data; /* same field in cqe */
                        break;

                case HERMON_CQE_RCV_RDMAWR_IMM:
                        flags |= IBT_WC_IMMED_DATA_PRESENT;
                        type = IBT_WRC_RECV_RDMAWI;
                        break;

                default:

                        HERMON_WARNING(state, "unknown recv CQE type");
                        wc->wc_status = IBT_WC_LOCAL_QP_OP_ERR;
                        return;
                }
        }
        wc->wc_type = type;
        wc->wc_flags = flags;
        wc->wc_status = IBT_WC_SUCCESS;
}

/*
 * hermon_cq_errcqe_consume()
 *    Context: Can be called from interrupt or base context.
 */
static void
hermon_cq_errcqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
    hermon_hw_cqe_t *cqe, ibt_wc_t *wc)
{
        uint32_t                imm_eth_pkey_cred;
        uint_t                  status;
        ibt_wc_status_t         ibt_status;

        /*
         * Fetch the Work Request ID using the information in the CQE.
         * See hermon_wr.c for more details.
         */
        wc->wc_id = hermon_wrid_get_entry(cq, cqe);

        /*
         * Parse the CQE opcode to determine completion type.  We know that
         * the CQE is an error completion, so we extract only the completion
         * status/syndrome here.
         */
        imm_eth_pkey_cred = HERMON_CQE_ERROR_SYNDROME_GET(cq, cqe);
        status = imm_eth_pkey_cred;
        if (status != HERMON_CQE_WR_FLUSHED_ERR)
                IBTF_DPRINTF_L2("CQE ERR", "cqe %p QPN %x indx %x status 0x%x  "
                    "vendor syndrome %x", cqe, HERMON_CQE_QPNUM_GET(cq, cqe),
                    HERMON_CQE_WQECNTR_GET(cq, cqe), status,
                    HERMON_CQE_ERROR_VENDOR_SYNDROME_GET(cq, cqe));
        switch (status) {
        case HERMON_CQE_LOC_LEN_ERR:
                HERMON_WARNING(state, HERMON_FMA_LOCLEN);
                ibt_status = IBT_WC_LOCAL_LEN_ERR;
                break;

        case HERMON_CQE_LOC_OP_ERR:
                HERMON_WARNING(state, HERMON_FMA_LOCQPOP);
                ibt_status = IBT_WC_LOCAL_QP_OP_ERR;
                break;

        case HERMON_CQE_LOC_PROT_ERR:
                HERMON_WARNING(state, HERMON_FMA_LOCPROT);
                ibt_status = IBT_WC_LOCAL_PROTECT_ERR;
                IBTF_DPRINTF_L2("ERRCQE", "is at %p", cqe);
                if (hermon_should_panic) {
                        cmn_err(CE_PANIC, "Hermon intentional PANIC - "
                            "Local Protection Error\n");
                }
                break;

        case HERMON_CQE_WR_FLUSHED_ERR:
                ibt_status = IBT_WC_WR_FLUSHED_ERR;
                break;

        case HERMON_CQE_MW_BIND_ERR:
                HERMON_WARNING(state, HERMON_FMA_MWBIND);
                ibt_status = IBT_WC_MEM_WIN_BIND_ERR;
                break;

        case HERMON_CQE_BAD_RESPONSE_ERR:
                HERMON_WARNING(state, HERMON_FMA_RESP);
                ibt_status = IBT_WC_BAD_RESPONSE_ERR;
                break;

        case HERMON_CQE_LOCAL_ACCESS_ERR:
                HERMON_WARNING(state, HERMON_FMA_LOCACC);
                ibt_status = IBT_WC_LOCAL_ACCESS_ERR;
                break;

        case HERMON_CQE_REM_INV_REQ_ERR:
                HERMON_WARNING(state, HERMON_FMA_REMREQ);
                ibt_status = IBT_WC_REMOTE_INVALID_REQ_ERR;
                break;

        case HERMON_CQE_REM_ACC_ERR:
                HERMON_WARNING(state, HERMON_FMA_REMACC);
                ibt_status = IBT_WC_REMOTE_ACCESS_ERR;
                break;

        case HERMON_CQE_REM_OP_ERR:
                HERMON_WARNING(state, HERMON_FMA_REMOP);
                ibt_status = IBT_WC_REMOTE_OP_ERR;
                break;

        case HERMON_CQE_TRANS_TO_ERR:
                HERMON_WARNING(state, HERMON_FMA_XPORTCNT);
                ibt_status = IBT_WC_TRANS_TIMEOUT_ERR;
                break;

        case HERMON_CQE_RNRNAK_TO_ERR:
                HERMON_WARNING(state, HERMON_FMA_RNRCNT);
                ibt_status = IBT_WC_RNR_NAK_TIMEOUT_ERR;
                break;

        /*
         * The following error codes are not supported in the Hermon driver
         * as they relate only to Reliable Datagram completion statuses:
         *    case HERMON_CQE_LOCAL_RDD_VIO_ERR:
         *    case HERMON_CQE_REM_INV_RD_REQ_ERR:
         *    case HERMON_CQE_EEC_REM_ABORTED_ERR:
         *    case HERMON_CQE_INV_EEC_NUM_ERR:
         *    case HERMON_CQE_INV_EEC_STATE_ERR:
         *    case HERMON_CQE_LOC_EEC_ERR:
         */

        default:
                HERMON_WARNING(state, "unknown error CQE status");
                HERMON_FMANOTE(state, HERMON_FMA_UNKN);
                ibt_status = IBT_WC_LOCAL_QP_OP_ERR;
                break;
        }

        wc->wc_status = ibt_status;
}


/*
 * hermon_cq_resize_helper()
 *    Context: Can be called only from user or kernel context.
 */
void
hermon_cq_resize_helper(hermon_state_t *state, hermon_cqhdl_t cq)
{
        hermon_cqhdl_t          resize_hdl;
        int                     status;

        /*
         * we're here because we found the special cqe opcode, so we have
         * to update the cq_handle, release the old resources, clear the
         * flag in the cq_hdl, and release the resize_hdl.  When we return
         * above, it will take care of the rest
         */
        ASSERT(MUTEX_HELD(&cq->cq_lock));

        resize_hdl = cq->cq_resize_hdl;

        /*
         * Deregister the memory for the old Completion Queue.  Note: We
         * really can't return error here because we have no good way to
         * cleanup.  Plus, the deregistration really shouldn't ever happen.
         * So, if it does, it is an indication that something has gone
         * seriously wrong.  So we print a warning message and return error
         * (knowing, of course, that the "old" CQ memory will be leaked)
         */
        status = hermon_mr_deregister(state, &cq->cq_mrhdl, HERMON_MR_DEREG_ALL,
            HERMON_SLEEP);
        if (status != DDI_SUCCESS) {
                HERMON_WARNING(state, "failed to deregister old CQ memory");
        }

        /* Next, free the memory from the old CQ buffer */
        hermon_queue_free(&cq->cq_cqinfo);

        /* now we can update the cq_hdl with the new things saved */

        cq->cq_buf   = resize_hdl->cq_buf;
        cq->cq_mrhdl = resize_hdl->cq_mrhdl;
        cq->cq_bufsz = resize_hdl->cq_bufsz;
        cq->cq_log_cqsz = resize_hdl->cq_log_cqsz;
        cq->cq_umap_dhp = cq->cq_resize_hdl->cq_umap_dhp;
        cq->cq_resize_hdl = 0;
        bcopy(&resize_hdl->cq_cqinfo, &cq->cq_cqinfo,
            sizeof (struct hermon_qalloc_info_s));

        /* finally, release the resizing handle */
        kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
}


/*
 * hermon_cq_entries_flush()
 * Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
void
hermon_cq_entries_flush(hermon_state_t *state, hermon_qphdl_t qp)
{
        hermon_cqhdl_t          cq;
        hermon_hw_cqe_t         *cqe, *next_cqe;
        hermon_srqhdl_t         srq;
        hermon_workq_hdr_t      *wq;
        uint32_t                cons_indx, tail_cons_indx, wrap_around_mask;
        uint32_t                new_indx, check_indx, qpnum;
        uint32_t                shift, mask;
        int                     outstanding_cqes;

        qpnum = qp->qp_qpnum;
        if ((srq = qp->qp_srqhdl) != NULL)
                wq = qp->qp_srqhdl->srq_wq_wqhdr;
        else
                wq = NULL;
        cq = qp->qp_rq_cqhdl;

        if (cq == NULL) {
                cq = qp->qp_sq_cqhdl;
        }

do_send_cq:     /* loop back to here if send_cq is not the same as recv_cq */
        if (cq == NULL)
                return;

        cons_indx = cq->cq_consindx;
        shift = cq->cq_log_cqsz;
        mask = cq->cq_bufsz;
        wrap_around_mask = mask - 1;

        /* Calculate the pointer to the first CQ entry */
        cqe = &cq->cq_buf[cons_indx & wrap_around_mask];

        /*
         * Loop through the CQ looking for entries owned by software.  If an
         * entry is owned by software then we increment an 'outstanding_cqes'
         * count to know how many entries total we have on our CQ.  We use this
         * value further down to know how many entries to loop through looking
         * for our same QP number.
         */
        outstanding_cqes = 0;
        tail_cons_indx = cons_indx;
        while (HERMON_CQE_OWNER_IS_SW(cq, cqe, tail_cons_indx, shift, mask)) {
                /* increment total cqes count */
                outstanding_cqes++;

                /* increment the consumer index */
                tail_cons_indx++;

                /* update the pointer to the next cq entry */
                cqe = &cq->cq_buf[tail_cons_indx & wrap_around_mask];
        }

        /*
         * Using the 'tail_cons_indx' that was just set, we now know how many
         * total CQEs possible there are.  Set the 'check_indx' and the
         * 'new_indx' to the last entry identified by 'tail_cons_indx'
         */
        check_indx = new_indx = (tail_cons_indx - 1);

        while (--outstanding_cqes >= 0) {
                cqe = &cq->cq_buf[check_indx & wrap_around_mask];

                /*
                 * If the QP number is the same in the CQE as the QP, then
                 * we must "consume" it.  If it is for an SRQ wqe, then we
                 * also must free the wqe back onto the free list of the SRQ.
                 */
                if (qpnum == HERMON_CQE_QPNUM_GET(cq, cqe)) {
                        if (srq && (HERMON_CQE_SENDRECV_GET(cq, cqe) ==
                            HERMON_COMPLETION_RECV)) {
                                uint64_t *desc;
                                int indx;

                                /* Add wqe back to SRQ free list */
                                indx = HERMON_CQE_WQEADDRSZ_GET(cq, cqe) &
                                    wq->wq_mask;
                                desc = HERMON_SRQ_WQE_ADDR(srq, wq->wq_tail);
                                ((uint16_t *)desc)[1] = htons(indx);
                                wq->wq_tail = indx;
                        }
                } else {        /* CQEs for other QPNs need to remain */
                        if (check_indx != new_indx) {
                                next_cqe =
                                    &cq->cq_buf[new_indx & wrap_around_mask];
                                /* Copy the CQE into the "next_cqe" pointer. */
                                bcopy(cqe, next_cqe, sizeof (hermon_hw_cqe_t));
                        }
                        new_indx--;     /* move index to next CQE to fill */
                }
                check_indx--;           /* move index to next CQE to check */
        }

        /*
         * Update consumer index to be the 'new_indx'.  This moves it past all
         * removed entries.  Because 'new_indx' is pointing to the last
         * previously valid SW owned entry, we add 1 to point the cons_indx to
         * the first HW owned entry.
         */
        cons_indx = (new_indx + 1);

        /*
         * Now we only ring the doorbell (to update the consumer index) if
         * we've actually consumed a CQ entry.  If we found no QP number
         * matches above, then we would not have removed anything.  So only if
         * something was removed do we ring the doorbell.
         */
        if (cq->cq_consindx != cons_indx) {
                /*
                 * Update the consumer index in both the CQ handle and the
                 * doorbell record.
                 */
                cq->cq_consindx = cons_indx;

                hermon_cq_update_ci_doorbell(cq);

        }
        if (cq != qp->qp_sq_cqhdl) {
                cq = qp->qp_sq_cqhdl;
                goto do_send_cq;
        }
}

/*
 * hermon_get_cq_sched_list()
 *    Context: Only called from attach() path context
 *
 * Read properties, creating entries in hs_cq_sched_list with
 * information about the requested "expected" and "minimum"
 * number of MSI-X interrupt vectors per list entry.
 */
static int
hermon_get_cq_sched_list(hermon_state_t *state)
{
        char **listp, ulp_prop[HERMON_CQH_MAX + 4];
        uint_t nlist, i, j, ndata;
        int *data;
        size_t len;
        hermon_cq_sched_t *cq_schedp;

        if (ddi_prop_lookup_string_array(DDI_DEV_T_ANY, state->hs_dip,
            DDI_PROP_DONTPASS, "cqh-group-list", &listp, &nlist) !=
            DDI_PROP_SUCCESS)
                return (0);

        state->hs_cq_sched_array_size = nlist;
        state->hs_cq_sched_array = cq_schedp = kmem_zalloc(nlist *
            sizeof (hermon_cq_sched_t), KM_SLEEP);
        for (i = 0; i < nlist; i++) {
                if ((len = strlen(listp[i])) >= HERMON_CQH_MAX) {
                        cmn_err(CE_CONT, "'cqh' property name too long\n");
                        goto game_over;
                }
                for (j = 0; j < i; j++) {
                        if (strcmp(listp[j], listp[i]) == 0) {
                                cmn_err(CE_CONT, "Duplicate 'cqh' property\n");
                                goto game_over;
                        }
                }
                (void) strncpy(cq_schedp[i].cqs_name, listp[i], HERMON_CQH_MAX);
                ulp_prop[0] = 'c';
                ulp_prop[1] = 'q';
                ulp_prop[2] = 'h';
                ulp_prop[3] = '-';
                (void) strncpy(ulp_prop + 4, listp[i], len + 1);
                if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->hs_dip,
                    DDI_PROP_DONTPASS, ulp_prop, &data, &ndata) !=
                    DDI_PROP_SUCCESS) {
                        cmn_err(CE_CONT, "property '%s' not found\n", ulp_prop);
                        goto game_over;
                }
                if (ndata != 2) {
                        cmn_err(CE_CONT, "property '%s' does not "
                            "have 2 integers\n", ulp_prop);
                        goto game_over_free_data;
                }
                cq_schedp[i].cqs_desired = data[0];
                cq_schedp[i].cqs_minimum = data[1];
                cq_schedp[i].cqs_refcnt = 0;
                ddi_prop_free(data);
        }
        if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->hs_dip,
            DDI_PROP_DONTPASS, "cqh-default", &data, &ndata) !=
            DDI_PROP_SUCCESS) {
                cmn_err(CE_CONT, "property 'cqh-default' not found\n");
                goto game_over;
        }
        if (ndata != 2) {
                cmn_err(CE_CONT, "property 'cqh-default' does not "
                    "have 2 integers\n");
                goto game_over_free_data;
        }
        cq_schedp = &state->hs_cq_sched_default;
        cq_schedp->cqs_desired = data[0];
        cq_schedp->cqs_minimum = data[1];
        cq_schedp->cqs_refcnt = 0;
        ddi_prop_free(data);
        ddi_prop_free(listp);
        return (1);             /* game on */

game_over_free_data:
        ddi_prop_free(data);
game_over:
        cmn_err(CE_CONT, "Error in 'cqh' properties in hermon.conf\n");
        cmn_err(CE_CONT, "completion handler groups not being used\n");
        kmem_free(cq_schedp, nlist * sizeof (hermon_cq_sched_t));
        state->hs_cq_sched_array_size = 0;
        ddi_prop_free(listp);
        return (0);
}

/*
 * hermon_cq_sched_init()
 *    Context: Only called from attach() path context
 *
 * Read the hermon.conf properties looking for cq_sched info,
 * creating reserved pools of MSI-X interrupt ranges for the
 * specified ULPs.
 */
int
hermon_cq_sched_init(hermon_state_t *state)
{
        hermon_cq_sched_t *cq_schedp, *defp;
        int i, desired, array_size;

        mutex_init(&state->hs_cq_sched_lock, NULL, MUTEX_DRIVER,
            DDI_INTR_PRI(state->hs_intrmsi_pri));

        mutex_enter(&state->hs_cq_sched_lock);
        state->hs_cq_sched_array = NULL;

        /* initialize cq_sched_default */
        defp = &state->hs_cq_sched_default;
        defp->cqs_start_hid = 1;
        defp->cqs_len = state->hs_intrmsi_allocd;
        defp->cqs_next_alloc = defp->cqs_len - 1;
        (void) strncpy(defp->cqs_name, "default", 8);

        /* Read properties to determine which ULPs use cq_sched */
        if (hermon_get_cq_sched_list(state) == 0)
                goto done;

        /* Determine if we have enough vectors, or if we have to scale down */
        desired = defp->cqs_desired;    /* default desired (from hermon.conf) */
        if (desired <= 0)
                goto done;              /* all interrupts in the default pool */
        cq_schedp = state->hs_cq_sched_array;
        array_size = state->hs_cq_sched_array_size;
        for (i = 0; i < array_size; i++)
                desired += cq_schedp[i].cqs_desired;
        if (desired > state->hs_intrmsi_allocd) {
                cmn_err(CE_CONT, "#interrupts allocated (%d) is less than "
                    "the #interrupts desired (%d)\n",
                    state->hs_intrmsi_allocd, desired);
                cmn_err(CE_CONT, "completion handler groups not being used\n");
                goto done;              /* all interrupts in the default pool */
        }
        /* Game on.  For each cq_sched group, reserve the MSI-X range */
        for (i = 0; i < array_size; i++) {
                desired = cq_schedp[i].cqs_desired;
                cq_schedp[i].cqs_start_hid = defp->cqs_start_hid;
                cq_schedp[i].cqs_len = desired;
                cq_schedp[i].cqs_next_alloc = desired - 1;
                defp->cqs_len -= desired;
                defp->cqs_start_hid += desired;
        }
        /* reset default's start allocation seed */
        state->hs_cq_sched_default.cqs_next_alloc =
            state->hs_cq_sched_default.cqs_len - 1;

done:
        mutex_exit(&state->hs_cq_sched_lock);
        return (IBT_SUCCESS);
}

void
hermon_cq_sched_fini(hermon_state_t *state)
{
        mutex_enter(&state->hs_cq_sched_lock);
        if (state->hs_cq_sched_array_size) {
                kmem_free(state->hs_cq_sched_array, sizeof (hermon_cq_sched_t) *
                    state->hs_cq_sched_array_size);
                state->hs_cq_sched_array_size = 0;
                state->hs_cq_sched_array = NULL;
        }
        mutex_exit(&state->hs_cq_sched_lock);
        mutex_destroy(&state->hs_cq_sched_lock);
}

int
hermon_cq_sched_alloc(hermon_state_t *state, ibt_cq_sched_attr_t *attr,
    hermon_cq_sched_t **cq_sched_pp)
{
        hermon_cq_sched_t       *cq_schedp;
        int                     i;
        char                    *name;
        ibt_cq_sched_flags_t    flags;

        flags = attr->cqs_flags;
        if ((flags & (IBT_CQS_SCHED_GROUP | IBT_CQS_EXACT_SCHED_GROUP)) == 0) {
                *cq_sched_pp = NULL;
                return (IBT_SUCCESS);
        }
        name = attr->cqs_pool_name;

        mutex_enter(&state->hs_cq_sched_lock);
        cq_schedp = state->hs_cq_sched_array;
        for (i = 0; i < state->hs_cq_sched_array_size; i++, cq_schedp++) {
                if (strcmp(name, cq_schedp->cqs_name) == 0) {
                        if (cq_schedp->cqs_len != 0)
                                cq_schedp->cqs_refcnt++;
                        break;  /* found it */
                }
        }
        if ((i == state->hs_cq_sched_array_size) ||     /* not found, or */
            (cq_schedp->cqs_len == 0)) /* defined, but no dedicated intr's */
                cq_schedp = NULL;
        mutex_exit(&state->hs_cq_sched_lock);

        *cq_sched_pp = cq_schedp;       /* set to valid hdl, or to NULL */
        if ((cq_schedp == NULL) &&
            (attr->cqs_flags & IBT_CQS_EXACT_SCHED_GROUP))
                return (IBT_CQ_NO_SCHED_GROUP);
        else
                return (IBT_SUCCESS);
}

int
hermon_cq_sched_free(hermon_state_t *state, hermon_cq_sched_t *cq_schedp)
{
        if (cq_schedp != NULL) {
                /* Just decrement refcnt */
                mutex_enter(&state->hs_cq_sched_lock);
                if (cq_schedp->cqs_refcnt == 0)
                        HERMON_WARNING(state, "cq_sched free underflow\n");
                else
                        cq_schedp->cqs_refcnt--;
                mutex_exit(&state->hs_cq_sched_lock);
        }
        return (IBT_SUCCESS);
}