/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright (c) 2021, the University of Queensland
 * Copyright 2020 RackTop Systems, Inc.
 * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 */

/*
 * Mellanox Connect-X 4/5/6 driver.
 */

#include <sys/modctl.h>
#include <sys/conf.h>
#include <sys/devops.h>
#include <sys/sysmacros.h>
#include <sys/disp.h>
#include <sys/sdt.h>

#include <sys/mac_provider.h>

#include <mlxcx.h>

/*
 * CTASSERT(s) to cover bad values which would induce bugs.
 */
CTASSERT(MLXCX_CQ_LWM_GAP >= MLXCX_CQ_HWM_GAP);

/*
 * Disable interrupts.
 * The act of calling ddi_intr_disable() does not guarantee an interrupt
 * routine is not running, so flag the vector as quiescing and wait
 * for anything active to finish.
 */
void
mlxcx_intr_disable(mlxcx_t *mlxp)
{
        int i;

        mlxcx_cmd_eq_disable(mlxp);

        for (i = 0; i < mlxp->mlx_intr_count; ++i) {
                mlxcx_event_queue_t *mleq = &mlxp->mlx_eqs[i];

                mutex_enter(&mleq->mleq_mtx);

                if ((mleq->mleq_state & MLXCX_EQ_INTR_ENABLED) == 0) {
                        mutex_exit(&mleq->mleq_mtx);
                        continue;
                }

                (void) ddi_intr_disable(mlxp->mlx_intr_handles[i]);

                mleq->mleq_state |= MLXCX_EQ_INTR_QUIESCE;
                while ((mleq->mleq_state & MLXCX_EQ_INTR_ACTIVE) != 0)
                        cv_wait(&mleq->mleq_cv, &mleq->mleq_mtx);

                mleq->mleq_state &= ~MLXCX_EQ_INTR_ENABLED;

                mutex_exit(&mleq->mleq_mtx);
        }
}

void
mlxcx_intr_teardown(mlxcx_t *mlxp)
{
        int i;
        int ret;

        for (i = 0; i < mlxp->mlx_intr_count; ++i) {
                mlxcx_event_queue_t *mleq = &mlxp->mlx_eqs[i];

                mutex_enter(&mleq->mleq_mtx);
                VERIFY0(mleq->mleq_state & MLXCX_EQ_ALLOC);
                if (mleq->mleq_state & MLXCX_EQ_CREATED)
                        VERIFY(mleq->mleq_state & MLXCX_EQ_DESTROYED);
                if (i >= mlxp->mlx_intr_cq0) {
                        VERIFY(avl_is_empty(&mleq->mleq_cqs));
                        avl_destroy(&mleq->mleq_cqs);
                }
                mutex_exit(&mleq->mleq_mtx);
                (void) ddi_intr_remove_handler(mlxp->mlx_intr_handles[i]);
                ret = ddi_intr_free(mlxp->mlx_intr_handles[i]);
                if (ret != DDI_SUCCESS) {
                        mlxcx_warn(mlxp, "failed to free interrupt %d: %d",
                            i, ret);
                }
                mutex_destroy(&mleq->mleq_mtx);
                cv_destroy(&mleq->mleq_cv);
        }
        kmem_free(mlxp->mlx_intr_handles, mlxp->mlx_intr_size);
        kmem_free(mlxp->mlx_eqs, mlxp->mlx_eqs_size);
        mlxp->mlx_intr_handles = NULL;
        mlxp->mlx_eqs = NULL;
}

/*
 * Get the next SW-owned entry on the event queue, or NULL if we reach the end.
 */
static mlxcx_eventq_ent_t *
mlxcx_eq_next(mlxcx_event_queue_t *mleq)
{
        mlxcx_eventq_ent_t *ent;
        ddi_fm_error_t err;
        uint_t ci;
        const uint_t swowner = ((mleq->mleq_cc >> mleq->mleq_entshift) & 1);

        /*
         * This should only be called from interrupt context to ensure
         * correctness of mleq_cc.
         */
        ASSERT(servicing_interrupt());
        ASSERT(mleq->mleq_state & MLXCX_EQ_CREATED);
        ASSERT0(mleq->mleq_state & MLXCX_EQ_DESTROYED);

        /* mleq_nents is always a power of 2 */
        ci = mleq->mleq_cc & (mleq->mleq_nents - 1);

        ent = &mleq->mleq_ent[ci];
        VERIFY0(ddi_dma_sync(mleq->mleq_dma.mxdb_dma_handle,
            (uintptr_t)ent - (uintptr_t)mleq->mleq_ent,
            sizeof (mlxcx_eventq_ent_t), DDI_DMA_SYNC_FORCPU));
        ddi_fm_dma_err_get(mleq->mleq_dma.mxdb_dma_handle, &err,
            DDI_FME_VERSION);
        if (err.fme_status == DDI_FM_OK && (ent->mleqe_owner & 1) == swowner) {
                /* The PRM says we have to membar here, so we're doing it */
                membar_consumer();
                ++mleq->mleq_cc;
                return (ent);
        }
        /*
         * In the case of a DMA error, we should re-arm this EQ and then come
         * back and try again when the device wakes us back up.
         *
         * Hopefully the fault will be gone by then.
         */
        ddi_fm_dma_err_clear(mleq->mleq_dma.mxdb_dma_handle, DDI_FME_VERSION);

        return (NULL);
}

void
mlxcx_arm_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq)
{
        uint_t try = 0;
        ddi_fm_error_t err;
        bits32_t v = new_bits32();

        /*
         * This is only called during initialization when the EQ is
         * armed for the first time, and when re-armed at the end of
         * interrupt processing.
         */
        ASSERT(mutex_owned(&mleq->mleq_mtx) || servicing_interrupt());
        ASSERT(mleq->mleq_state & MLXCX_EQ_CREATED);
        ASSERT0(mleq->mleq_state & MLXCX_EQ_DESTROYED);
        ASSERT0(mleq->mleq_state & MLXCX_EQ_ARMED);
        ASSERT0(mleq->mleq_state & MLXCX_EQ_POLLING);

        mleq->mleq_state |= MLXCX_EQ_ARMED;
        mleq->mleq_cc_armed = mleq->mleq_cc;

        set_bits32(&v, MLXCX_EQ_ARM_EQN, mleq->mleq_num);
        set_bits32(&v, MLXCX_EQ_ARM_CI, mleq->mleq_cc);

retry:
        mlxcx_uar_put32(mlxp, mleq->mleq_uar, MLXCX_UAR_EQ_ARM,
            from_bits32(v));
        ddi_fm_acc_err_get(mlxp->mlx_regs_handle, &err,
            DDI_FME_VERSION);
        if (err.fme_status == DDI_FM_OK)
                return;
        if (try++ < mlxcx_doorbell_tries) {
                ddi_fm_acc_err_clear(mlxp->mlx_regs_handle, DDI_FME_VERSION);
                goto retry;
        }
        ddi_fm_service_impact(mlxp->mlx_dip, DDI_SERVICE_LOST);
}

static void
mlxcx_update_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq)
{
        bits32_t v = new_bits32();
        ddi_fm_error_t err;

        /*
         * This should only be called from interrupt context to ensure
         * correctness of mleq_cc.
         */
        ASSERT(servicing_interrupt());
        ASSERT(mleq->mleq_state & MLXCX_EQ_CREATED);
        ASSERT0(mleq->mleq_state & MLXCX_EQ_DESTROYED);
        ASSERT0(mleq->mleq_state & MLXCX_EQ_ARMED);

        set_bits32(&v, MLXCX_EQ_ARM_EQN, mleq->mleq_num);
        set_bits32(&v, MLXCX_EQ_ARM_CI, mleq->mleq_cc);

        mlxcx_uar_put32(mlxp, mleq->mleq_uar, MLXCX_UAR_EQ_NOARM,
            from_bits32(v));
        ddi_fm_acc_err_get(mlxp->mlx_regs_handle, &err,
            DDI_FME_VERSION);
        ddi_fm_acc_err_clear(mlxp->mlx_regs_handle, DDI_FME_VERSION);
        /*
         * Ignore the error, if it's still happening when we try to re-arm the
         * EQ, we will note the impact then.
         */
}

static mlxcx_completionq_ent_t *
mlxcx_cq_next(mlxcx_completion_queue_t *mlcq)
{
        mlxcx_completionq_ent_t *ent;
        ddi_fm_error_t err;
        uint_t ci;
        const uint_t swowner = ((mlcq->mlcq_cc >> mlcq->mlcq_entshift) & 1);

        ASSERT(mutex_owned(&mlcq->mlcq_mtx));
        ASSERT(mlcq->mlcq_state & MLXCX_CQ_CREATED);
        ASSERT0(mlcq->mlcq_state & MLXCX_CQ_DESTROYED);

        /* mlcq_nents is always a power of 2 */
        ci = mlcq->mlcq_cc & (mlcq->mlcq_nents - 1);

        ent = &mlcq->mlcq_ent[ci];
        VERIFY0(ddi_dma_sync(mlcq->mlcq_dma.mxdb_dma_handle,
            (uintptr_t)ent - (uintptr_t)mlcq->mlcq_ent,
            sizeof (mlxcx_completionq_ent_t), DDI_DMA_SYNC_FORCPU));
        ddi_fm_dma_err_get(mlcq->mlcq_dma.mxdb_dma_handle, &err,
            DDI_FME_VERSION);
        if (err.fme_status == DDI_FM_OK && (ent->mlcqe_owner & 1) == swowner) {
                /* The PRM says we have to membar here, so we're doing it */
                membar_consumer();
                ++mlcq->mlcq_cc;
                return (ent);
        }
        ddi_fm_dma_err_clear(mlcq->mlcq_dma.mxdb_dma_handle, DDI_FME_VERSION);

        return (NULL);
}

void
mlxcx_update_cqci(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq)
{
        ddi_fm_error_t err;
        uint_t try = 0;

        mlcq->mlcq_doorbell->mlcqd_update_ci = to_be24(mlcq->mlcq_cc);

retry:
        MLXCX_DMA_SYNC(mlcq->mlcq_doorbell_dma, DDI_DMA_SYNC_FORDEV);
        ddi_fm_dma_err_get(mlcq->mlcq_doorbell_dma.mxdb_dma_handle, &err,
            DDI_FME_VERSION);
        if (err.fme_status != DDI_FM_OK) {
                if (try++ < mlxcx_doorbell_tries) {
                        ddi_fm_dma_err_clear(
                            mlcq->mlcq_doorbell_dma.mxdb_dma_handle,
                            DDI_FME_VERSION);
                        goto retry;
                } else {
                        ddi_fm_service_impact(mlxp->mlx_dip, DDI_SERVICE_LOST);
                        return;
                }
        }
}

void
mlxcx_arm_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq)
{
        bits32_t dbval = new_bits32();
        uint64_t udbval;
        ddi_fm_error_t err;
        uint_t try = 0;

        ASSERT(mutex_owned(&mlcq->mlcq_arm_mtx));
        ASSERT(mutex_owned(&mlcq->mlcq_mtx));
        ASSERT(mlcq->mlcq_state & MLXCX_CQ_CREATED);
        ASSERT0(mlcq->mlcq_state & MLXCX_CQ_DESTROYED);

        if (mlcq->mlcq_state & MLXCX_CQ_ARMED) {
                ASSERT3U(mlcq->mlcq_ec, >, mlcq->mlcq_ec_armed);
        }

        if (mlcq->mlcq_state & MLXCX_CQ_TEARDOWN)
                return;

        atomic_or_uint(&mlcq->mlcq_state, MLXCX_CQ_ARMED);
        mlcq->mlcq_cc_armed = mlcq->mlcq_cc;
        mlcq->mlcq_ec_armed = mlcq->mlcq_ec;

        set_bits32(&dbval, MLXCX_CQ_ARM_SEQ, mlcq->mlcq_ec);
        set_bits32(&dbval, MLXCX_CQ_ARM_CI, mlcq->mlcq_cc);

        udbval = (uint64_t)from_bits32(dbval) << 32;
        udbval |= mlcq->mlcq_num & 0xffffff;

        mlcq->mlcq_doorbell->mlcqd_update_ci = to_be24(mlcq->mlcq_cc);
        mlcq->mlcq_doorbell->mlcqd_arm_ci = dbval;

retry:
        MLXCX_DMA_SYNC(mlcq->mlcq_doorbell_dma, DDI_DMA_SYNC_FORDEV);
        ddi_fm_dma_err_get(mlcq->mlcq_doorbell_dma.mxdb_dma_handle, &err,
            DDI_FME_VERSION);
        if (err.fme_status != DDI_FM_OK) {
                if (try++ < mlxcx_doorbell_tries) {
                        ddi_fm_dma_err_clear(
                            mlcq->mlcq_doorbell_dma.mxdb_dma_handle,
                            DDI_FME_VERSION);
                        goto retry;
                } else {
                        goto err;
                }
        }

        mlxcx_uar_put64(mlxp, mlcq->mlcq_uar, MLXCX_UAR_CQ_ARM, udbval);
        ddi_fm_acc_err_get(mlxp->mlx_regs_handle, &err,
            DDI_FME_VERSION);
        if (err.fme_status == DDI_FM_OK)
                return;
        if (try++ < mlxcx_doorbell_tries) {
                ddi_fm_acc_err_clear(mlxp->mlx_regs_handle, DDI_FME_VERSION);
                goto retry;
        }

err:
        ddi_fm_service_impact(mlxp->mlx_dip, DDI_SERVICE_LOST);
}

const char *
mlxcx_event_name(mlxcx_event_t evt)
{
        switch (evt) {
        case MLXCX_EVENT_COMPLETION:
                return ("COMPLETION");
        case MLXCX_EVENT_PATH_MIGRATED:
                return ("PATH_MIGRATED");
        case MLXCX_EVENT_COMM_ESTABLISH:
                return ("COMM_ESTABLISH");
        case MLXCX_EVENT_SENDQ_DRAIN:
                return ("SENDQ_DRAIN");
        case MLXCX_EVENT_LAST_WQE:
                return ("LAST_WQE");
        case MLXCX_EVENT_SRQ_LIMIT:
                return ("SRQ_LIMIT");
        case MLXCX_EVENT_DCT_ALL_CLOSED:
                return ("DCT_ALL_CLOSED");
        case MLXCX_EVENT_DCT_ACCKEY_VIOL:
                return ("DCT_ACCKEY_VIOL");
        case MLXCX_EVENT_CQ_ERROR:
                return ("CQ_ERROR");
        case MLXCX_EVENT_WQ_CATASTROPHE:
                return ("WQ_CATASTROPHE");
        case MLXCX_EVENT_PATH_MIGRATE_FAIL:
                return ("PATH_MIGRATE_FAIL");
        case MLXCX_EVENT_PAGE_FAULT:
                return ("PAGE_FAULT");
        case MLXCX_EVENT_WQ_INVALID_REQ:
                return ("WQ_INVALID_REQ");
        case MLXCX_EVENT_WQ_ACCESS_VIOL:
                return ("WQ_ACCESS_VIOL");
        case MLXCX_EVENT_SRQ_CATASTROPHE:
                return ("SRQ_CATASTROPHE");
        case MLXCX_EVENT_INTERNAL_ERROR:
                return ("INTERNAL_ERROR");
        case MLXCX_EVENT_PORT_STATE:
                return ("PORT_STATE");
        case MLXCX_EVENT_GPIO:
                return ("GPIO");
        case MLXCX_EVENT_PORT_MODULE:
                return ("PORT_MODULE");
        case MLXCX_EVENT_TEMP_WARNING:
                return ("TEMP_WARNING");
        case MLXCX_EVENT_REMOTE_CONFIG:
                return ("REMOTE_CONFIG");
        case MLXCX_EVENT_DCBX_CHANGE:
                return ("DCBX_CHANGE");
        case MLXCX_EVENT_DOORBELL_CONGEST:
                return ("DOORBELL_CONGEST");
        case MLXCX_EVENT_STALL_VL:
                return ("STALL_VL");
        case MLXCX_EVENT_CMD_COMPLETION:
                return ("CMD_COMPLETION");
        case MLXCX_EVENT_PAGE_REQUEST:
                return ("PAGE_REQUEST");
        case MLXCX_EVENT_NIC_VPORT:
                return ("NIC_VPORT");
        case MLXCX_EVENT_EC_PARAMS_CHANGE:
                return ("EC_PARAMS_CHANGE");
        case MLXCX_EVENT_XRQ_ERROR:
                return ("XRQ_ERROR");
        }
        return ("UNKNOWN");
}

/* Should be called only when link state has changed. */
void
mlxcx_update_link_state(mlxcx_t *mlxp, mlxcx_port_t *port)
{
        link_state_t ls;

        mutex_enter(&port->mlp_mtx);
        (void) mlxcx_cmd_query_port_status(mlxp, port);
        (void) mlxcx_cmd_query_port_speed(mlxp, port);
        (void) mlxcx_cmd_query_port_fec(mlxp, port);

        switch (port->mlp_oper_status) {
        case MLXCX_PORT_STATUS_UP:
        case MLXCX_PORT_STATUS_UP_ONCE:
                ls = LINK_STATE_UP;
                break;
        case MLXCX_PORT_STATUS_DOWN:
                ls = LINK_STATE_DOWN;
                break;
        default:
                ls = LINK_STATE_UNKNOWN;
        }

        if (mlxp->mlx_mac_hdl != NULL)
                mac_link_update(mlxp->mlx_mac_hdl, ls);

        mutex_exit(&port->mlp_mtx);
}

CTASSERT(MLXCX_MANAGE_PAGES_MAX_PAGES < UINT_MAX);

static void
mlxcx_give_pages_once(mlxcx_t *mlxp, size_t npages)
{
        ddi_device_acc_attr_t acc;
        ddi_dma_attr_t attr;
        mlxcx_dev_page_t *mdp;
        mlxcx_dev_page_t **pages;
        size_t i;
        const ddi_dma_cookie_t *ck;

        /*
         * If this isn't enough, the HCA will ask for more
         */
        npages = MIN(npages, MLXCX_MANAGE_PAGES_MAX_PAGES);

        pages = kmem_zalloc(sizeof (*pages) * npages, KM_SLEEP);

        for (i = 0; i < npages; i++) {
                mdp = kmem_zalloc(sizeof (mlxcx_dev_page_t), KM_SLEEP);
                mlxcx_dma_acc_attr(mlxp, &acc);
                mlxcx_dma_page_attr(mlxp, &attr);
                if (!mlxcx_dma_alloc(mlxp, &mdp->mxdp_dma, &attr, &acc,
                    B_TRUE, MLXCX_HW_PAGE_SIZE, B_TRUE)) {
                        mlxcx_warn(mlxp, "failed to allocate 4k page %u/%lu", i,
                            npages);
                        kmem_free(mdp, sizeof (mlxcx_dev_page_t));
                        goto cleanup_npages;
                }
                ck = mlxcx_dma_cookie_one(&mdp->mxdp_dma);
                mdp->mxdp_pa = ck->dmac_laddress;
                pages[i] = mdp;
        }

        mutex_enter(&mlxp->mlx_pagemtx);

        if (!mlxcx_cmd_give_pages(mlxp,
            MLXCX_MANAGE_PAGES_OPMOD_GIVE_PAGES, npages, pages)) {
                mlxcx_warn(mlxp, "!hardware refused our gift of %lu "
                    "pages!", npages);
                mutex_exit(&mlxp->mlx_pagemtx);
                goto cleanup_npages;
        }

        for (i = 0; i < npages; i++) {
                avl_add(&mlxp->mlx_pages, pages[i]);
        }
        mlxp->mlx_npages += npages;
        mutex_exit(&mlxp->mlx_pagemtx);

        kmem_free(pages, sizeof (*pages) * npages);

        return;

cleanup_npages:
        for (i = 0; i < npages; i++) {
                if ((mdp = pages[i]) == NULL)
                        break;

                mlxcx_dma_free(&mdp->mxdp_dma);
                kmem_free(mdp, sizeof (mlxcx_dev_page_t));
        }
        /* Tell the hardware we had an allocation failure. */
        (void) mlxcx_cmd_give_pages(mlxp, MLXCX_MANAGE_PAGES_OPMOD_ALLOC_FAIL,
            0, NULL);
        mutex_exit(&mlxp->mlx_pagemtx);

        kmem_free(pages, sizeof (*pages) * npages);
}

static void
mlxcx_take_pages_once(mlxcx_t *mlxp, size_t npages)
{
        uint_t i;
        int32_t ret;
        uint64_t *pas;
        mlxcx_dev_page_t *mdp, probe;

        pas = kmem_alloc(sizeof (*pas) * npages, KM_SLEEP);

        if (!mlxcx_cmd_return_pages(mlxp, npages, pas, &ret)) {
                kmem_free(pas, sizeof (*pas) * npages);
                return;
        }

        mutex_enter(&mlxp->mlx_pagemtx);

        ASSERT0(avl_is_empty(&mlxp->mlx_pages));

        for (i = 0; i < ret; i++) {
                bzero(&probe, sizeof (probe));
                probe.mxdp_pa = pas[i];

                mdp = avl_find(&mlxp->mlx_pages, &probe, NULL);

                if (mdp != NULL) {
                        avl_remove(&mlxp->mlx_pages, mdp);
                        mlxp->mlx_npages--;
                        mlxcx_dma_free(&mdp->mxdp_dma);
                        kmem_free(mdp, sizeof (mlxcx_dev_page_t));
                } else {
                        mlxcx_warn(mlxp, "hardware returned a page "
                            "with PA 0x%" PRIx64 " but we have no "
                            "record of giving out such a page", pas[i]);
                }
        }

        mutex_exit(&mlxp->mlx_pagemtx);

        kmem_free(pas, sizeof (*pas) * npages);
}

static void
mlxcx_pages_task(void *arg)
{
        mlxcx_async_param_t *param = arg;
        mlxcx_t *mlxp = param->mla_mlx;
        int32_t npages;

        /*
         * We can drop the pending status now, as we've extracted what
         * is needed to process the pages request.
         *
         * Even though we should never get another pages request until
         * we have responded to this, along with the guard in mlxcx_sync_intr,
         * this safely allows the reuse of mlxcx_async_param_t.
         */
        mutex_enter(&param->mla_mtx);
        npages = param->mla_pages.mlp_npages;
        param->mla_pending = B_FALSE;
        bzero(&param->mla_pages, sizeof (param->mla_pages));
        mutex_exit(&param->mla_mtx);

        /*
         * The PRM describes npages as: "Number of missing / unneeded pages
         * (signed number, msb indicate sign)". The implication is that
         * it will not be zero. We are expected to use this to give or
         * take back pages (based on the sign) using the MANAGE_PAGES
         * command but we can't determine whether to give or take
         * when npages is zero. So we do nothing.
         */
        if (npages > 0) {
                mlxcx_give_pages_once(mlxp, npages);
        } else if (npages < 0) {
                mlxcx_take_pages_once(mlxp, -1 * npages);
        }
}

static void
mlxcx_link_state_task(void *arg)
{
        mlxcx_async_param_t *param = arg;
        mlxcx_port_t *port;
        mlxcx_t *mlxp;

        /*
         * Gather the argruments from the parameters and clear the
         * pending status.
         *
         * The pending status must be cleared *before* we update the
         * link state. This is both safe and required to ensure we always
         * have the correct link state. It is safe because taskq_ents are
         * reusable (by the caller of taskq_dispatch_ent()) once the
         * task function has started executing. It is necessarily before
         * updating the link state to guarantee further link state change
         * events are not missed and we always have the current link state.
         */
        mutex_enter(&param->mla_mtx);
        mlxp = param->mla_mlx;
        port = param->mla_port;
        param->mla_pending = B_FALSE;
        mutex_exit(&param->mla_mtx);

        mlxcx_update_link_state(mlxp, port);
}

static const char *
mlxcx_module_error_string(mlxcx_module_error_type_t err)
{
        switch (err) {
        case MLXCX_MODULE_ERR_POWER_BUDGET:
                return ("POWER_BUDGET");
        case MLXCX_MODULE_ERR_LONG_RANGE:
                return ("LONG_RANGE");
        case MLXCX_MODULE_ERR_BUS_STUCK:
                return ("BUS_STUCK");
        case MLXCX_MODULE_ERR_NO_EEPROM:
                return ("NO_EEPROM");
        case MLXCX_MODULE_ERR_ENFORCEMENT:
                return ("ENFORCEMENT");
        case MLXCX_MODULE_ERR_UNKNOWN_IDENT:
                return ("UNKNOWN_IDENT");
        case MLXCX_MODULE_ERR_HIGH_TEMP:
                return ("HIGH_TEMP");
        case MLXCX_MODULE_ERR_CABLE_SHORTED:
                return ("CABLE_SHORTED");
        default:
                return ("UNKNOWN");
        }
}

static void
mlxcx_report_module_error(mlxcx_t *mlxp, mlxcx_evdata_port_mod_t *evd)
{
        uint64_t ena;
        char buf[FM_MAX_CLASS];
        const char *lename;
        const char *ename;
        const char *stname;
        uint_t eno = 0;
        mlxcx_module_status_t state = evd->mled_port_mod_module_status;

        switch (state) {
        case MLXCX_MODULE_ERROR:
                stname = "error";
                eno = evd->mled_port_mod_error_type;
                lename = mlxcx_module_error_string(eno);
                switch (eno) {
                case MLXCX_MODULE_ERR_ENFORCEMENT:
                        ename = DDI_FM_TXR_ERROR_WHITELIST;
                        break;
                case MLXCX_MODULE_ERR_UNKNOWN_IDENT:
                case MLXCX_MODULE_ERR_NO_EEPROM:
                        ename = DDI_FM_TXR_ERROR_NOTSUPP;
                        break;
                case MLXCX_MODULE_ERR_HIGH_TEMP:
                        ename = DDI_FM_TXR_ERROR_OVERTEMP;
                        break;
                case MLXCX_MODULE_ERR_POWER_BUDGET:
                case MLXCX_MODULE_ERR_LONG_RANGE:
                case MLXCX_MODULE_ERR_CABLE_SHORTED:
                        ename = DDI_FM_TXR_ERROR_HWFAIL;
                        break;
                case MLXCX_MODULE_ERR_BUS_STUCK:
                default:
                        ename = DDI_FM_TXR_ERROR_UNKNOWN;
                }
                break;
        default:
                return;
        }

        (void) snprintf(buf, FM_MAX_CLASS, "%s.%s",
            DDI_FM_NIC, DDI_FM_TXR_ERROR);
        ena = fm_ena_generate(0, FM_ENA_FMT1);
        if (!DDI_FM_EREPORT_CAP(mlxp->mlx_fm_caps))
                return;

        ddi_fm_ereport_post(mlxp->mlx_dip, buf, ena, DDI_NOSLEEP,
            /* compulsory FM props */
            FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0,
            /* generic NIC txr error event props */
            "error", DATA_TYPE_STRING, ename,
            "port_index", DATA_TYPE_UINT8, 0,
            "txr_index", DATA_TYPE_UINT8, evd->mled_port_mod_module,
            /* local props */
            "mlxcx_state", DATA_TYPE_STRING, stname,
            "mlxcx_error", DATA_TYPE_STRING, lename,
            "mlxcx_error_num", DATA_TYPE_UINT8, eno,
            NULL);
        ddi_fm_service_impact(mlxp->mlx_dip, DDI_SERVICE_LOST);
}

/*
 * Common beginning of interrupt processing.
 * Confirm interrupt hasn't been disabled, verify its state and
 * mark the vector as active.
 */
static boolean_t
mlxcx_intr_ini(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq)
{
        mutex_enter(&mleq->mleq_mtx);

        if ((mleq->mleq_state & MLXCX_EQ_INTR_ENABLED) == 0) {
                mutex_exit(&mleq->mleq_mtx);
                return (B_FALSE);
        }

        if (!(mleq->mleq_state & MLXCX_EQ_ALLOC) ||
            !(mleq->mleq_state & MLXCX_EQ_CREATED) ||
            (mleq->mleq_state & MLXCX_EQ_DESTROYED)) {
                mlxcx_warn(mlxp, "intr %d in bad eq state",
                    mleq->mleq_intr_index);
                mutex_exit(&mleq->mleq_mtx);
                return (B_FALSE);
        }

        mleq->mleq_state |= MLXCX_EQ_INTR_ACTIVE;
        mutex_exit(&mleq->mleq_mtx);

        return (B_TRUE);
}

/*
 * End of interrupt processing.
 * Mark vector as no longer active and if shutdown is blocked on this vector,
 * wake it up.
 */
static void
mlxcx_intr_fini(mlxcx_event_queue_t *mleq)
{
        mutex_enter(&mleq->mleq_mtx);
        if ((mleq->mleq_state & MLXCX_EQ_INTR_QUIESCE) != 0)
                cv_signal(&mleq->mleq_cv);

        mleq->mleq_state &= ~MLXCX_EQ_INTR_ACTIVE;
        mutex_exit(&mleq->mleq_mtx);
}

static uint_t
mlxcx_intr_async(caddr_t arg, caddr_t arg2)
{
        mlxcx_t *mlxp = (mlxcx_t *)arg;
        mlxcx_event_queue_t *mleq = (mlxcx_event_queue_t *)arg2;
        mlxcx_eventq_ent_t *ent;
        mlxcx_async_param_t *param;
        uint_t portn;
        uint16_t func;

        if (!mlxcx_intr_ini(mlxp, mleq))
                return (DDI_INTR_CLAIMED);

        ent = mlxcx_eq_next(mleq);
        if (ent == NULL) {
                goto done;
        }

        ASSERT(mleq->mleq_state & MLXCX_EQ_ARMED);
        mleq->mleq_state &= ~MLXCX_EQ_ARMED;

        for (; ent != NULL; ent = mlxcx_eq_next(mleq)) {
                DTRACE_PROBE2(event, mlxcx_t *, mlxp, mlxcx_eventq_ent_t *,
                    ent);

                /*
                 * Handle events which can be processed while we're still in
                 * mlxcx_attach(). Everything on the mlxcx_t which these events
                 * use must be allocated and set up prior to the call to
                 * mlxcx_setup_async_eqs().
                 */
                switch (ent->mleqe_event_type) {
                case MLXCX_EVENT_CMD_COMPLETION:
                        mlxcx_cmd_completion(mlxp, ent);
                        continue;
                case MLXCX_EVENT_PAGE_REQUEST:
                        func = from_be16(ent->mleqe_page_request.
                            mled_page_request_function_id);
                        VERIFY3U(func, <=, MLXCX_FUNC_ID_MAX);

                        param = &mlxp->mlx_npages_req[func];
                        mutex_enter(&param->mla_mtx);
                        if (param->mla_pending) {
                                /*
                                 * The PRM states we will not get another
                                 * page request event until any pending have
                                 * been posted as complete to the HCA.
                                 * This will guard against this anyway.
                                 */
                                mutex_exit(&param->mla_mtx);
                                mlxcx_warn(mlxp, "Unexpected page request "
                                    "whilst another is pending");
                                continue;
                        }
                        param->mla_pages.mlp_npages =
                            (int32_t)from_be32(ent->mleqe_page_request.
                            mled_page_request_num_pages);
                        param->mla_pages.mlp_func = func;
                        param->mla_pending = B_TRUE;
                        ASSERT3P(param->mla_mlx, ==, mlxp);
                        mutex_exit(&param->mla_mtx);

                        taskq_dispatch_ent(mlxp->mlx_async_tq, mlxcx_pages_task,
                            param, 0, &param->mla_tqe);
                        continue;
                }

                /*
                 * All other events should be ignored while in attach.
                 */
                mutex_enter(&mleq->mleq_mtx);
                if (mleq->mleq_state & MLXCX_EQ_ATTACHING) {
                        mutex_exit(&mleq->mleq_mtx);
                        continue;
                }
                mutex_exit(&mleq->mleq_mtx);

                switch (ent->mleqe_event_type) {
                case MLXCX_EVENT_PORT_STATE:
                        portn = get_bits8(
                            ent->mleqe_port_state.mled_port_state_port_num,
                            MLXCX_EVENT_PORT_NUM) - 1;
                        if (portn >= mlxp->mlx_nports)
                                break;

                        param = &mlxp->mlx_ports[portn].mlx_port_event;
                        mutex_enter(&param->mla_mtx);
                        if (param->mla_pending) {
                                /*
                                 * There is a link state event pending
                                 * processing. When that event is handled
                                 * it will get the current link state.
                                 */
                                mutex_exit(&param->mla_mtx);
                                break;
                        }

                        ASSERT3P(param->mla_mlx, ==, mlxp);
                        ASSERT3P(param->mla_port, ==, &mlxp->mlx_ports[portn]);

                        param->mla_pending = B_TRUE;
                        mutex_exit(&param->mla_mtx);

                        taskq_dispatch_ent(mlxp->mlx_async_tq,
                            mlxcx_link_state_task, param, 0, &param->mla_tqe);
                        break;
                case MLXCX_EVENT_PORT_MODULE:
                        mlxcx_report_module_error(mlxp, &ent->mleqe_port_mod);
                        break;
                default:
                        mlxcx_warn(mlxp, "unhandled event 0x%x on intr %d",
                            ent->mleqe_event_type, mleq->mleq_intr_index);
                }
        }

        mlxcx_arm_eq(mlxp, mleq);

done:
        mlxcx_intr_fini(mleq);
        return (DDI_INTR_CLAIMED);
}

static boolean_t
mlxcx_process_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq, mblk_t **mpp,
    size_t bytelim)
{
        mlxcx_work_queue_t *wq = mlcq->mlcq_wq;
        mlxcx_completionq_ent_t *cent;
        mblk_t *mp, *cmp, *nmp;
        mlxcx_buffer_t *buf;
        boolean_t found, added;
        size_t bytes = 0;
        uint_t rx_frames = 0;
        uint_t comp_cnt = 0;
        int64_t wqebbs, bufcnt;

        *mpp = NULL;

        if (!(mlcq->mlcq_state & MLXCX_CQ_ALLOC) ||
            !(mlcq->mlcq_state & MLXCX_CQ_CREATED) ||
            (mlcq->mlcq_state & MLXCX_CQ_DESTROYED) ||
            (mlcq->mlcq_state & MLXCX_CQ_TEARDOWN)) {
                return (B_FALSE);
        }

        nmp = cmp = mp = NULL;

        wqebbs = 0;
        bufcnt = 0;
        for (cent = mlxcx_cq_next(mlcq); cent != NULL;
            cent = mlxcx_cq_next(mlcq)) {
                /*
                 * Teardown and ring stop can atomic_or this flag
                 * into our state if they want us to stop early.
                 */
                if (mlcq->mlcq_state & MLXCX_CQ_TEARDOWN)
                        return (B_FALSE);

                comp_cnt++;
                if (cent->mlcqe_opcode == MLXCX_CQE_OP_REQ &&
                    cent->mlcqe_send_wqe_opcode == MLXCX_WQE_OP_NOP) {
                        /* NOP */
                        atomic_dec_64(&wq->mlwq_wqebb_used);
                        goto nextcq;
                }

lookagain:
                /*
                 * Generally the buffer we're looking for will be
                 * at the front of the list, so this loop won't
                 * need to look far.
                 */
                buf = list_head(&mlcq->mlcq_buffers);
                found = B_FALSE;
                while (buf != NULL) {
                        if ((buf->mlb_wqe_index & UINT16_MAX) ==
                            from_be16(cent->mlcqe_wqe_counter)) {
                                found = B_TRUE;
                                break;
                        }
                        buf = list_next(&mlcq->mlcq_buffers, buf);
                }

                if (!found) {
                        /*
                         * If there's any buffers waiting on the
                         * buffers_b list, then merge those into
                         * the main list and have another look.
                         *
                         * The wq enqueue routines push new buffers
                         * into buffers_b so that they can avoid
                         * taking the mlcq_mtx and blocking us for
                         * every single packet.
                         */
                        added = B_FALSE;
                        mutex_enter(&mlcq->mlcq_bufbmtx);
                        if (!list_is_empty(&mlcq->mlcq_buffers_b)) {
                                list_move_tail(&mlcq->mlcq_buffers,
                                    &mlcq->mlcq_buffers_b);
                                added = B_TRUE;
                        }
                        mutex_exit(&mlcq->mlcq_bufbmtx);
                        if (added)
                                goto lookagain;

                        /*
                         * This check could go just after the lookagain
                         * label, but it is a hot code path so we don't
                         * want to unnecessarily grab a lock and check
                         * a flag for a relatively rare event (the ring
                         * being stopped).
                         */
                        mutex_enter(&wq->mlwq_mtx);
                        if ((wq->mlwq_state & MLXCX_WQ_STARTED) == 0) {
                                mutex_exit(&wq->mlwq_mtx);
                                goto nextcq;
                        }
                        mutex_exit(&wq->mlwq_mtx);

                        buf = list_head(&mlcq->mlcq_buffers);
                        mlxcx_warn(mlxp, "got completion on CQ %x but "
                            "no buffer matching wqe found: %x (first "
                            "buffer counter = %x)", mlcq->mlcq_num,
                            from_be16(cent->mlcqe_wqe_counter),
                            buf == NULL ? UINT32_MAX :
                            buf->mlb_wqe_index);
                        mlxcx_fm_ereport(mlxp, DDI_FM_DEVICE_INVAL_STATE);
                        goto nextcq;
                }

                /*
                 * The buf is likely to be freed below, count this now.
                 */
                wqebbs += buf->mlb_wqebbs;

                list_remove(&mlcq->mlcq_buffers, buf);
                bufcnt++;

                switch (mlcq->mlcq_wq->mlwq_type) {
                case MLXCX_WQ_TYPE_SENDQ:
                        mlxcx_tx_completion(mlxp, mlcq, cent, buf);
                        break;
                case MLXCX_WQ_TYPE_RECVQ:
                        nmp = mlxcx_rx_completion(mlxp, mlcq, cent, buf);
                        bytes += from_be32(cent->mlcqe_byte_cnt);
                        if (nmp != NULL) {
                                if (cmp != NULL) {
                                        cmp->b_next = nmp;
                                        cmp = nmp;
                                } else {
                                        mp = cmp = nmp;
                                }

                                rx_frames++;
                        }
                        break;
                }

                /*
                 * Update the consumer index with what has been processed,
                 * followed by driver counters. It is important to tell the
                 * hardware first, otherwise when we throw more packets at
                 * it, it may get an overflow error.
                 * We do this whenever we've processed enough to bridge the
                 * high->low water mark.
                 */
                if (bufcnt > (MLXCX_CQ_LWM_GAP - MLXCX_CQ_HWM_GAP)) {
                        mlxcx_update_cqci(mlxp, mlcq);
                        /*
                         * Both these variables are incremented using
                         * atomics as they are modified in other code paths
                         * (Eg during tx) which hold different locks.
                         */
                        atomic_add_64(&mlcq->mlcq_bufcnt, -bufcnt);
                        atomic_add_64(&wq->mlwq_wqebb_used, -wqebbs);
                        wqebbs = 0;
                        bufcnt = 0;
                        comp_cnt = 0;
                }
nextcq:
                if (rx_frames > mlxp->mlx_props.mldp_rx_per_cq ||
                    (bytelim != 0 && bytes > bytelim))
                        break;
        }

        if (comp_cnt > 0) {
                mlxcx_update_cqci(mlxp, mlcq);
                atomic_add_64(&mlcq->mlcq_bufcnt, -bufcnt);
                atomic_add_64(&wq->mlwq_wqebb_used, -wqebbs);
        }

        *mpp = mp;
        return (B_TRUE);
}


mblk_t *
mlxcx_rx_poll(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq, size_t bytelim)
{
        mblk_t *mp = NULL;

        ASSERT(mutex_owned(&mlcq->mlcq_mtx));

        ASSERT(mlcq->mlcq_wq != NULL);
        ASSERT3U(mlcq->mlcq_wq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ);

        (void) mlxcx_process_cq(mlxp, mlcq, &mp, bytelim);

        return (mp);
}

static uint_t
mlxcx_intr_n(caddr_t arg, caddr_t arg2)
{
        mlxcx_t *mlxp = (mlxcx_t *)arg;
        mlxcx_event_queue_t *mleq = (mlxcx_event_queue_t *)arg2;
        mlxcx_eventq_ent_t *ent;
        mlxcx_completion_queue_t *mlcq, probe;
        mlxcx_work_queue_t *mlwq;
        mblk_t *mp = NULL;
        boolean_t tellmac = B_FALSE;

        if (!mlxcx_intr_ini(mlxp, mleq))
                return (DDI_INTR_CLAIMED);

        ent = mlxcx_eq_next(mleq);
        if (ent == NULL) {
                if (++mleq->mleq_badintrs > mlxcx_stuck_intr_count) {
                        mlxcx_fm_ereport(mlxp, DDI_FM_DEVICE_BADINT_LIMIT);
                        ddi_fm_service_impact(mlxp->mlx_dip, DDI_SERVICE_LOST);
                        (void) ddi_intr_disable(mlxp->mlx_intr_handles[
                            mleq->mleq_intr_index]);
                }
                goto done;
        }
        mleq->mleq_badintrs = 0;

        mutex_enter(&mleq->mleq_mtx);
        ASSERT(mleq->mleq_state & MLXCX_EQ_ARMED);
        mleq->mleq_state &= ~MLXCX_EQ_ARMED;
#if defined(DEBUG)
        /*
         * If we're still in mlxcx_attach and an intr_n fired, something really
         * weird is going on. This shouldn't happen in the absence of a driver
         * or firmware bug, so in the interests of minimizing branches in this
         * function this check is under DEBUG.
         */
        if (mleq->mleq_state & MLXCX_EQ_ATTACHING) {
                mutex_exit(&mleq->mleq_mtx);
                mlxcx_warn(mlxp, "intr_n (%u) fired during attach, disabling "
                    "vector", mleq->mleq_intr_index);
                mlxcx_fm_ereport(mlxp, DDI_FM_DEVICE_INVAL_STATE);
                ddi_fm_service_impact(mlxp->mlx_dip, DDI_SERVICE_LOST);
                (void) ddi_intr_disable(mlxp->mlx_intr_handles[
                    mleq->mleq_intr_index]);
                goto done;
        }
#endif
        mutex_exit(&mleq->mleq_mtx);

        for (; ent != NULL; ent = mlxcx_eq_next(mleq)) {
                ASSERT3U(ent->mleqe_event_type, ==, MLXCX_EVENT_COMPLETION);

                probe.mlcq_num =
                    from_be24(ent->mleqe_completion.mled_completion_cqn);
                mutex_enter(&mleq->mleq_mtx);
                mlcq = avl_find(&mleq->mleq_cqs, &probe, NULL);
                mutex_exit(&mleq->mleq_mtx);

                if (mlcq == NULL)
                        goto update_eq;

                mlwq = mlcq->mlcq_wq;

                /*
                 * mlcq_arm_mtx is used to avoid race conditions between
                 * this interrupt routine and the transition from polling
                 * back to interrupt mode. When exiting poll mode the
                 * CQ is likely to be un-armed, which means there will
                 * be no events for the CQ coming though here,
                 * consequently very low contention on mlcq_arm_mtx.
                 *
                 * mlcq_arm_mtx must be released before calls into mac
                 * layer in order to avoid deadlocks.
                 */
                mutex_enter(&mlcq->mlcq_arm_mtx);
                mlcq->mlcq_ec++;
                atomic_and_uint(&mlcq->mlcq_state, ~MLXCX_CQ_ARMED);

                if (mutex_tryenter(&mlcq->mlcq_mtx) == 0) {
                        /*
                         * If we failed to take the mutex because the
                         * polling function has it, just move on.
                         * We don't want to block other CQs behind
                         * this one.
                         */
                        if ((mlcq->mlcq_state & MLXCX_CQ_POLLING) != 0) {
                                mutex_exit(&mlcq->mlcq_arm_mtx);
                                goto update_eq;
                        }

                        /* Otherwise we will wait. */
                        mutex_enter(&mlcq->mlcq_mtx);
                }

                if ((mlcq->mlcq_state & MLXCX_CQ_POLLING) == 0 &&
                    mlxcx_process_cq(mlxp, mlcq, &mp, 0)) {
                        /*
                         * The ring is not in polling mode and we processed
                         * some completion queue entries.
                         */
                        if ((mlcq->mlcq_state & MLXCX_CQ_BLOCKED_MAC) != 0 &&
                            mlcq->mlcq_bufcnt < mlcq->mlcq_buflwm) {
                                atomic_and_uint(&mlcq->mlcq_state,
                                    ~MLXCX_CQ_BLOCKED_MAC);
                                tellmac = B_TRUE;
                        }

                        if ((mlwq->mlwq_state & MLXCX_WQ_BLOCKED_MAC) != 0 &&
                            mlwq->mlwq_wqebb_used < mlwq->mlwq_buflwm) {
                                atomic_and_uint(&mlwq->mlwq_state,
                                    ~MLXCX_WQ_BLOCKED_MAC);
                                tellmac = B_TRUE;
                        }

                        mlxcx_arm_cq(mlxp, mlcq);

                        mutex_exit(&mlcq->mlcq_mtx);
                        mutex_exit(&mlcq->mlcq_arm_mtx);

                        if (tellmac) {
                                mac_tx_ring_update(mlxp->mlx_mac_hdl,
                                    mlcq->mlcq_mac_hdl);
                                tellmac = B_FALSE;
                        }

                        if (mp != NULL) {
                                mac_rx_ring(mlxp->mlx_mac_hdl,
                                    mlcq->mlcq_mac_hdl, mp, mlcq->mlcq_mac_gen);
                        }
                } else {
                        mutex_exit(&mlcq->mlcq_mtx);
                        mutex_exit(&mlcq->mlcq_arm_mtx);
                }

update_eq:
                /*
                 * Updating the consumer counter for an EQ requires a write
                 * to the UAR, which is possibly expensive.
                 *
                 * Try to do it only often enough to stop us wrapping around.
                 */
                if ((mleq->mleq_cc & 0x7) == 0)
                        mlxcx_update_eq(mlxp, mleq);
        }

        mlxcx_arm_eq(mlxp, mleq);

done:
        mlxcx_intr_fini(mleq);
        return (DDI_INTR_CLAIMED);
}

boolean_t
mlxcx_intr_setup(mlxcx_t *mlxp)
{
        dev_info_t *dip = mlxp->mlx_dip;
        int ret;
        int nintrs = 0;
        int navail = 0;
        int types, i;
        mlxcx_eventq_type_t eqt = MLXCX_EQ_TYPE_ANY;

        ret = ddi_intr_get_supported_types(dip, &types);
        if (ret != DDI_SUCCESS) {
                mlxcx_warn(mlxp, "Failed to get supported interrupt types");
                return (B_FALSE);
        }

        if (!(types & DDI_INTR_TYPE_MSIX)) {
                mlxcx_warn(mlxp, "MSI-X interrupts not available, but mlxcx "
                    "requires MSI-X");
                return (B_FALSE);
        }

        ret = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_MSIX, &nintrs);
        if (ret != DDI_SUCCESS) {
                mlxcx_warn(mlxp, "Failed to get number of interrupts");
                return (B_FALSE);
        }
        if (nintrs < 2) {
                mlxcx_warn(mlxp, "%d MSI-X interrupts supported, but mlxcx "
                    "requires 2", nintrs);
                return (B_FALSE);
        }

        ret = ddi_intr_get_navail(dip, DDI_INTR_TYPE_MSIX, &navail);
        if (ret != DDI_SUCCESS) {
                mlxcx_warn(mlxp,
                    "Failed to get number of available interrupts");
                return (B_FALSE);
        }
        if (navail < 2) {
                mlxcx_warn(mlxp, "%d MSI-X interrupts available, but mlxcx "
                    "requires 2", navail);
                return (B_FALSE);
        }

        mlxp->mlx_intr_size = navail * sizeof (ddi_intr_handle_t);
        mlxp->mlx_intr_handles = kmem_alloc(mlxp->mlx_intr_size, KM_SLEEP);
        /*
         * Interrupts for Completion Queues events start from vector 1
         * up to available vectors. Vector 0 is used for asynchronous
         * events.
         */
        mlxp->mlx_intr_cq0 = 1;

        ret = ddi_intr_alloc(dip, mlxp->mlx_intr_handles, DDI_INTR_TYPE_MSIX,
            0, navail, &mlxp->mlx_intr_count, DDI_INTR_ALLOC_NORMAL);
        if (ret != DDI_SUCCESS) {
                mlxcx_warn(mlxp, "Failed to allocate %d interrupts", navail);
                mlxcx_intr_teardown(mlxp);
                return (B_FALSE);
        }
        if (mlxp->mlx_intr_count < mlxp->mlx_intr_cq0 + 1) {
                mlxcx_warn(mlxp, "%d MSI-X interrupts allocated, but mlxcx "
                    "requires %d", mlxp->mlx_intr_count,
                    mlxp->mlx_intr_cq0 + 1);
                mlxcx_intr_teardown(mlxp);
                return (B_FALSE);
        }
        mlxp->mlx_intr_type = DDI_INTR_TYPE_MSIX;

        ret = ddi_intr_get_pri(mlxp->mlx_intr_handles[0], &mlxp->mlx_intr_pri);
        if (ret != DDI_SUCCESS) {
                mlxcx_warn(mlxp, "Failed to get interrupt priority");
                mlxcx_intr_teardown(mlxp);
                return (B_FALSE);
        }

        /*
         * Set the interrupt priority for the asynchronous handler higher
         * than the ring handlers. Some operations which issue commands,
         * and thus rely on the async interrupt handler for posting
         * completion, do so with a CQ mutex held. The CQ mutex is also
         * acquired during ring processing, so if the ring processing vector
         * happens to be assigned to the same CPU as the async vector
         * it can hold off the async interrupt thread and lead to a deadlock.
         * By assigning a higher priority to the async vector, it will
         * always be dispatched.
         */
        mlxp->mlx_async_intr_pri = mlxp->mlx_intr_pri;
        if (mlxp->mlx_async_intr_pri < LOCK_LEVEL) {
                mlxp->mlx_async_intr_pri++;
        } else {
                mlxp->mlx_intr_pri--;
        }

        mlxp->mlx_eqs_size = mlxp->mlx_intr_count *
            sizeof (mlxcx_event_queue_t);
        mlxp->mlx_eqs = kmem_zalloc(mlxp->mlx_eqs_size, KM_SLEEP);

        /*
         * In the failure path, mlxcx_intr_teardown() expects this
         * mutex and avl tree to be init'ed - so do it now.
         */
        for (i = 0; i < mlxp->mlx_intr_count; ++i) {
                uint_t pri = (i == 0) ? mlxp->mlx_async_intr_pri :
                    mlxp->mlx_intr_pri;

                mutex_init(&mlxp->mlx_eqs[i].mleq_mtx, NULL, MUTEX_DRIVER,
                    DDI_INTR_PRI(pri));
                cv_init(&mlxp->mlx_eqs[i].mleq_cv, NULL, CV_DRIVER, NULL);

                if (i < mlxp->mlx_intr_cq0)
                        continue;

                avl_create(&mlxp->mlx_eqs[i].mleq_cqs, mlxcx_cq_compare,
                    sizeof (mlxcx_completion_queue_t),
                    offsetof(mlxcx_completion_queue_t, mlcq_eq_entry));
        }

        while (mlxp->mlx_async_intr_pri > DDI_INTR_PRI_MIN) {
                ret = ddi_intr_set_pri(mlxp->mlx_intr_handles[0],
                    mlxp->mlx_async_intr_pri);
                if (ret == DDI_SUCCESS)
                        break;
                mlxcx_note(mlxp,
                    "!Failed to set interrupt priority to %u for "
                    "async interrupt vector", mlxp->mlx_async_intr_pri);
                /*
                 * If it was not possible to set the IPL for the async
                 * interrupt to the desired value, then try a lower priority.
                 * Some PSMs can only accommodate a limited number of vectors
                 * at eatch priority level (or group of priority levels). Since
                 * the async priority must be set higher than the ring
                 * handlers, lower both. The ring handler priority is set
                 * below.
                 */
                mlxp->mlx_async_intr_pri--;
                mlxp->mlx_intr_pri--;
        }

        if (mlxp->mlx_async_intr_pri == DDI_INTR_PRI_MIN) {
                mlxcx_warn(mlxp, "Failed to find an interrupt priority for "
                    "async interrupt vector");
                mlxcx_intr_teardown(mlxp);
                return (B_FALSE);
        }

        ret = ddi_intr_add_handler(mlxp->mlx_intr_handles[0], mlxcx_intr_async,
            (caddr_t)mlxp, (caddr_t)&mlxp->mlx_eqs[0]);
        if (ret != DDI_SUCCESS) {
                mlxcx_warn(mlxp, "Failed to add async interrupt handler");
                mlxcx_intr_teardown(mlxp);
                return (B_FALSE);
        }

        /*
         * If we have enough interrupts, set their "type" fields so that we
         * avoid mixing RX and TX queues on the same EQs.
         */
        if (mlxp->mlx_intr_count >= 8) {
                eqt = MLXCX_EQ_TYPE_RX;
        }

        for (i = mlxp->mlx_intr_cq0; i < mlxp->mlx_intr_count; ++i) {
                mlxp->mlx_eqs[i].mleq_intr_index = i;

                mlxp->mlx_eqs[i].mleq_type = eqt;
                /*
                 * If eqt is still ANY, just leave it set to that
                 * (no else here).
                 */
                if (eqt == MLXCX_EQ_TYPE_RX) {
                        eqt = MLXCX_EQ_TYPE_TX;
                } else if (eqt == MLXCX_EQ_TYPE_TX) {
                        eqt = MLXCX_EQ_TYPE_RX;
                }

                while (mlxp->mlx_intr_pri >= DDI_INTR_PRI_MIN) {
                        ret = ddi_intr_set_pri(mlxp->mlx_intr_handles[i],
                            mlxp->mlx_intr_pri);
                        if (ret == DDI_SUCCESS)
                                break;
                        mlxcx_note(mlxp, "!Failed to set interrupt priority to "
                            "%u for interrupt vector %d",
                            mlxp->mlx_intr_pri, i);
                        mlxp->mlx_intr_pri--;
                }
                if (mlxp->mlx_intr_pri < DDI_INTR_PRI_MIN) {
                        mlxcx_warn(mlxp,
                            "Failed to find an interrupt priority for "
                            "interrupt vector %d", i);
                        mlxcx_intr_teardown(mlxp);
                        return (B_FALSE);
                }

                ret = ddi_intr_add_handler(mlxp->mlx_intr_handles[i],
                    mlxcx_intr_n, (caddr_t)mlxp, (caddr_t)&mlxp->mlx_eqs[i]);
                if (ret != DDI_SUCCESS) {
                        mlxcx_warn(mlxp, "Failed to add interrupt handler %d",
                            i);
                        mlxcx_intr_teardown(mlxp);
                        return (B_FALSE);
                }
        }

        return (B_TRUE);
}