/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * tavor_rsrc.c
 *    Tavor Resource Management Routines
 *
 *    Implements all the routines necessary for setup, teardown, and
 *    alloc/free of all Tavor resources, including those that are managed
 *    by Tavor hardware or which live in Tavor's direct attached DDR memory.
 */

#include <sys/sysmacros.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>
#include <sys/vmem.h>
#include <sys/bitmap.h>

#include <sys/ib/adapters/tavor/tavor.h>

/*
 * The following routines are used for initializing and destroying
 * the resource pools used by the Tavor resource allocation routines.
 * They consist of four classes of object:
 *
 * Mailboxes:  The "In" and "Out" mailbox types are used by the Tavor
 *    command interface routines.  Mailboxes are used to pass information
 *    back and forth to the Tavor firmware.  Either type of mailbox may
 *    be allocated from Tavor's direct attached DDR memory or from system
 *    memory (although currently all "In" mailboxes are in DDR and all "out"
 *    mailboxes come from system memory.
 *
 * HW entry objects:  These objects represent resources required by the Tavor
 *    hardware.  These objects include things like Queue Pair contexts (QPC),
 *    Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for
 *    supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory
 *    Protection Table entries (MPT), Memory Translation Table entries (MTT).
 *
 *    What these objects all have in common is that they are each required
 *    to come from DDR memory, they are always allocated from tables, and
 *    they are not to be directly accessed (read or written) by driver
 *    software.
 *    One notable exceptions to this rule are the Extended QP contexts (EQPC),
 *    and the UAR scratch area (UAR_SCR), both of which are not directly
 *    accessible through the Tavor resource allocation routines, but both
 *    of which are also required to reside in DDR memory and are not to be
 *    manipulated by driver software (they are privately managed by Tavor
 *    hardware).
 *    The other notable exceptions are the UAR pages (UAR_PG) which are
 *    allocated from the UAR address space rather than DDR, and the UD
 *    address vectors (UDAV) which are similar to the common object types
 *    with the major difference being that UDAVs _are_ directly read and
 *    written by driver software.
 *
 * SW handle objects: These objects represent resources required by Tavor
 *    driver software.  They are primarily software tracking structures,
 *    which are allocated from system memory (using kmem_cache).  Several of
 *    the objects have both a "constructor" and "destructor" method
 *    associated with them (see below).
 *
 * Protection Domain (PD) handle objects:  These objects are very much like
 *    a SW handle object with the notable difference that all PD handle
 *    objects have an actual Protection Domain number (PD) associated with
 *    them (and the PD number is allocated/managed through a separate
 *    vmem_arena specifically set aside for this purpose.
 */

static int tavor_rsrc_mbox_init(tavor_state_t *state,
    tavor_rsrc_mbox_info_t *info);
static void tavor_rsrc_mbox_fini(tavor_state_t *state,
    tavor_rsrc_mbox_info_t *info);

static int tavor_rsrc_hw_entries_init(tavor_state_t *state,
    tavor_rsrc_hw_entry_info_t *info);
static void tavor_rsrc_hw_entries_fini(tavor_state_t *state,
    tavor_rsrc_hw_entry_info_t *info);

static int tavor_rsrc_sw_handles_init(tavor_state_t *state,
    tavor_rsrc_sw_hdl_info_t *info);
static void tavor_rsrc_sw_handles_fini(tavor_state_t *state,
    tavor_rsrc_sw_hdl_info_t *info);

static int tavor_rsrc_pd_handles_init(tavor_state_t *state,
    tavor_rsrc_sw_hdl_info_t *info);
static void tavor_rsrc_pd_handles_fini(tavor_state_t *state,
    tavor_rsrc_sw_hdl_info_t *info);

/*
 * The following routines are used for allocating and freeing the specific
 * types of objects described above from their associated resource pools.
 */
static int tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info,
    uint_t num, tavor_rsrc_t *hdl);
static void tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info,
    tavor_rsrc_t *hdl);

static int tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info,
    uint_t num, uint_t num_align, ddi_acc_handle_t acc_handle,
    uint_t sleepflag, tavor_rsrc_t *hdl);
static void tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info,
    tavor_rsrc_t *hdl);

static int tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
    uint_t sleepflag, tavor_rsrc_t *hdl);
static void tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info,
    tavor_rsrc_t *hdl);

static int tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
    uint_t sleepflag, tavor_rsrc_t *hdl);
static void tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info,
    tavor_rsrc_t *hdl);

/*
 * The following routines are the constructors and destructors for several
 * of the SW handle type objects.  For certain types of SW handles objects
 * (all of which are implemented using kmem_cache), we need to do some
 * special field initialization (specifically, mutex_init/destroy).  These
 * routines enable that init and teardown.
 */
static int tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags);
static void tavor_rsrc_pdhdl_destructor(void *pd, void *state);
static int tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags);
static void tavor_rsrc_cqhdl_destructor(void *cq, void *state);
static int tavor_rsrc_qphdl_constructor(void *cq, void *priv, int flags);
static void tavor_rsrc_qphdl_destructor(void *cq, void *state);
static int tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags);
static void tavor_rsrc_srqhdl_destructor(void *srq, void *state);
static int tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags);
static void tavor_rsrc_refcnt_destructor(void *rc, void *state);
static int tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags);
static void tavor_rsrc_ahhdl_destructor(void *ah, void *state);
static int tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags);
static void tavor_rsrc_mrhdl_destructor(void *mr, void *state);

/*
 * Special routine to calculate and return the size of a MCG object based
 * on current driver configuration (specifically, the number of QP per MCG
 * that has been configured.
 */
static int tavor_rsrc_mcg_entry_get_size(tavor_state_t *state,
    uint_t *mcg_size_shift);


/*
 * tavor_rsrc_alloc()
 *
 *    Context: Can be called from interrupt or base context.
 *    The "sleepflag" parameter is used by all object allocators to
 *    determine whether to SLEEP for resources or not.
 */
int
tavor_rsrc_alloc(tavor_state_t *state, tavor_rsrc_type_t rsrc, uint_t num,
    uint_t sleepflag, tavor_rsrc_t **hdl)
{
        tavor_rsrc_pool_info_t  *rsrc_pool;
        tavor_rsrc_t            *tmp_rsrc_hdl;
        int                     flag, status = DDI_FAILURE;

        ASSERT(state != NULL);
        ASSERT(hdl != NULL);

        rsrc_pool = &state->ts_rsrc_hdl[rsrc];
        ASSERT(rsrc_pool != NULL);

        /*
         * Allocate space for the object used to track the resource handle
         */
        flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
        tmp_rsrc_hdl = (tavor_rsrc_t *)kmem_cache_alloc(state->ts_rsrc_cache,
            flag);
        if (tmp_rsrc_hdl == NULL) {
                return (DDI_FAILURE);
        }
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl))

        /*
         * Set rsrc_hdl type.  This is later used by the tavor_rsrc_free call
         * to know what type of resource is being freed.
         */
        tmp_rsrc_hdl->rsrc_type = rsrc;

        /*
         * Depending on resource type, call the appropriate alloc routine
         */
        switch (rsrc_pool->rsrc_type) {
        case TAVOR_IN_MBOX:
        case TAVOR_OUT_MBOX:
        case TAVOR_INTR_IN_MBOX:
        case TAVOR_INTR_OUT_MBOX:
                status = tavor_rsrc_mbox_alloc(rsrc_pool, num, tmp_rsrc_hdl);
                break;

        case TAVOR_QPC:
        case TAVOR_CQC:
        case TAVOR_SRQC:
        case TAVOR_EQC:
        case TAVOR_RDB:
                /*
                 * Because these objects are NOT accessed by Tavor driver
                 * software, we set the acc_handle parameter to zero.  But
                 * if they are allocated in multiples, we specify here that
                 * they must be aligned on a more restrictive boundary.
                 */
                status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 0,
                    sleepflag, tmp_rsrc_hdl);
                break;

        case TAVOR_MPT:
                /*
                 * Because these MPT objects are sometimes accessed by Tavor
                 * driver software (FMR), we set the acc_handle parameter.  But
                 * if they are allocated in multiples, we specify here that
                 * they must be aligned on a more restrictive boundary.
                 */
                status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num,
                    state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
                break;

        case TAVOR_MCG:
                /*
                 * Tavor MCG entries are also NOT accessed by Tavor driver
                 * software, but because MCG entries do not have the same
                 * alignnment restrictions we loosen the constraint here.
                 */
                status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 0,
                    sleepflag, tmp_rsrc_hdl);
                break;

        case TAVOR_MTT:
        case TAVOR_UDAV:
                /*
                 * Because MTT segments are among the few HW resources that
                 * may be allocated in odd numbers, we specify a less
                 * restrictive alignment than for the above resources.
                 *
                 * Also because both UDAV and MTT segment objects are read
                 * and/or written by Tavor driver software, we set the
                 * acc_handle parameter to point to the ddi_acc_handle_t for
                 * the Tavor DDR memory.
                 */
                status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
                    state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
                break;

        case TAVOR_UARPG:
                /*
                 * Because UAR pages are written by Tavor driver software (for
                 * doorbells), we set the acc_handle parameter to point to
                 * the ddi_acc_handle_t for the Tavor UAR memory.
                 */
                status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
                    state->ts_reg_uarhdl, sleepflag, tmp_rsrc_hdl);
                break;

        case TAVOR_MRHDL:
        case TAVOR_EQHDL:
        case TAVOR_CQHDL:
        case TAVOR_SRQHDL:
        case TAVOR_AHHDL:
        case TAVOR_QPHDL:
        case TAVOR_REFCNT:
                status = tavor_rsrc_swhdl_alloc(rsrc_pool, sleepflag,
                    tmp_rsrc_hdl);
                break;

        case TAVOR_PDHDL:
                status = tavor_rsrc_pdhdl_alloc(rsrc_pool, sleepflag,
                    tmp_rsrc_hdl);
                break;

        default:
                TAVOR_WARNING(state, "unexpected resource type in alloc");
                break;
        }

        /*
         * If the resource allocation failed, then free the special resource
         * tracking structure and return failure.  Otherwise return the
         * handle for the resource tracking structure.
         */
        if (status != DDI_SUCCESS) {
                kmem_cache_free(state->ts_rsrc_cache, tmp_rsrc_hdl);
                tmp_rsrc_hdl = NULL;
                return (DDI_FAILURE);
        } else {
                *hdl = tmp_rsrc_hdl;
                return (DDI_SUCCESS);
        }
}


/*
 * tavor_rsrc_free()
 *    Context: Can be called from interrupt or base context.
 */
void
tavor_rsrc_free(tavor_state_t *state, tavor_rsrc_t **hdl)
{
        tavor_rsrc_pool_info_t  *rsrc_pool;

        ASSERT(state != NULL);
        ASSERT(hdl != NULL);

        rsrc_pool = &state->ts_rsrc_hdl[(*hdl)->rsrc_type];
        ASSERT(rsrc_pool != NULL);

        /*
         * Depending on resource type, call the appropriate free routine
         */
        switch (rsrc_pool->rsrc_type) {
        case TAVOR_IN_MBOX:
        case TAVOR_OUT_MBOX:
        case TAVOR_INTR_IN_MBOX:
        case TAVOR_INTR_OUT_MBOX:
                tavor_rsrc_mbox_free(rsrc_pool, *hdl);
                break;

        case TAVOR_QPC:
        case TAVOR_CQC:
        case TAVOR_SRQC:
        case TAVOR_EQC:
        case TAVOR_RDB:
        case TAVOR_MCG:
        case TAVOR_MPT:
        case TAVOR_MTT:
        case TAVOR_UDAV:
        case TAVOR_UARPG:
                tavor_rsrc_hw_entry_free(rsrc_pool, *hdl);
                break;

        case TAVOR_MRHDL:
        case TAVOR_EQHDL:
        case TAVOR_CQHDL:
        case TAVOR_SRQHDL:
        case TAVOR_AHHDL:
        case TAVOR_QPHDL:
        case TAVOR_REFCNT:
                tavor_rsrc_swhdl_free(rsrc_pool, *hdl);
                break;

        case TAVOR_PDHDL:
                tavor_rsrc_pdhdl_free(rsrc_pool, *hdl);
                break;

        default:
                TAVOR_WARNING(state, "unexpected resource type in free");
                break;
        }

        /*
         * Free the special resource tracking structure, set the handle to
         * NULL, and return.
         */
        kmem_cache_free(state->ts_rsrc_cache, *hdl);
        *hdl = NULL;
}


/*
 * tavor_rsrc_init_phase1()
 *
 *    Completes the first phase of Tavor resource/configuration init.
 *    This involves creating the kmem_cache for the "tavor_rsrc_t"
 *    structs, allocating the space for the resource pool handles,
 *    and setting up the "Out" mailboxes.
 *
 *    When this function completes, the Tavor driver is ready to
 *    post the following commands which return information only in the
 *    "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER
 *    If any of these commands are to be posted at this time, they must be
 *    done so only when "spinning" (as the outstanding command list and
 *    EQ setup code has not yet run)
 *
 *    Context: Only called from attach() path context
 */
int
tavor_rsrc_init_phase1(tavor_state_t *state)
{
        tavor_rsrc_pool_info_t          *rsrc_pool;
        tavor_rsrc_mbox_info_t          mbox_info;
        tavor_rsrc_cleanup_level_t      cleanup;
        tavor_cfg_profile_t             *cfgprof;
        uint64_t                        num, size;
        int                             status;
        char                            *rsrc_name;

        ASSERT(state != NULL);

        /* This is where Phase 1 of resource initialization begins */
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL0;

        /* Build kmem cache name from Tavor instance */
        rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_RSRC_CACHE);

        /*
         * Create the kmem_cache for "tavor_rsrc_t" structures
         * (kmem_cache_create will SLEEP until successful)
         */
        state->ts_rsrc_cache = kmem_cache_create(rsrc_name,
            sizeof (tavor_rsrc_t), 0, NULL, NULL, NULL, NULL, NULL, 0);

        /*
         * Allocate an array of tavor_rsrc_pool_info_t's (used in all
         * subsequent resource allocations)
         */
        state->ts_rsrc_hdl = kmem_zalloc(TAVOR_NUM_RESOURCES *
            sizeof (tavor_rsrc_pool_info_t), KM_SLEEP);

        cfgprof = state->ts_cfg_profile;

        /*
         * Initialize the resource pool for "Out" mailboxes.  Notice that
         * the number of "Out" mailboxes, their size, and their location
         * (DDR or system memory) is configurable.  By default, however,
         * all "Out" mailboxes are located in system memory only (because
         * they are primarily read from and never written to)
         */
        num  =  ((uint64_t)1 << cfgprof->cp_log_num_outmbox);
        size =  ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
        rsrc_pool->rsrc_type      = TAVOR_OUT_MBOX;
        rsrc_pool->rsrc_loc       = TAVOR_IN_SYSMEM;
        rsrc_pool->rsrc_pool_size = (size * num);
        rsrc_pool->rsrc_shift     = cfgprof->cp_log_outmbox_size;
        rsrc_pool->rsrc_quantum   = size;
        rsrc_pool->rsrc_align     = TAVOR_MBOX_ALIGN;
        rsrc_pool->rsrc_state     = state;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_OUTMBOX_VMEM);
        mbox_info.mbi_num         = num;
        mbox_info.mbi_size        = size;
        mbox_info.mbi_rsrcpool    = rsrc_pool;
        mbox_info.mbi_rsrcname    = rsrc_name;
        status = tavor_rsrc_mbox_init(state, &mbox_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp1_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL1;

        /*
         * Initialize the Tavor "Out" mailbox list.  This step actually uses
         * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
         * "Out" mailboxes, bind them for DMA access, and arrange them into
         * an easily accessed fast-allocation mechanism (see tavor_cmd.c
         * for more details)
         */
        status = tavor_outmbox_list_init(state);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp1_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL2;

        /*
         * Initialize the resource pool for interrupt "Out" mailboxes.  Notice
         * that the number of interrupt "Out" mailboxes, their size, and their
         * location (DDR or system memory) is configurable.  By default,
         * however, all interrupt "Out" mailboxes are located in system memory
         * only (because they are primarily read from and never written to)
         */
        num  =  ((uint64_t)1 << cfgprof->cp_log_num_intr_outmbox);
        size =  ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_OUT_MBOX];
        rsrc_pool->rsrc_type      = TAVOR_INTR_OUT_MBOX;
        rsrc_pool->rsrc_loc       = TAVOR_IN_SYSMEM;
        rsrc_pool->rsrc_pool_size = (size * num);
        rsrc_pool->rsrc_shift     = cfgprof->cp_log_outmbox_size;
        rsrc_pool->rsrc_quantum   = size;
        rsrc_pool->rsrc_align     = TAVOR_MBOX_ALIGN;
        rsrc_pool->rsrc_state     = state;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_OUTMBOX_VMEM);
        mbox_info.mbi_num         = num;
        mbox_info.mbi_size        = size;
        mbox_info.mbi_rsrcpool    = rsrc_pool;
        mbox_info.mbi_rsrcname    = rsrc_name;
        status = tavor_rsrc_mbox_init(state, &mbox_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp1_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL3;

        /*
         * Initialize the Tavor "Out" mailbox list.  This step actually uses
         * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
         * "Out" mailboxes, bind them for DMA access, and arrange them into
         * an easily accessed fast-allocation mechanism (see tavor_cmd.c
         * for more details)
         */
        status = tavor_intr_outmbox_list_init(state);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp1_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;

        kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
        return (DDI_SUCCESS);

rsrcinitp1_fail:
        kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
        return (status);
}


/*
 * tavor_rsrc_init_phase2()
 *    Context: Only called from attach() path context
 */
int
tavor_rsrc_init_phase2(tavor_state_t *state)
{
        tavor_rsrc_sw_hdl_info_t        hdl_info;
        tavor_rsrc_hw_entry_info_t      entry_info;
        tavor_rsrc_mbox_info_t          mbox_info;
        tavor_rsrc_pool_info_t          *rsrc_pool;
        tavor_rsrc_cleanup_level_t      cleanup;
        tavor_cfg_profile_t             *cfgprof;
        uint64_t                        num, max, size, num_prealloc;
        uint64_t                        ddr_size, fw_size;
        uint_t                          mcg_size, mcg_size_shift;
        uint_t                          uarscr_size, mttsegment_sz;
        int                             status;
        char                            *rsrc_name;

        ASSERT(state != NULL);

        /* Phase 2 initialization begins where Phase 1 left off */
        cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;

        /*
         * Calculate the extent of the DDR size and portion of which that
         * is already reserved for Tavor firmware.  (Note: this information
         * is available because the QUERY_DDR and QUERY_FW commands have
         * been posted to Tavor firmware prior to calling this routine)
         */
        ddr_size = state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1;
        fw_size  = state->ts_fw.fw_endaddr - state->ts_fw.fw_baseaddr + 1;

        /* Build the DDR vmem arena name from Tavor instance */
        rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_VMEM);

        /*
         * Do a vmem_create for the entire DDR range (not including the
         * portion consumed by Tavor firmware).  This creates the vmem arena
         * from which all other DDR objects (specifically, tables of HW
         * entries) will be allocated.
         */
        state->ts_ddrvmem = vmem_create(rsrc_name,
            (void *)(uintptr_t)state->ts_ddr.ddr_baseaddr, (ddr_size - fw_size),
            sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP);
        if (state->ts_ddrvmem == NULL) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL5;

        /*
         * Initialize the resource pools for all objects that exist in
         * Tavor DDR memory.  This includes ("In") mailboxes, context tables
         * (QPC, CQC, EQC, etc...), and other miscellaneous HW objects.
         */
        cfgprof = state->ts_cfg_profile;

        /*
         * Initialize the resource pool for the MPT table entries.  Notice
         * that the number of MPTs is configurable.  The configured value must
         * be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.  Note also that a certain
         * number of MPTs must be set aside for Tavor firmware use.
         */
        num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
        max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
        num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt);
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT];
        rsrc_pool->rsrc_type      = TAVOR_MPT;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (TAVOR_MPT_SIZE * num);
        rsrc_pool->rsrc_shift     = TAVOR_MPT_SIZE_SHIFT;
        rsrc_pool->rsrc_quantum   = TAVOR_MPT_SIZE;
        rsrc_pool->rsrc_align     = (TAVOR_MPT_SIZE * num);
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = NULL;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MPT_VMEM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;
        status = tavor_rsrc_hw_entries_init(state, &entry_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL6;

        /*
         * Initialize the resource pool for the MTT table entries.  Notice
         * that the number of MTTs is configurable.  The configured value must
         * be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.  Note also that a certain
         * number of MTT segments must be set aside for Tavor firmware use.
         */
        mttsegment_sz = (TAVOR_MTTSEG_SIZE << TAVOR_MTT_SIZE_SHIFT);
        num = ((uint64_t)1 << cfgprof->cp_log_num_mttseg);
        max = ((uint64_t)1 << state->ts_devlim.log_max_mttseg);
        num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mttseg);
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
        rsrc_pool->rsrc_type      = TAVOR_MTT;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (TAVOR_MTT_SIZE * num);
        rsrc_pool->rsrc_shift     = TAVOR_MTT_SIZE_SHIFT;
        rsrc_pool->rsrc_quantum   = mttsegment_sz;
        rsrc_pool->rsrc_align     = (TAVOR_MTT_SIZE * num);
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = NULL;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MTT_VMEM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;
        status = tavor_rsrc_hw_entries_init(state, &entry_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL7;

        /*
         * Initialize the resource pool for the QPC table entries.  Notice
         * that the number of QPs is configurable.  The configured value must
         * be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.  Note also that a certain
         * number of QP contexts must be set aside for Tavor firmware use.
         */
        num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
        max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
        num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_qp);
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC];
        rsrc_pool->rsrc_type      = TAVOR_QPC;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (TAVOR_QPC_SIZE * num);
        rsrc_pool->rsrc_shift     = TAVOR_QPC_SIZE_SHIFT;
        rsrc_pool->rsrc_quantum   = TAVOR_QPC_SIZE;
        rsrc_pool->rsrc_align     = (TAVOR_QPC_SIZE * num);
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = NULL;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_QPC_VMEM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;
        status = tavor_rsrc_hw_entries_init(state, &entry_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL8;

        /*
         * Initialize the resource pool for the RDB table entries.  Notice
         * that the number of RDBs is configurable.  The configured value must
         * be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.
         */
        num = ((uint64_t)1 << cfgprof->cp_log_num_rdb);
        max = ((uint64_t)1 << state->ts_devlim.log_max_ra_glob);
        num_prealloc = 0;
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB];
        rsrc_pool->rsrc_type      = TAVOR_RDB;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (TAVOR_RDB_SIZE * num);
        rsrc_pool->rsrc_shift     = TAVOR_RDB_SIZE_SHIFT;
        rsrc_pool->rsrc_quantum   = TAVOR_RDB_SIZE;
        rsrc_pool->rsrc_align     = (TAVOR_RDB_SIZE * num);
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = NULL;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_RDB_VMEM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;
        status = tavor_rsrc_hw_entries_init(state, &entry_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL9;

        /*
         * Initialize the resource pool for the CQC table entries.  Notice
         * that the number of CQs is configurable.  The configured value must
         * be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.  Note also that a certain
         * number of CQ contexts must be set aside for Tavor firmware use.
         */
        num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
        max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
        num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_cq);
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC];
        rsrc_pool->rsrc_type      = TAVOR_CQC;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (TAVOR_CQC_SIZE * num);
        rsrc_pool->rsrc_shift     = TAVOR_CQC_SIZE_SHIFT;
        rsrc_pool->rsrc_quantum   = TAVOR_CQC_SIZE;
        rsrc_pool->rsrc_align     = (TAVOR_CQC_SIZE * num);
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = NULL;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_CQC_VMEM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;
        status = tavor_rsrc_hw_entries_init(state, &entry_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL10;

        /*
         * Initialize the resource pool for the Extended QPC table entries.
         * Notice that the number of EQPCs must be the same as the number
         * of QP contexts.  So the initialization is constructed in a
         * similar way as above (for TAVOR_QPC).  One notable difference
         * here, however, is that by setting the rsrc_quantum field to
         * zero (indicating a zero-sized object) we indicate that the
         * object is not allocatable.  The EQPC table is, in fact, managed
         * internally by the hardware and it is, therefore, unnecessary to
         * initialize an additional vmem_arena for this type of object.
         */
        num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
        max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
        num_prealloc = 0;
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC];
        rsrc_pool->rsrc_type      = TAVOR_EQPC;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (TAVOR_EQPC_SIZE * num);
        rsrc_pool->rsrc_shift     = 0;
        rsrc_pool->rsrc_quantum   = 0;
        rsrc_pool->rsrc_align     = TAVOR_EQPC_SIZE;
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = NULL;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQPC_VMEM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;
        status = tavor_rsrc_hw_entries_init(state, &entry_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL11;

        /*
         * Initialize the resource pool for the UD address vector table
         * entries.  Notice that the number of UDAVs is configurable.  The
         * configured value must be less that the maximum value (obtained
         * from the QUERY_DEV_LIM command) or the initialization will fail.
         */
        num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
        max = ((uint64_t)1 << state->ts_devlim.log_max_av);
        num_prealloc = 0;
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UDAV];
        rsrc_pool->rsrc_type      = TAVOR_UDAV;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (TAVOR_UDAV_SIZE * num);
        rsrc_pool->rsrc_shift     = TAVOR_UDAV_SIZE_SHIFT;
        rsrc_pool->rsrc_quantum   = TAVOR_UDAV_SIZE;
        rsrc_pool->rsrc_align     = TAVOR_UDAV_SIZE;
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = NULL;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UDAV_VMEM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;
        status = tavor_rsrc_hw_entries_init(state, &entry_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL12;

        /*
         * Initialize the resource pool for the UAR scratch table entries.
         * Notice that the number of UARSCRs is configurable.  The configured
         * value must be less that the maximum value (obtained from the
         * QUERY_DEV_LIM command) or the initialization will fail.
         * Like the EQPCs above, UARSCR objects are not allocatable.  The
         * UARSCR table is also managed internally by the hardware and it
         * is, therefore, unnecessary to initialize an additional vmem_arena
         * for this type of object.  We indicate this by setting the
         * rsrc_quantum field to zero (indicating a zero-sized object).
         */
        uarscr_size = state->ts_devlim.uarscr_entry_sz;
        num = ((uint64_t)1 << cfgprof->cp_log_num_uar);
        max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
            PAGESHIFT));
        num_prealloc = 0;
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
        rsrc_pool->rsrc_type      = TAVOR_UAR_SCR;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (uarscr_size * num);
        rsrc_pool->rsrc_shift     = 0;
        rsrc_pool->rsrc_quantum   = 0;
        rsrc_pool->rsrc_align     = uarscr_size;
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = NULL;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UARSCR_VMEM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;
        status = tavor_rsrc_hw_entries_init(state, &entry_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL13;

        /*
         * Initialize the resource pool for the SRQC table entries.  Notice
         * that the number of SRQs is configurable.  The configured value must
         * be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.  Note also that a certain
         * number of SRQ contexts must be set aside for Tavor firmware use.
         *
         * Note: We only allocate these resources if SRQ is enabled in the
         * config profile; see below.
         */
        num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
        max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
        num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_srq);

        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC];
        rsrc_pool->rsrc_type      = TAVOR_SRQC;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (TAVOR_SRQC_SIZE * num);
        rsrc_pool->rsrc_shift     = TAVOR_SRQC_SIZE_SHIFT;
        rsrc_pool->rsrc_quantum   = TAVOR_SRQC_SIZE;
        rsrc_pool->rsrc_align     = (TAVOR_SRQC_SIZE * num);
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = NULL;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_SRQC_VMEM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;

        /*
         * SRQ support is configurable.  Only if SRQ is enabled (the default)
         * do we actually try to configure these resources.  Otherwise, we
         * simply set the cleanup level and continue on to the next resource
         */
        if (state->ts_cfg_profile->cp_srq_enable != 0) {
                status = tavor_rsrc_hw_entries_init(state, &entry_info);
                if (status != DDI_SUCCESS) {
                        tavor_rsrc_fini(state, cleanup);
                        goto rsrcinitp2_fail;
                }
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL14;

        /*
         * Initialize the resource pool for "In" mailboxes.  Notice that
         * the number of "In" mailboxes, their size, and their location
         * (DDR or system memory) is configurable.  By default, however,
         * all "In" mailboxes are located in system memory only (because
         * they are primarily written to and rarely read from)
         */
        num  =  ((uint64_t)1 << cfgprof->cp_log_num_inmbox);
        size =  ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
        rsrc_pool->rsrc_type      = TAVOR_IN_MBOX;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (size * num);
        rsrc_pool->rsrc_shift     = cfgprof->cp_log_inmbox_size;
        rsrc_pool->rsrc_quantum   = size;
        rsrc_pool->rsrc_align     = TAVOR_MBOX_ALIGN;
        rsrc_pool->rsrc_state     = state;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INMBOX_VMEM);
        mbox_info.mbi_num         = num;
        mbox_info.mbi_size        = size;
        mbox_info.mbi_rsrcpool    = rsrc_pool;
        mbox_info.mbi_rsrcname    = rsrc_name;
        status = tavor_rsrc_mbox_init(state, &mbox_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL15;

        /*
         * Initialize the Tavor "In" mailbox list.  This step actually uses
         * the tavor_rsrc_alloc() for TAVOR_IN_MBOX to preallocate the
         * "In" mailboxes, bind them for DMA access, and arrange them into
         * an easily accessed fast-allocation mechanism (see tavor_cmd.c
         * for more details)
         */
        status = tavor_inmbox_list_init(state);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL16;

        /*
         * Initialize the resource pool for interrupt "In" mailboxes.  Notice
         * that the number of interrupt "In" mailboxes, their size, and their
         * location (DDR or system memory) is configurable.  By default,
         * however, all interrupt "In" mailboxes are located in system memory
         * only (because they are primarily written to and rarely read from)
         */
        num  =  ((uint64_t)1 << cfgprof->cp_log_num_intr_inmbox);
        size =  ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_IN_MBOX];
        rsrc_pool->rsrc_type      = TAVOR_INTR_IN_MBOX;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (size * num);
        rsrc_pool->rsrc_shift     = cfgprof->cp_log_inmbox_size;
        rsrc_pool->rsrc_quantum   = size;
        rsrc_pool->rsrc_align     = TAVOR_MBOX_ALIGN;
        rsrc_pool->rsrc_state     = state;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_INMBOX_VMEM);
        mbox_info.mbi_num         = num;
        mbox_info.mbi_size        = size;
        mbox_info.mbi_rsrcpool    = rsrc_pool;
        mbox_info.mbi_rsrcname    = rsrc_name;
        status = tavor_rsrc_mbox_init(state, &mbox_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL17;

        /*
         * Initialize the Tavor interrupt "In" mailbox list.  This step
         * actually uses the tavor_rsrc_alloc() for TAVOR_IN_MBOX to
         * preallocate the interrupt "In" mailboxes, bind them for DMA access,
         * and arrange them into an easily accessed fast-allocation mechanism
         * (see tavor_cmd.c for more details)
         */
        status = tavor_intr_inmbox_list_init(state);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL18;

        /*
         * Initialize the Tavor command handling interfaces.  This step
         * sets up the outstanding command tracking mechanism for easy access
         * and fast allocation (see tavor_cmd.c for more details).
         */
        status = tavor_outstanding_cmdlist_init(state);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL19;

        /*
         * Calculate (and validate) the size of Multicast Group (MCG) entries
         */
        status = tavor_rsrc_mcg_entry_get_size(state, &mcg_size_shift);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        mcg_size = TAVOR_MCGMEM_SZ(state);

        /*
         * Initialize the resource pool for the MCG table entries.  Notice
         * that the number of MCGs is configurable.  The configured value must
         * be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.  Note also that a certain
         * number of MCGs must be set aside for Tavor firmware use (they
         * correspond to the number of MCGs used by the internal hash
         * function.
         */
        num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
        max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
        num_prealloc = ((uint64_t)1 << cfgprof->cp_log_num_mcg_hash);
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG];
        rsrc_pool->rsrc_type      = TAVOR_MCG;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (mcg_size * num);
        rsrc_pool->rsrc_shift     = mcg_size_shift;
        rsrc_pool->rsrc_quantum   = mcg_size;
        rsrc_pool->rsrc_align     = mcg_size;
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = NULL;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MCG_VMEM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;
        status = tavor_rsrc_hw_entries_init(state, &entry_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL20;

        /*
         * Initialize the resource pool for the EQC table entries.  Notice
         * that the number of EQs is hardcoded.  The hardcoded value should
         * be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.
         */
        num = TAVOR_NUM_EQ;
        max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
        num_prealloc = 0;
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC];
        rsrc_pool->rsrc_type      = TAVOR_EQC;
        rsrc_pool->rsrc_loc       = TAVOR_IN_DDR;
        rsrc_pool->rsrc_pool_size = (TAVOR_EQC_SIZE * num);
        rsrc_pool->rsrc_shift     = TAVOR_EQC_SIZE_SHIFT;
        rsrc_pool->rsrc_quantum   = TAVOR_EQC_SIZE;
        rsrc_pool->rsrc_align     = (TAVOR_EQC_SIZE * num);
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = NULL;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQC_VMEM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;
        status = tavor_rsrc_hw_entries_init(state, &entry_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL21;

        /*
         * Initialize the resource pools for all objects that exist in
         * system memory.  This includes PD handles, MR handle, EQ handles,
         * QP handles, etc.  These objects are almost entirely managed using
         * kmem_cache routines.  (See comment above for more detail)
         */

        /*
         * Initialize the resource pool for the PD handles.  Notice
         * that the number of PDHDLs is configurable.  The configured value
         * must be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.  Note also that the PD
         * handle has constructor and destructor methods associated with it.
         */
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_PDHDL];
        rsrc_pool->rsrc_type     = TAVOR_PDHDL;
        rsrc_pool->rsrc_loc      = TAVOR_IN_SYSMEM;
        rsrc_pool->rsrc_quantum  = sizeof (struct tavor_sw_pd_s);
        rsrc_pool->rsrc_state    = state;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_PDHDL_CACHE);
        hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_pd);
        hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_pd);
        hdl_info.swi_rsrcpool    = rsrc_pool;
        hdl_info.swi_constructor = tavor_rsrc_pdhdl_constructor;
        hdl_info.swi_destructor  = tavor_rsrc_pdhdl_destructor;
        hdl_info.swi_rsrcname    = rsrc_name;
        hdl_info.swi_flags       = TAVOR_SWHDL_KMEMCACHE_INIT;
        status = tavor_rsrc_pd_handles_init(state, &hdl_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL22;

        /*
         * Initialize the resource pool for the MR handles.  Notice
         * that the number of MRHDLs is configurable.  The configured value
         * must be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.
         */
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MRHDL];
        rsrc_pool->rsrc_type     = TAVOR_MRHDL;
        rsrc_pool->rsrc_loc      = TAVOR_IN_SYSMEM;
        rsrc_pool->rsrc_quantum  = sizeof (struct tavor_sw_mr_s);
        rsrc_pool->rsrc_state    = state;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_MRHDL_CACHE);
        hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
        hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
        hdl_info.swi_rsrcpool    = rsrc_pool;
        hdl_info.swi_constructor = tavor_rsrc_mrhdl_constructor;
        hdl_info.swi_destructor  = tavor_rsrc_mrhdl_destructor;
        hdl_info.swi_rsrcname    = rsrc_name;
        hdl_info.swi_flags       = TAVOR_SWHDL_KMEMCACHE_INIT;
        status = tavor_rsrc_sw_handles_init(state, &hdl_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL23;

        /*
         * Initialize the resource pool for the EQ handles.  Notice
         * that the number of EQHDLs is hardcoded.  The hardcoded value
         * should be less that the maximum value (obtained from the
         * QUERY_DEV_LIM command) or the initialization will fail.
         */
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQHDL];
        rsrc_pool->rsrc_type     = TAVOR_EQHDL;
        rsrc_pool->rsrc_loc      = TAVOR_IN_SYSMEM;
        rsrc_pool->rsrc_quantum  = sizeof (struct tavor_sw_eq_s);
        rsrc_pool->rsrc_state    = state;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_EQHDL_CACHE);
        hdl_info.swi_num = TAVOR_NUM_EQ;
        hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
        hdl_info.swi_rsrcpool    = rsrc_pool;
        hdl_info.swi_constructor = NULL;
        hdl_info.swi_destructor  = NULL;
        hdl_info.swi_rsrcname    = rsrc_name;
        hdl_info.swi_flags       = TAVOR_SWHDL_KMEMCACHE_INIT;
        status = tavor_rsrc_sw_handles_init(state, &hdl_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL24;

        /*
         * Initialize the resource pool for the CQ handles.  Notice
         * that the number of CQHDLs is configurable.  The configured value
         * must be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.  Note also that the CQ
         * handle has constructor and destructor methods associated with it.
         */
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQHDL];
        rsrc_pool->rsrc_type     = TAVOR_CQHDL;
        rsrc_pool->rsrc_loc      = TAVOR_IN_SYSMEM;
        rsrc_pool->rsrc_quantum  = sizeof (struct tavor_sw_cq_s);
        rsrc_pool->rsrc_state    = state;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_CQHDL_CACHE);
        hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
        hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
        hdl_info.swi_rsrcpool    = rsrc_pool;
        hdl_info.swi_constructor = tavor_rsrc_cqhdl_constructor;
        hdl_info.swi_destructor  = tavor_rsrc_cqhdl_destructor;
        hdl_info.swi_rsrcname    = rsrc_name;
        hdl_info.swi_flags       = (TAVOR_SWHDL_KMEMCACHE_INIT |
            TAVOR_SWHDL_TABLE_INIT);
        hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
        status = tavor_rsrc_sw_handles_init(state, &hdl_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }

        /*
         * Save away the pointer to the central list of CQ handle pointers
         * This this is used as a mechanism to enable fast CQnumber-to-CQhandle
         * lookup during EQ event processing.  The table is a list of
         * tavor_cqhdl_t allocated by the above routine because of the
         * TAVOR_SWHDL_TABLE_INIT flag.  The table has as many tavor_cqhdl_t
         * as the number of CQs.
         */
        state->ts_cqhdl = hdl_info.swi_table_ptr;
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL25;

        /*
         * Initialize the resource pool for the SRQ handles.  Notice
         * that the number of SRQHDLs is configurable.  The configured value
         * must be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.  Note also that the SRQ
         * handle has constructor and destructor methods associated with it.
         *
         * Note: We only allocate these resources if SRQ is enabled in the
         * config profile; see below.
         */
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQHDL];
        rsrc_pool->rsrc_type     = TAVOR_SRQHDL;
        rsrc_pool->rsrc_loc      = TAVOR_IN_SYSMEM;
        rsrc_pool->rsrc_quantum  = sizeof (struct tavor_sw_srq_s);
        rsrc_pool->rsrc_state    = state;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_SRQHDL_CACHE);
        hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
        hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
        hdl_info.swi_rsrcpool    = rsrc_pool;
        hdl_info.swi_constructor = tavor_rsrc_srqhdl_constructor;
        hdl_info.swi_destructor  = tavor_rsrc_srqhdl_destructor;
        hdl_info.swi_rsrcname    = rsrc_name;
        hdl_info.swi_flags       = (TAVOR_SWHDL_KMEMCACHE_INIT |
            TAVOR_SWHDL_TABLE_INIT);
        hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);

        /*
         * SRQ support is configurable.  Only if SRQ is enabled (the default)
         * do we actually try to configure these resources.  Otherwise, we
         * simply set the cleanup level and continue on to the next resource
         */
        if (state->ts_cfg_profile->cp_srq_enable != 0) {
                status = tavor_rsrc_sw_handles_init(state, &hdl_info);
                if (status != DDI_SUCCESS) {
                        tavor_rsrc_fini(state, cleanup);
                        goto rsrcinitp2_fail;
                }

                /*
                 * Save away the pointer to the central list of SRQ handle
                 * pointers This this is used as a mechanism to enable fast
                 * SRQnumber-to-SRQhandle lookup.  The table is a list of
                 * tavor_srqhdl_t allocated by the above routine because of the
                 * TAVOR_SWHDL_TABLE_INIT flag.  The table has as many
                 * tavor_srqhdl_t as the number of SRQs.
                 */
                state->ts_srqhdl = hdl_info.swi_table_ptr;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL26;

        /*
         * Initialize the resource pool for the address handles.  Notice
         * that the number of AHHDLs is configurable.  The configured value
         * must be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.
         */
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_AHHDL];
        rsrc_pool->rsrc_type     = TAVOR_AHHDL;
        rsrc_pool->rsrc_loc      = TAVOR_IN_SYSMEM;
        rsrc_pool->rsrc_quantum  = sizeof (struct tavor_sw_ah_s);
        rsrc_pool->rsrc_state    = state;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_AHHDL_CACHE);
        hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
        hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_av);
        hdl_info.swi_rsrcpool    = rsrc_pool;
        hdl_info.swi_constructor = tavor_rsrc_ahhdl_constructor;
        hdl_info.swi_destructor  = tavor_rsrc_ahhdl_destructor;
        hdl_info.swi_rsrcname    = rsrc_name;
        hdl_info.swi_flags       = TAVOR_SWHDL_KMEMCACHE_INIT;
        status = tavor_rsrc_sw_handles_init(state, &hdl_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL27;

        /*
         * Initialize the resource pool for the QP handles.  Notice
         * that the number of QPHDLs is configurable.  The configured value
         * must be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.  Note also that the QP
         * handle has constructor and destructor methods associated with it.
         */
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPHDL];
        rsrc_pool->rsrc_type     = TAVOR_QPHDL;
        rsrc_pool->rsrc_loc      = TAVOR_IN_SYSMEM;
        rsrc_pool->rsrc_quantum  = sizeof (struct tavor_sw_qp_s);
        rsrc_pool->rsrc_state    = state;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_QPHDL_CACHE);
        hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
        hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
        hdl_info.swi_rsrcpool    = rsrc_pool;
        hdl_info.swi_constructor = tavor_rsrc_qphdl_constructor;
        hdl_info.swi_destructor  = tavor_rsrc_qphdl_destructor;
        hdl_info.swi_rsrcname    = rsrc_name;
        hdl_info.swi_flags       = (TAVOR_SWHDL_KMEMCACHE_INIT |
            TAVOR_SWHDL_TABLE_INIT);
        hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
        status = tavor_rsrc_sw_handles_init(state, &hdl_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }

        /*
         * Save away the pointer to the central list of QP handle pointers
         * This this is used as a mechanism to enable fast QPnumber-to-QPhandle
         * lookup during CQ event processing.  The table is a list of
         * tavor_qphdl_t allocated by the above routine because of the
         * TAVOR_SWHDL_TABLE_INIT flag.  The table has as many tavor_qphdl_t
         * as the number of QPs.
         */
        state->ts_qphdl = hdl_info.swi_table_ptr;
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL28;

        /*
         * Initialize the resource pool for the reference count handles.
         * Notice that the number of REFCNTs is configurable, but it's value
         * is set to the number of MPTs.  Since REFCNTs are used to support
         * shared memory regions, it is possible that we might require as
         * one REFCNT for every MPT.
         */
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_REFCNT];
        rsrc_pool->rsrc_type     = TAVOR_REFCNT;
        rsrc_pool->rsrc_loc      = TAVOR_IN_SYSMEM;
        rsrc_pool->rsrc_quantum  = sizeof (tavor_sw_refcnt_t);
        rsrc_pool->rsrc_state    = state;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_REFCNT_CACHE);
        hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
        hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
        hdl_info.swi_rsrcpool    = rsrc_pool;
        hdl_info.swi_constructor = tavor_rsrc_refcnt_constructor;
        hdl_info.swi_destructor  = tavor_rsrc_refcnt_destructor;
        hdl_info.swi_rsrcname    = rsrc_name;
        hdl_info.swi_flags       = TAVOR_SWHDL_KMEMCACHE_INIT;
        status = tavor_rsrc_sw_handles_init(state, &hdl_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL29;

        /*
         * Initialize the resource pool for the MCG handles.  Notice that for
         * these MCG handles, we are allocating a table of structures (used to
         * keep track of the MCG entries that are being written to hardware
         * and to speed up multicast attach/detach operations).
         */
        hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
        hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
        hdl_info.swi_flags = TAVOR_SWHDL_TABLE_INIT;
        hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
        status = tavor_rsrc_sw_handles_init(state, &hdl_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        state->ts_mcghdl = hdl_info.swi_table_ptr;
        cleanup = TAVOR_RSRC_CLEANUP_LEVEL30;

        /*
         * Initialize the resource pools for all objects that exist in
         * UAR memory.  The only objects that are allocated from UAR memory
         * are the UAR pages which are used for holding Tavor hardware's
         * doorbell registers.
         */

        /*
         * Initialize the resource pool for the UAR pages.  Notice
         * that the number of UARPGs is configurable.  The configured value
         * must be less that the maximum value (obtained from the QUERY_DEV_LIM
         * command) or the initialization will fail.  Note also that by
         * specifying the rsrc_start parameter in advance, we direct the
         * initialization routine not to attempt to allocated space from the
         * Tavor DDR vmem_arena.
         */
        num  = ((uint64_t)1 << cfgprof->cp_log_num_uar);
        max  = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
            PAGESHIFT));
        num_prealloc = 0;
        rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UARPG];
        rsrc_pool->rsrc_type      = TAVOR_UARPG;
        rsrc_pool->rsrc_loc       = TAVOR_IN_UAR;
        rsrc_pool->rsrc_pool_size = (num << PAGESHIFT);
        rsrc_pool->rsrc_shift     = PAGESHIFT;
        rsrc_pool->rsrc_quantum   = PAGESIZE;
        rsrc_pool->rsrc_align     = PAGESIZE;
        rsrc_pool->rsrc_state     = state;
        rsrc_pool->rsrc_start     = (void *)state->ts_reg_uar_baseaddr;
        TAVOR_RSRC_NAME(rsrc_name, TAVOR_UAR_VMEM);
        entry_info.hwi_num        = num;
        entry_info.hwi_max        = max;
        entry_info.hwi_prealloc   = num_prealloc;
        entry_info.hwi_rsrcpool   = rsrc_pool;
        entry_info.hwi_rsrcname   = rsrc_name;
        status = tavor_rsrc_hw_entries_init(state, &entry_info);
        if (status != DDI_SUCCESS) {
                tavor_rsrc_fini(state, cleanup);
                goto rsrcinitp2_fail;
        }
        cleanup = TAVOR_RSRC_CLEANUP_ALL;

        kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
        return (DDI_SUCCESS);

rsrcinitp2_fail:
        kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
        return (status);
}


/*
 * tavor_rsrc_fini()
 *    Context: Only called from attach() and/or detach() path contexts
 */
void
tavor_rsrc_fini(tavor_state_t *state, tavor_rsrc_cleanup_level_t clean)
{
        tavor_rsrc_sw_hdl_info_t        hdl_info;
        tavor_rsrc_hw_entry_info_t      entry_info;
        tavor_rsrc_mbox_info_t          mbox_info;
        tavor_cfg_profile_t             *cfgprof;

        ASSERT(state != NULL);

        cfgprof = state->ts_cfg_profile;

        switch (clean) {
        /*
         * If we add more resources that need to be cleaned up here, we should
         * ensure that TAVOR_RSRC_CLEANUP_ALL is still the first entry (i.e.
         * corresponds to the last resource allocated).
         */
        case TAVOR_RSRC_CLEANUP_ALL:
                /* Cleanup the UAR page resource pool */
                entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UARPG];
                tavor_rsrc_hw_entries_fini(state, &entry_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL30:
                /* Cleanup the central MCG handle pointers list */
                hdl_info.swi_rsrcpool  = NULL;
                hdl_info.swi_table_ptr = state->ts_mcghdl;
                hdl_info.swi_num =
                    ((uint64_t)1 << cfgprof->cp_log_num_mcg);
                hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
                tavor_rsrc_sw_handles_fini(state, &hdl_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL29:
                /* Cleanup the reference count resource pool */
                hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_REFCNT];
                hdl_info.swi_table_ptr = NULL;
                tavor_rsrc_sw_handles_fini(state, &hdl_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL28:
                /* Cleanup the QP handle resource pool */
                hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_QPHDL];
                hdl_info.swi_table_ptr = state->ts_qphdl;
                hdl_info.swi_num =
                    ((uint64_t)1 << cfgprof->cp_log_num_qp);
                hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
                tavor_rsrc_sw_handles_fini(state, &hdl_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL27:
                /* Cleanup the address handle resource pool */
                hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_AHHDL];
                hdl_info.swi_table_ptr = NULL;
                tavor_rsrc_sw_handles_fini(state, &hdl_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL26:
                /*
                 * Cleanup the SRQ handle resource pool.
                 *
                 * Note: We only clean up if SRQ is enabled.  Otherwise we
                 * simply fallthrough to the next resource cleanup.
                 */
                if (state->ts_cfg_profile->cp_srq_enable != 0) {
                        hdl_info.swi_rsrcpool  =
                            &state->ts_rsrc_hdl[TAVOR_SRQHDL];
                        hdl_info.swi_table_ptr = state->ts_srqhdl;
                        hdl_info.swi_num =
                            ((uint64_t)1 << cfgprof->cp_log_num_srq);
                        hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
                        tavor_rsrc_sw_handles_fini(state, &hdl_info);
                }
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL25:
                /* Cleanup the CQ handle resource pool */
                hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_CQHDL];
                hdl_info.swi_table_ptr = state->ts_cqhdl;
                hdl_info.swi_num =
                    ((uint64_t)1 << cfgprof->cp_log_num_cq);
                hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
                tavor_rsrc_sw_handles_fini(state, &hdl_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL24:
                /* Cleanup the EQ handle resource pool */
                hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_EQHDL];
                hdl_info.swi_table_ptr = NULL;
                tavor_rsrc_sw_handles_fini(state, &hdl_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL23:
                /* Cleanup the MR handle resource pool */
                hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_MRHDL];
                hdl_info.swi_table_ptr = NULL;
                tavor_rsrc_sw_handles_fini(state, &hdl_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL22:
                /* Cleanup the PD handle resource pool */
                hdl_info.swi_rsrcpool  = &state->ts_rsrc_hdl[TAVOR_PDHDL];
                hdl_info.swi_table_ptr = NULL;
                tavor_rsrc_pd_handles_fini(state, &hdl_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL21:
                /* Cleanup the EQC table resource pool */
                entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQC];
                tavor_rsrc_hw_entries_fini(state, &entry_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL20:
                /* Cleanup the MCG table resource pool */
                entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MCG];
                tavor_rsrc_hw_entries_fini(state, &entry_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL19:
                /* Cleanup the outstanding command list  */
                tavor_outstanding_cmdlist_fini(state);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL18:
                /* Cleanup the "In" mailbox list  */
                tavor_intr_inmbox_list_fini(state);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL17:
                /* Cleanup the interrupt "In" mailbox resource pool */
                mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
                    TAVOR_INTR_IN_MBOX];
                tavor_rsrc_mbox_fini(state, &mbox_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL16:
                /* Cleanup the "In" mailbox list  */
                tavor_inmbox_list_fini(state);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL15:
                /* Cleanup the "In" mailbox resource pool */
                mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
                tavor_rsrc_mbox_fini(state, &mbox_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL14:
                /*
                 * Cleanup the SRQC table resource pool.
                 *
                 * Note: We only clean up if SRQ is enabled.  Otherwise we
                 * simply fallthrough to the next resource cleanup.
                 */
                if (state->ts_cfg_profile->cp_srq_enable != 0) {
                        entry_info.hwi_rsrcpool =
                            &state->ts_rsrc_hdl[TAVOR_SRQC];
                        tavor_rsrc_hw_entries_fini(state, &entry_info);
                }
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL13:
                /* Cleanup the UAR scratch table resource pool */
                entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
                tavor_rsrc_hw_entries_fini(state, &entry_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL12:
                /* Cleanup the UDAV table resource pool */
                entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UDAV];
                tavor_rsrc_hw_entries_fini(state, &entry_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL11:
                /* Cleanup the EQPC table resource pool */
                entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQPC];
                tavor_rsrc_hw_entries_fini(state, &entry_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL10:
                /* Cleanup the CQC table resource pool */
                entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQC];
                tavor_rsrc_hw_entries_fini(state, &entry_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL9:
                /* Cleanup the RDB table resource pool */
                entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_RDB];
                tavor_rsrc_hw_entries_fini(state, &entry_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL8:
                /* Cleanup the QPC table resource pool */
                entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPC];
                tavor_rsrc_hw_entries_fini(state, &entry_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL7:
                /* Cleanup the MTT table resource pool */
                entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MTT];
                tavor_rsrc_hw_entries_fini(state, &entry_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL6:
                /* Cleanup the MPT table resource pool */
                entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MPT];
                tavor_rsrc_hw_entries_fini(state, &entry_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL5:
                /* Destroy the vmem arena for DDR memory */
                vmem_destroy(state->ts_ddrvmem);
                break;

        /*
         * The cleanup below comes from the "Phase 1" initialization step.
         * (see tavor_rsrc_init_phase1() above)
         */
        case TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE:
                /* Cleanup the interrupt "Out" mailbox list  */
                tavor_intr_outmbox_list_fini(state);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL3:
                /* Cleanup the "Out" mailbox resource pool */
                mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
                    TAVOR_INTR_OUT_MBOX];
                tavor_rsrc_mbox_fini(state, &mbox_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL2:
                /* Cleanup the "Out" mailbox list  */
                tavor_outmbox_list_fini(state);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL1:
                /* Cleanup the "Out" mailbox resource pool */
                mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
                tavor_rsrc_mbox_fini(state, &mbox_info);
                /* FALLTHROUGH */

        case TAVOR_RSRC_CLEANUP_LEVEL0:
                /* Free the array of tavor_rsrc_pool_info_t's */
                kmem_free(state->ts_rsrc_hdl, TAVOR_NUM_RESOURCES *
                    sizeof (tavor_rsrc_pool_info_t));
                kmem_cache_destroy(state->ts_rsrc_cache);
                break;

        default:
                TAVOR_WARNING(state, "unexpected resource cleanup level");
        }
}


/*
 * tavor_rsrc_mbox_init()
 *    Context: Only called from attach() path context
 */
static int
tavor_rsrc_mbox_init(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
{
        tavor_rsrc_pool_info_t  *rsrc_pool;
        tavor_rsrc_priv_mbox_t  *priv;
        vmem_t                  *vmp;
        uint64_t                offset;
        uint_t                  dma_xfer_mode;

        ASSERT(state != NULL);
        ASSERT(info != NULL);

        rsrc_pool = info->mbi_rsrcpool;
        ASSERT(rsrc_pool != NULL);

        dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;

        /* Allocate and initialize mailbox private structure */
        priv = kmem_zalloc(sizeof (tavor_rsrc_priv_mbox_t), KM_SLEEP);
        priv->pmb_dip           = state->ts_dip;
        priv->pmb_acchdl        = state->ts_reg_ddrhdl;
        priv->pmb_devaccattr    = state->ts_reg_accattr;
        priv->pmb_xfer_mode     = dma_xfer_mode;

        /*
         * Initialize many of the default DMA attributes.  Then set alignment
         * and scatter-gather restrictions specific for mailbox memory.
         */
        tavor_dma_attr_init(&priv->pmb_dmaattr);
        priv->pmb_dmaattr.dma_attr_align  = TAVOR_MBOX_ALIGN;
        priv->pmb_dmaattr.dma_attr_sgllen = 1;

        rsrc_pool->rsrc_private = priv;

        /* Is object in DDR memory or system memory? */
        if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
                rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
                    rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
                    0, 0, NULL, NULL, VM_SLEEP);
                if (rsrc_pool->rsrc_ddr_offset == NULL) {
                        /* Unable to alloc space for mailboxes */
                        kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
                        return (DDI_FAILURE);
                }

                /* Calculate offset and starting point (in DDR) */
                offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
                    state->ts_ddr.ddr_baseaddr);
                rsrc_pool->rsrc_start =
                    (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
                    offset);

                /* Create new vmem arena for the mailboxes */
                vmp = vmem_create(info->mbi_rsrcname,
                    rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
                    rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
                if (vmp == NULL) {
                        /* Unable to create vmem arena */
                        vmem_xfree(state->ts_ddrvmem,
                            rsrc_pool->rsrc_ddr_offset,
                            rsrc_pool->rsrc_pool_size);
                        kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
                        return (DDI_FAILURE);
                }
                rsrc_pool->rsrc_vmp = vmp;
        } else {
                rsrc_pool->rsrc_ddr_offset = NULL;
                rsrc_pool->rsrc_start = NULL;
                rsrc_pool->rsrc_vmp = NULL;
        }

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_mbox_fini()
 *    Context: Only called from attach() and/or detach() path contexts
 */
static void
tavor_rsrc_mbox_fini(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
{
        tavor_rsrc_pool_info_t  *rsrc_pool;

        ASSERT(state != NULL);
        ASSERT(info != NULL);

        rsrc_pool = info->mbi_rsrcpool;
        ASSERT(rsrc_pool != NULL);

        /* If mailboxes are DDR memory, then destroy and free up vmem */
        if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {

                /* Destroy the specially created mbox vmem arena */
                vmem_destroy(rsrc_pool->rsrc_vmp);

                /* Free up the region from the ddr_vmem arena */
                vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
                    rsrc_pool->rsrc_pool_size);
        }

        /* Free up the private struct */
        kmem_free(rsrc_pool->rsrc_private, sizeof (tavor_rsrc_priv_mbox_t));
}


/*
 * tavor_rsrc_hw_entries_init()
 *    Context: Only called from attach() path context
 */
static int
tavor_rsrc_hw_entries_init(tavor_state_t *state,
    tavor_rsrc_hw_entry_info_t *info)
{
        tavor_rsrc_pool_info_t  *rsrc_pool;
        tavor_rsrc_t            *rsvd_rsrc = NULL;
        vmem_t                  *vmp;
        uint64_t                num_hwentry, max_hwentry, num_prealloc;
        uint64_t                offset;
        int                     status;

        ASSERT(state != NULL);
        ASSERT(info != NULL);

        rsrc_pool       = info->hwi_rsrcpool;
        ASSERT(rsrc_pool != NULL);
        num_hwentry     = info->hwi_num;
        max_hwentry     = info->hwi_max;
        num_prealloc    = info->hwi_prealloc;

        /* Make sure number of HW entries makes sense */
        if (num_hwentry > max_hwentry) {
                return (DDI_FAILURE);
        }

        /*
         * Determine if we need to allocate DDR space to set up the
         * "rsrc_start" pointer.  Not necessary if "rsrc_start" has already
         * been initialized (as is the case for the UAR page init).
         */
        if (rsrc_pool->rsrc_start == NULL) {
                /* Make sure HW entries table is aligned as specified */
                rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
                    rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
                    0, 0, NULL, NULL, VM_NOSLEEP | VM_FIRSTFIT);
                if (rsrc_pool->rsrc_ddr_offset == NULL) {
                        /* Unable to alloc space for aligned HW table */
                        return (DDI_FAILURE);
                }

                /* Calculate offset and starting point (in DDR) */
                offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
                    state->ts_ddr.ddr_baseaddr);
                rsrc_pool->rsrc_start =
                    (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
                    offset);
        } else {
                rsrc_pool->rsrc_ddr_offset = rsrc_pool->rsrc_start;
        }

        /*
         * Create new vmem arena for the HW entries table (if rsrc_quantum
         * is non-zero).  Otherwise if rsrc_quantum is zero, then these HW
         * entries are not going to be dynamically allocatable (i.e. they
         * won't be allocated/freed through tavor_rsrc_alloc/free).  This
         * latter option is used for EQPC and UARSCR resource which are, in
         * fact, managed by the Tavor hardware.
         */
        if (rsrc_pool->rsrc_quantum != 0) {
                vmp = vmem_create(info->hwi_rsrcname,
                    rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
                    rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
                if (vmp == NULL) {
                        /* Unable to create vmem arena */
                        if (rsrc_pool->rsrc_ddr_offset !=
                            rsrc_pool->rsrc_start) {
                                vmem_xfree(state->ts_ddrvmem,
                                    rsrc_pool->rsrc_ddr_offset,
                                    rsrc_pool->rsrc_pool_size);
                        }
                        return (DDI_FAILURE);
                }
                rsrc_pool->rsrc_vmp = vmp;
        } else {
                rsrc_pool->rsrc_vmp = NULL;
        }

        /* The first HW entries may be reserved by Tavor firmware */
        if (num_prealloc != 0) {
                status = tavor_rsrc_alloc(state, rsrc_pool->rsrc_type,
                    num_prealloc, TAVOR_SLEEP, &rsvd_rsrc);
                if (status != DDI_SUCCESS) {
                        /* Unable to preallocate the reserved HW entries */
                        if (rsrc_pool->rsrc_vmp != NULL) {
                                vmem_destroy(rsrc_pool->rsrc_vmp);
                        }
                        if (rsrc_pool->rsrc_ddr_offset !=
                            rsrc_pool->rsrc_start) {
                                vmem_xfree(state->ts_ddrvmem,
                                    rsrc_pool->rsrc_ddr_offset,
                                    rsrc_pool->rsrc_pool_size);
                        }
                        return (DDI_FAILURE);
                }
        }
        rsrc_pool->rsrc_private = rsvd_rsrc;

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_hw_entries_fini()
 *    Context: Only called from attach() and/or detach() path contexts
 */
static void
tavor_rsrc_hw_entries_fini(tavor_state_t *state,
    tavor_rsrc_hw_entry_info_t *info)
{
        tavor_rsrc_pool_info_t  *rsrc_pool;
        tavor_rsrc_t            *rsvd_rsrc;

        ASSERT(state != NULL);
        ASSERT(info != NULL);

        rsrc_pool = info->hwi_rsrcpool;
        ASSERT(rsrc_pool != NULL);

        /* Free up any "reserved" (i.e. preallocated) HW entries */
        rsvd_rsrc = (tavor_rsrc_t *)rsrc_pool->rsrc_private;
        if (rsvd_rsrc != NULL) {
                tavor_rsrc_free(state, &rsvd_rsrc);
        }

        /*
         * If we've actually setup a vmem arena for the HW entries, then
         * destroy it now
         */
        if (rsrc_pool->rsrc_vmp != NULL) {
                vmem_destroy(rsrc_pool->rsrc_vmp);
        }

        /*
         * Determine if a region was allocated from the tavor_ddr_vmem
         * arena (and free it up if necessary)
         */
        if (rsrc_pool->rsrc_ddr_offset != rsrc_pool->rsrc_start) {
                vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
                    rsrc_pool->rsrc_pool_size);
        }
}


/*
 * tavor_rsrc_sw_handles_init()
 *    Context: Only called from attach() path context
 */
/* ARGSUSED */
static int
tavor_rsrc_sw_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
{
        tavor_rsrc_pool_info_t  *rsrc_pool;
        uint64_t                num_swhdl, max_swhdl, prealloc_sz;

        ASSERT(state != NULL);
        ASSERT(info != NULL);

        rsrc_pool       = info->swi_rsrcpool;
        ASSERT(rsrc_pool != NULL);
        num_swhdl       = info->swi_num;
        max_swhdl       = info->swi_max;
        prealloc_sz     = info->swi_prealloc_sz;

        /* Make sure number of SW handles makes sense */
        if (num_swhdl > max_swhdl) {
                return (DDI_FAILURE);
        }

        /*
         * Depending on the flags parameter, create a kmem_cache for some
         * number of software handle structures.  Note: kmem_cache_create()
         * will SLEEP until successful.
         */
        if (info->swi_flags & TAVOR_SWHDL_KMEMCACHE_INIT) {
                rsrc_pool->rsrc_private = kmem_cache_create(
                    info->swi_rsrcname, rsrc_pool->rsrc_quantum, 0,
                    info->swi_constructor, info->swi_destructor, NULL,
                    rsrc_pool->rsrc_state, NULL, 0);
        }

        /* Allocate the central list of SW handle pointers */
        if (info->swi_flags & TAVOR_SWHDL_TABLE_INIT) {
                info->swi_table_ptr = kmem_zalloc(num_swhdl * prealloc_sz,
                    KM_SLEEP);
        }

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_sw_handles_fini()
 *    Context: Only called from attach() and/or detach() path contexts
 */
/* ARGSUSED */
static void
tavor_rsrc_sw_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
{
        tavor_rsrc_pool_info_t  *rsrc_pool;
        uint64_t                num_swhdl, prealloc_sz;

        ASSERT(state != NULL);
        ASSERT(info != NULL);

        rsrc_pool       = info->swi_rsrcpool;
        num_swhdl       = info->swi_num;
        prealloc_sz     = info->swi_prealloc_sz;

        /*
         * If a "software handle" kmem_cache exists for this resource, then
         * destroy it now
         */
        if (rsrc_pool != NULL) {
                kmem_cache_destroy(rsrc_pool->rsrc_private);
        }

        /* Free up this central list of SW handle pointers */
        if (info->swi_table_ptr != NULL) {
                kmem_free(info->swi_table_ptr, num_swhdl * prealloc_sz);
        }
}


/*
 * tavor_rsrc_pd_handles_init()
 *    Context: Only called from attach() path context
 */
static int
tavor_rsrc_pd_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
{
        tavor_rsrc_pool_info_t  *rsrc_pool;
        vmem_t                  *vmp;
        char                    vmem_name[TAVOR_RSRC_NAME_MAXLEN];
        int                     status;

        ASSERT(state != NULL);
        ASSERT(info != NULL);

        rsrc_pool = info->swi_rsrcpool;
        ASSERT(rsrc_pool != NULL);

        /* Initialize the resource pool for software handle table */
        status = tavor_rsrc_sw_handles_init(state, info);
        if (status != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        /* Build vmem arena name from Tavor instance */
        TAVOR_RSRC_NAME(vmem_name, TAVOR_PDHDL_VMEM);

        /* Create new vmem arena for PD numbers */
        vmp = vmem_create(vmem_name, (caddr_t)1, info->swi_num, 1, NULL,
            NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
        if (vmp == NULL) {
                /* Unable to create vmem arena */
                info->swi_table_ptr = NULL;
                tavor_rsrc_sw_handles_fini(state, info);
                return (DDI_FAILURE);
        }
        rsrc_pool->rsrc_vmp = vmp;

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_pd_handles_fini()
 *    Context: Only called from attach() and/or detach() path contexts
 */
static void
tavor_rsrc_pd_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
{
        tavor_rsrc_pool_info_t  *rsrc_pool;

        ASSERT(state != NULL);
        ASSERT(info != NULL);

        rsrc_pool = info->swi_rsrcpool;

        /* Destroy the specially created UAR scratch table vmem arena */
        vmem_destroy(rsrc_pool->rsrc_vmp);

        /* Destroy the "tavor_sw_pd_t" kmem_cache */
        tavor_rsrc_sw_handles_fini(state, info);
}


/*
 * tavor_rsrc_mbox_alloc()
 *    Context: Only called from attach() path context
 */
static int
tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
    tavor_rsrc_t *hdl)
{
        tavor_rsrc_priv_mbox_t  *priv;
        void                    *addr;
        caddr_t                 kaddr;
        uint64_t                offset;
        size_t                  real_len, temp_len;
        int                     status;

        ASSERT(pool_info != NULL);
        ASSERT(hdl != NULL);

        /* Get the private pointer for the mailboxes */
        priv = pool_info->rsrc_private;
        ASSERT(priv != NULL);

        /*
         * Allocate a DMA handle for the mailbox.  This will be used for
         * two purposes (potentially).  First, it could be used below in
         * the call to ddi_dma_mem_alloc() - if the mailbox is to come from
         * system memory.  Second, it is definitely used later to bind
         * the mailbox for DMA access from/by the hardware.
         */
        status = ddi_dma_alloc_handle(priv->pmb_dip, &priv->pmb_dmaattr,
            DDI_DMA_SLEEP, NULL, &hdl->tr_dmahdl);
        if (status != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        /* Is mailbox in DDR memory or system memory? */
        if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
                /* Use vmem_alloc() to get DDR address of mbox */
                hdl->tr_len = (num * pool_info->rsrc_quantum);
                addr = vmem_alloc(pool_info->rsrc_vmp, hdl->tr_len,
                    VM_SLEEP);
                if (addr == NULL) {
                        /* No more DDR available for mailbox entries */
                        ddi_dma_free_handle(&hdl->tr_dmahdl);
                        return (DDI_FAILURE);
                }
                hdl->tr_acchdl = priv->pmb_acchdl;

                /* Calculate kernel virtual address (from the DDR offset) */
                offset = ((uintptr_t)addr -
                    (uintptr_t)pool_info->rsrc_ddr_offset);
                hdl->tr_addr = (void *)(uintptr_t)(offset +
                    (uintptr_t)pool_info->rsrc_start);

        } else { /* TAVOR_IN_SYSMEM */

                /* Use ddi_dma_mem_alloc() to get memory for mailbox */
                temp_len = (num * pool_info->rsrc_quantum);
                status = ddi_dma_mem_alloc(hdl->tr_dmahdl, temp_len,
                    &priv->pmb_devaccattr, priv->pmb_xfer_mode, DDI_DMA_SLEEP,
                    NULL, &kaddr, &real_len, &hdl->tr_acchdl);
                if (status != DDI_SUCCESS) {
                        /* No more sys memory available for mailbox entries */
                        ddi_dma_free_handle(&hdl->tr_dmahdl);
                        return (DDI_FAILURE);
                }
                hdl->tr_addr = (void *)kaddr;
                hdl->tr_len  = real_len;
        }

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_mbox_free()
 *    Context: Can be called from interrupt or base context.
 */
static void
tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
{
        void            *addr;
        uint64_t        offset;

        ASSERT(pool_info != NULL);
        ASSERT(hdl != NULL);

        /* Is mailbox in DDR memory or system memory? */
        if (pool_info->rsrc_loc == TAVOR_IN_DDR) {

                /* Calculate the allocated address (the mbox's DDR offset) */
                offset = ((uintptr_t)hdl->tr_addr -
                    (uintptr_t)pool_info->rsrc_start);
                addr = (void *)(uintptr_t)(offset +
                    (uintptr_t)pool_info->rsrc_ddr_offset);

                /* Use vmem_free() to free up DDR memory for mailbox */
                vmem_free(pool_info->rsrc_vmp, addr, hdl->tr_len);

        } else { /* TAVOR_IN_SYSMEM */

                /* Use ddi_dma_mem_free() to free up sys memory for mailbox */
                ddi_dma_mem_free(&hdl->tr_acchdl);
        }

        /* Free the DMA handle for the mailbox */
        ddi_dma_free_handle(&hdl->tr_dmahdl);
}


/*
 * tavor_rsrc_hw_entry_alloc()
 *    Context: Can be called from interrupt or base context.
 */
static int
tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
    uint_t num_align, ddi_acc_handle_t acc_handle, uint_t sleepflag,
    tavor_rsrc_t *hdl)
{
        void            *addr;
        uint64_t        offset;
        uint32_t        align;
        int             flag;

        ASSERT(pool_info != NULL);
        ASSERT(hdl != NULL);

        /*
         * Tavor hardware entries (QPC, CQC, EQC, MPT, MTT, etc.) do not
         * use dma_handle (because they are in Tavor locally attached DDR
         * memory) and, generally, don't use the acc_handle (because the
         * entries are not directly accessed by software).  The exceptions
         * to this rule are the UARPG and UDAV entries.
         */

        /*
         * Use vmem_xalloc() to get a properly aligned pointer (based on
         * the number requested) to the HW entry(ies).  This handles the
         * cases (for special QPCs and for RDB entries) where we need more
         * than one and need to ensure that they are properly aligned.
         */
        flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
        hdl->tr_len = (num * pool_info->rsrc_quantum);
        align       = (num_align * pool_info->rsrc_quantum);
        addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->tr_len,
            align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT);
        if (addr == NULL) {
                /* No more HW entries available */
                return (DDI_FAILURE);
        }

        /* If an access handle was provided, fill it in */
        if (acc_handle != 0) {
                hdl->tr_acchdl = acc_handle;
        }

        /* Calculate vaddr and HW table index (from the DDR offset) */
        offset = ((uintptr_t)addr - (uintptr_t)pool_info->rsrc_ddr_offset);
        hdl->tr_addr = (void *)(uintptr_t)(offset +
            (uintptr_t)pool_info->rsrc_start);
        hdl->tr_indx = (offset >> pool_info->rsrc_shift);

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_hw_entry_free()
 *    Context: Can be called from interrupt or base context.
 */
static void
tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
{
        void            *addr;
        uint64_t        offset;

        ASSERT(pool_info != NULL);
        ASSERT(hdl != NULL);

        /* Calculate the allocated address (the entry's DDR offset) */
        offset = ((uintptr_t)hdl->tr_addr - (uintptr_t)pool_info->rsrc_start);
        addr   = (void *)(uintptr_t)(offset +
            (uintptr_t)pool_info->rsrc_ddr_offset);

        /* Use vmem_xfree() to free up the HW table entry */
        vmem_xfree(pool_info->rsrc_vmp, addr, hdl->tr_len);
}


/*
 * tavor_rsrc_swhdl_alloc()
 *    Context: Can be called from interrupt or base context.
 */
static int
tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
    tavor_rsrc_t *hdl)
{
        void    *addr;
        int     flag;

        ASSERT(pool_info != NULL);
        ASSERT(hdl != NULL);

        /* Allocate the software handle structure */
        flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
        addr = kmem_cache_alloc(pool_info->rsrc_private, flag);
        if (addr == NULL) {
                return (DDI_FAILURE);
        }
        hdl->tr_len  = pool_info->rsrc_quantum;
        hdl->tr_addr = addr;

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_swhdl_free()
 *    Context: Can be called from interrupt or base context.
 */
static void
tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
{
        ASSERT(pool_info != NULL);
        ASSERT(hdl != NULL);

        /* Free the software handle structure */
        kmem_cache_free(pool_info->rsrc_private, hdl->tr_addr);
}


/*
 * tavor_rsrc_pdhdl_alloc()
 *    Context: Can be called from interrupt or base context.
 */
static int
tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
    tavor_rsrc_t *hdl)
{
        tavor_pdhdl_t   addr;
        void            *tmpaddr;
        int             flag, status;

        ASSERT(pool_info != NULL);
        ASSERT(hdl != NULL);

        /* Allocate the software handle */
        status = tavor_rsrc_swhdl_alloc(pool_info, sleepflag, hdl);
        if (status != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }
        addr = (tavor_pdhdl_t)hdl->tr_addr;
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr))

        /* Allocate a PD number for the handle */
        flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
        tmpaddr = vmem_alloc(pool_info->rsrc_vmp, 1, flag);
        if (tmpaddr == NULL) {
                /* No more PD number entries available */
                tavor_rsrc_swhdl_free(pool_info, hdl);
                return (DDI_FAILURE);
        }
        addr->pd_pdnum = (uint32_t)(uintptr_t)tmpaddr;
        addr->pd_rsrcp = hdl;
        hdl->tr_indx   = addr->pd_pdnum;

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_pdhdl_free()
 *    Context: Can be called from interrupt or base context.
 */
static void
tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
{
        ASSERT(pool_info != NULL);
        ASSERT(hdl != NULL);

        /* Use vmem_free() to free up the PD number */
        vmem_free(pool_info->rsrc_vmp, (void *)(uintptr_t)hdl->tr_indx, 1);

        /* Free the software handle structure */
        tavor_rsrc_swhdl_free(pool_info, hdl);
}


/*
 * tavor_rsrc_pdhdl_constructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static int
tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags)
{
        tavor_pdhdl_t   pdhdl;
        tavor_state_t   *state;

        pdhdl = (tavor_pdhdl_t)pd;
        state = (tavor_state_t *)priv;

        mutex_init(&pdhdl->pd_lock, NULL, MUTEX_DRIVER,
            DDI_INTR_PRI(state->ts_intrmsi_pri));

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_pdhdl_destructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static void
tavor_rsrc_pdhdl_destructor(void *pd, void *priv)
{
        tavor_pdhdl_t   pdhdl;

        pdhdl = (tavor_pdhdl_t)pd;

        mutex_destroy(&pdhdl->pd_lock);
}


/*
 * tavor_rsrc_cqhdl_constructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static int
tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags)
{
        tavor_cqhdl_t   cqhdl;
        tavor_state_t   *state;

        cqhdl = (tavor_cqhdl_t)cq;
        state = (tavor_state_t *)priv;

        mutex_init(&cqhdl->cq_lock, NULL, MUTEX_DRIVER,
            DDI_INTR_PRI(state->ts_intrmsi_pri));
        mutex_init(&cqhdl->cq_wrid_wqhdr_lock, NULL, MUTEX_DRIVER,
            DDI_INTR_PRI(state->ts_intrmsi_pri));

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_cqhdl_destructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static void
tavor_rsrc_cqhdl_destructor(void *cq, void *priv)
{
        tavor_cqhdl_t   cqhdl;

        cqhdl = (tavor_cqhdl_t)cq;

        mutex_destroy(&cqhdl->cq_wrid_wqhdr_lock);
        mutex_destroy(&cqhdl->cq_lock);
}


/*
 * tavor_rsrc_qphdl_constructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static int
tavor_rsrc_qphdl_constructor(void *qp, void *priv, int flags)
{
        tavor_qphdl_t   qphdl;
        tavor_state_t   *state;

        qphdl = (tavor_qphdl_t)qp;
        state = (tavor_state_t *)priv;

        mutex_init(&qphdl->qp_lock, NULL, MUTEX_DRIVER,
            DDI_INTR_PRI(state->ts_intrmsi_pri));

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_qphdl_destructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static void
tavor_rsrc_qphdl_destructor(void *qp, void *priv)
{
        tavor_qphdl_t   qphdl;

        qphdl = (tavor_qphdl_t)qp;

        mutex_destroy(&qphdl->qp_lock);
}


/*
 * tavor_rsrc_srqhdl_constructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static int
tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags)
{
        tavor_srqhdl_t  srqhdl;
        tavor_state_t   *state;

        srqhdl = (tavor_srqhdl_t)srq;
        state = (tavor_state_t *)priv;

        mutex_init(&srqhdl->srq_lock, NULL, MUTEX_DRIVER,
            DDI_INTR_PRI(state->ts_intrmsi_pri));

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_srqhdl_destructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static void
tavor_rsrc_srqhdl_destructor(void *srq, void *priv)
{
        tavor_srqhdl_t  srqhdl;

        srqhdl = (tavor_srqhdl_t)srq;

        mutex_destroy(&srqhdl->srq_lock);
}


/*
 * tavor_rsrc_refcnt_constructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static int
tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags)
{
        tavor_sw_refcnt_t       *refcnt;
        tavor_state_t           *state;

        refcnt = (tavor_sw_refcnt_t *)rc;
        state  = (tavor_state_t *)priv;

        mutex_init(&refcnt->swrc_lock, NULL, MUTEX_DRIVER,
            DDI_INTR_PRI(state->ts_intrmsi_pri));

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_refcnt_destructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static void
tavor_rsrc_refcnt_destructor(void *rc, void *priv)
{
        tavor_sw_refcnt_t       *refcnt;

        refcnt = (tavor_sw_refcnt_t *)rc;

        mutex_destroy(&refcnt->swrc_lock);
}


/*
 * tavor_rsrc_ahhdl_constructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static int
tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags)
{
        tavor_ahhdl_t   ahhdl;
        tavor_state_t   *state;

        ahhdl = (tavor_ahhdl_t)ah;
        state = (tavor_state_t *)priv;

        mutex_init(&ahhdl->ah_lock, NULL, MUTEX_DRIVER,
            DDI_INTR_PRI(state->ts_intrmsi_pri));

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_ahhdl_destructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static void
tavor_rsrc_ahhdl_destructor(void *ah, void *priv)
{
        tavor_ahhdl_t   ahhdl;

        ahhdl = (tavor_ahhdl_t)ah;

        mutex_destroy(&ahhdl->ah_lock);
}


/*
 * tavor_rsrc_mrhdl_constructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static int
tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags)
{
        tavor_mrhdl_t   mrhdl;
        tavor_state_t   *state;

        mrhdl = (tavor_mrhdl_t)mr;
        state = (tavor_state_t *)priv;

        mutex_init(&mrhdl->mr_lock, NULL, MUTEX_DRIVER,
            DDI_INTR_PRI(state->ts_intrmsi_pri));

        return (DDI_SUCCESS);
}


/*
 * tavor_rsrc_mrhdl_destructor()
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static void
tavor_rsrc_mrhdl_destructor(void *mr, void *priv)
{
        tavor_mrhdl_t   mrhdl;

        mrhdl = (tavor_mrhdl_t)mr;

        mutex_destroy(&mrhdl->mr_lock);
}


/*
 * tavor_rsrc_mcg_entry_get_size()
 */
static int
tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, uint_t *mcg_size_shift)
{
        uint_t  num_qp_per_mcg, max_qp_per_mcg, log2;

        /*
         * Round the configured number of QP per MCG to next larger
         * power-of-2 size and update.
         */
        num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg + 8;
        log2 = highbit(num_qp_per_mcg);
        if (ISP2(num_qp_per_mcg)) {
                log2 = log2 - 1;
        }
        state->ts_cfg_profile->cp_num_qp_per_mcg = (1 << log2) - 8;

        /* Now make sure number of QP per MCG makes sense */
        num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg;
        max_qp_per_mcg = (1 << state->ts_devlim.log_max_qp_mcg);
        if (num_qp_per_mcg > max_qp_per_mcg) {
                return (DDI_FAILURE);
        }

        /* Return the (shift) size of an individual MCG HW entry */
        *mcg_size_shift = log2 + 2;

        return (DDI_SUCCESS);
}