/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * This file implements the Work Queue Entry (WQE) management in IBMF.
 */

#include <sys/ib/mgt/ibmf/ibmf_impl.h>

extern int ibmf_trace_level;
extern int ibmf_send_wqes_per_port, ibmf_recv_wqes_per_port;

#define IBMF_INIT_SG_ELEMENT(sg, mem, lkey, size)       { \
        (sg).ds_va = (ib_vaddr_t)(uintptr_t)(mem);      \
        (sg).ds_key = (lkey);                           \
        (sg).ds_len = (size);                           \
}

#define IBMF_ADDR_TO_SEND_WR_ID(ptr, id)                \
        (id) = (ibt_wrid_t)(uintptr_t)(ptr)

#define IBMF_ADDR_TO_RECV_WR_ID(ptr, id)                 \
        (id) = ((ibt_wrid_t)(uintptr_t)(ptr) | IBMF_RCV_CQE)

#define IBMF_INIT_RMPP_HDR(hdrp, ver, type, respt, flg, status, seg, lennwl) { \
        (hdrp)->rmpp_version = (ver);                   \
        (hdrp)->rmpp_type = (type);                     \
        (hdrp)->rmpp_resp_time = (respt);               \
        (hdrp)->rmpp_flags = (flg);                     \
        (hdrp)->rmpp_status = (status);                 \
        (hdrp)->rmpp_segnum = (h2b32(seg));             \
        (hdrp)->rmpp_pyldlen_nwl = (h2b32(lennwl));     \
}

static int ibmf_send_wqe_cache_constructor(void *buf, void *cdrarg,
    int kmflags);
static void ibmf_send_wqe_cache_destructor(void *buf, void *cdrarg);
static int ibmf_recv_wqe_cache_constructor(void *buf, void *cdrarg,
    int kmflags);
static void ibmf_recv_wqe_cache_destructor(void *buf, void *cdrarg);
static int ibmf_i_extend_wqe_mem(ibmf_ci_t *cip,
    ibmf_qp_handle_t ibmf_qp_handle, ibmf_wqe_mgt_t *wqe_mgt,
    boolean_t block);

/*
 * ibmf_send_wqe_cache_constructor():
 *      Constructor for the kmem cache used for send WQEs for special QPs
 */
/* ARGSUSED */
static int
ibmf_send_wqe_cache_constructor(void *buf, void *cdrarg, int kmflags)
{
        ibmf_send_wqe_t         *send_wqe = (ibmf_send_wqe_t *)buf;
        ibmf_ci_t               *cip = (ibmf_ci_t *)cdrarg;
        ibmf_wqe_mgt_t          *wqe_mgt;

        IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_send_wqe_cache_constructor_start, IBMF_TNF_TRACE, "",
            "ibmf_send_wqe_cache_constructor() enter, buf = %p, cdarg = %p\n",
            tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg);

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqe))

        /* initialize send WQE context */
        send_wqe->send_sg_mem =
            (ib_vaddr_t)(uintptr_t)vmem_alloc(cip->ci_wqe_ib_vmem,
            IBMF_MEM_PER_WQE, kmflags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
        if (send_wqe->send_sg_mem == 0) {
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_send_wqe_cache_constructor_err, IBMF_TNF_ERROR, "",
                    "ibmf_send_wqe_cache_constructor(): %s\n", tnf_string, msg,
                    "Failed vmem allocation in send WQE cache constructor");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_send_wqe_cache_constructor_end, IBMF_TNF_TRACE, "",
                    "ibmf_send_wqe_cache_constructor() exit\n");
                return (-1);
        }

        mutex_enter(&cip->ci_wqe_mutex);
        wqe_mgt = cip->ci_wqe_mgt_list;

        /* Look for the WQE management struct that includes this address */
        while (wqe_mgt != NULL) {
                mutex_enter(&wqe_mgt->wqes_mutex);
                if ((send_wqe->send_sg_mem >= wqe_mgt->wqes_ib_mem) &&
                    (send_wqe->send_sg_mem < (wqe_mgt->wqes_ib_mem +
                    wqe_mgt->wqes_kmem_sz))) {
                        mutex_exit(&wqe_mgt->wqes_mutex);
                        break;
                }
                mutex_exit(&wqe_mgt->wqes_mutex);
                wqe_mgt = wqe_mgt->wqe_mgt_next;
        }

        if (wqe_mgt == NULL) {
                mutex_exit(&cip->ci_wqe_mutex);
                vmem_free(cip->ci_wqe_ib_vmem,
                    (void *)(uintptr_t)send_wqe->send_sg_mem, IBMF_MEM_PER_WQE);
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_send_wqe_cache_constructor_err, IBMF_TNF_ERROR, "",
                    "ibmf_send_wqe_cache_constructor(): %s\n", tnf_string, msg,
                    "Address not found in WQE mgt list");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_send_wqe_cache_constructor_end, IBMF_TNF_TRACE, "",
                    "ibmf_send_wqe_cache_constructor() exit\n");
                return (-1);
        }

        mutex_enter(&wqe_mgt->wqes_mutex);

        send_wqe->send_mem = (caddr_t)((uintptr_t)wqe_mgt->wqes_kmem +
            (uintptr_t)(send_wqe->send_sg_mem - wqe_mgt->wqes_ib_mem));
        bzero(send_wqe->send_mem, IBMF_MEM_PER_WQE);
        send_wqe->send_sg_lkey = wqe_mgt->wqes_ib_lkey;
        send_wqe->send_mem_hdl = wqe_mgt->wqes_ib_mem_hdl;
        send_wqe->send_wqe_flags = 0;
        send_wqe->send_wqe_next = NULL;

        mutex_exit(&wqe_mgt->wqes_mutex);
        mutex_exit(&cip->ci_wqe_mutex);

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_send_wqe_cache_constructor_end, IBMF_TNF_TRACE, "",
            "ibmf_send_wqe_cache_constructor() exit\n");

        return (0);
}

/*
 * ibmf_send_wqe_cache_destructor():
 *      Destructor for send WQE kmem cache for special QPs
 */
/* ARGSUSED */
static void
ibmf_send_wqe_cache_destructor(void *buf, void *cdrarg)
{
        ibmf_send_wqe_t         *send_wqe = (ibmf_send_wqe_t *)buf;
        ibmf_ci_t               *cip = (ibmf_ci_t *)cdrarg;

        IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_send_wqe_cache_destructor_start, IBMF_TNF_TRACE, "",
            "ibmf_send_wqe_cache_destructor() enter, buf = %p, cdarg = %p\n",
            tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg);

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqe))

        /* Free the vmem allocated for the WQE */
        vmem_free(cip->ci_wqe_ib_vmem,
            (void *)(uintptr_t)send_wqe->send_sg_mem, IBMF_MEM_PER_WQE);
        send_wqe->send_mem = NULL;

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_send_wqe_cache_destructor_end, IBMF_TNF_TRACE, "",
            "ibmf_send_wqe_cache_destructor() exit\n");
}

/*
 * ibmf_recv_wqe_cache_constructor():
 *      Constructor for receive WQE kmem cache for special QPs
 */
/* ARGSUSED */
static int
ibmf_recv_wqe_cache_constructor(void *buf, void *cdrarg, int kmflags)
{
        ibmf_recv_wqe_t         *recv_wqe = (ibmf_recv_wqe_t *)buf;
        ibmf_ci_t               *cip = (ibmf_ci_t *)cdrarg;
        ibmf_wqe_mgt_t          *wqe_mgt;

        IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_recv_wqe_cache_constructor_start, IBMF_TNF_TRACE, "",
            "ibmf_recv_wqe_cache_constructor() enter, buf = %p, cdarg = %p\n",
            tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg);

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*recv_wqe))

        /* initialize recv WQE context */
        recv_wqe->recv_sg_mem =
            (ib_vaddr_t)(uintptr_t)vmem_alloc(cip->ci_wqe_ib_vmem,
            IBMF_MEM_PER_WQE, kmflags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
        if (recv_wqe->recv_sg_mem == 0) {
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_recv_wqe_cache_constructor_err, IBMF_TNF_ERROR, "",
                    "ibmf_recv_wqe_cache_constructor(): %s\n", tnf_string, msg,
                    "Failed vmem allocation in receive WQE cache constructor");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_recv_wqe_cache_constructor_end, IBMF_TNF_TRACE, "",
                    "ibmf_recv_wqe_cache_constructor() exit\n");
                return (-1);
        }

        mutex_enter(&cip->ci_wqe_mutex);
        wqe_mgt = cip->ci_wqe_mgt_list;

        /* Look for the WQE management struct that includes this address */
        while (wqe_mgt != NULL) {
                mutex_enter(&wqe_mgt->wqes_mutex);
                if ((recv_wqe->recv_sg_mem >= wqe_mgt->wqes_ib_mem) &&
                    (recv_wqe->recv_sg_mem < (wqe_mgt->wqes_ib_mem +
                    wqe_mgt->wqes_kmem_sz))) {
                        mutex_exit(&wqe_mgt->wqes_mutex);
                        break;
                }
                mutex_exit(&wqe_mgt->wqes_mutex);
                wqe_mgt = wqe_mgt->wqe_mgt_next;
        }

        if (wqe_mgt == NULL) {
                mutex_exit(&cip->ci_wqe_mutex);
                vmem_free(cip->ci_wqe_ib_vmem,
                    (void *)(uintptr_t)recv_wqe->recv_sg_mem, IBMF_MEM_PER_WQE);
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_recv_wqe_cache_constructor_err, IBMF_TNF_ERROR, "",
                    "ibmf_recv_wqe_cache_constructor(): %s\n", tnf_string, msg,
                    "Address not found in WQE mgt list");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_recv_wqe_cache_constructor_end, IBMF_TNF_TRACE, "",
                    "ibmf_recv_wqe_cache_constructor() exit\n");
                return (-1);
        }

        mutex_enter(&wqe_mgt->wqes_mutex);

        recv_wqe->recv_mem = (caddr_t)((uintptr_t)wqe_mgt->wqes_kmem +
            (uintptr_t)(recv_wqe->recv_sg_mem - wqe_mgt->wqes_ib_mem));
        bzero(recv_wqe->recv_mem, IBMF_MEM_PER_WQE);
        recv_wqe->recv_sg_lkey = wqe_mgt->wqes_ib_lkey;
        recv_wqe->recv_mem_hdl = wqe_mgt->wqes_ib_mem_hdl;
        recv_wqe->recv_wqe_next = NULL;
        recv_wqe->recv_msg = NULL;
        recv_wqe->recv_wqe_flags = 0;

        mutex_exit(&wqe_mgt->wqes_mutex);
        mutex_exit(&cip->ci_wqe_mutex);

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_recv_wqe_cache_constructor_end, IBMF_TNF_TRACE, "",
            "ibmf_recv_wqe_cache_constructor() exit\n");

        return (0);
}

/*
 * ibmf_recv_wqe_cache_destructor():
 *      Destructor for receive WQE kmem cache for special QPs
 */
/* ARGSUSED */
static void
ibmf_recv_wqe_cache_destructor(void *buf, void *cdrarg)
{
        ibmf_recv_wqe_t         *recv_wqe = (ibmf_recv_wqe_t *)buf;
        ibmf_ci_t               *cip = (ibmf_ci_t *)cdrarg;

        IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_recv_wqe_cache_destructor_start, IBMF_TNF_TRACE, "",
            "ibmf_recv_wqe_cache_destructor() enter, buf = %p, cdarg = %p\n",
            tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg);

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*recv_wqe))

        /* Free the vmem allocated for the WQE */
        vmem_free(cip->ci_wqe_ib_vmem,
            (void *)(uintptr_t)recv_wqe->recv_sg_mem, IBMF_MEM_PER_WQE);
        recv_wqe->recv_mem = NULL;

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_recv_wqe_cache_destructor_end, IBMF_TNF_TRACE, "",
            "ibmf_recv_wqe_cache_destructor() exit\n");
}

/*
 * ibmf_altqp_send_wqe_cache_constructor():
 *      Constructor for the kmem cache used for send WQEs for alternate QPs
 */
/* ARGSUSED */
int
ibmf_altqp_send_wqe_cache_constructor(void *buf, void *cdrarg, int kmflags)
{
        ibmf_send_wqe_t *send_wqe = (ibmf_send_wqe_t *)buf;
        ibmf_alt_qp_t   *qp_ctx = (ibmf_alt_qp_t *)cdrarg;
        ibmf_wqe_mgt_t  *wqe_mgt;

        IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_altqp_send_wqe_cache_constructor_start, IBMF_TNF_TRACE, "",
            "ibmf_altqp_send_wqe_cache_constructor() enter, buf = %p, "
            "cdarg = %p\n", tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg);

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqe))

        /* initialize send WQE context */
        send_wqe->send_sg_mem = (ib_vaddr_t)(uintptr_t)vmem_alloc(
            qp_ctx->isq_wqe_ib_vmem, IBMF_MEM_PER_WQE,
            kmflags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
        if (send_wqe->send_sg_mem == 0) {
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_altqp_send_wqe_cache_constructor_err, IBMF_TNF_ERROR,
                    "", "ibmf_altqp_send_wqe_cache_constructor(): %s\n",
                    tnf_string, msg, "Failed vmem allocation in "
                    "alternate QP send WQE cache constructor");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_altqp_send_wqe_cache_constructor_end, IBMF_TNF_TRACE,
                    "", "ibmf_altqp_send_wqe_cache_constructor() exit\n");
                return (-1);
        }

        mutex_enter(&qp_ctx->isq_wqe_mutex);
        wqe_mgt = qp_ctx->isq_wqe_mgt_list;

        /* Look for the WQE management struct that includes this address */
        while (wqe_mgt != NULL) {
                mutex_enter(&wqe_mgt->wqes_mutex);
                if ((send_wqe->send_sg_mem >= wqe_mgt->wqes_ib_mem) &&
                    (send_wqe->send_sg_mem < (wqe_mgt->wqes_ib_mem +
                    wqe_mgt->wqes_kmem_sz))) {
                        mutex_exit(&wqe_mgt->wqes_mutex);
                        break;
                }
                mutex_exit(&wqe_mgt->wqes_mutex);
                wqe_mgt = wqe_mgt->wqe_mgt_next;
        }

        if (wqe_mgt == NULL) {
                mutex_exit(&qp_ctx->isq_wqe_mutex);
                vmem_free(qp_ctx->isq_wqe_ib_vmem,
                    (void *)(uintptr_t)send_wqe->send_sg_mem, IBMF_MEM_PER_WQE);
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_altqp_send_wqe_cache_constructor_err, IBMF_TNF_ERROR,
                    "", "ibmf_altqp_send_wqe_cache_constructor(): %s\n",
                    tnf_string, msg, "Address not found in WQE mgt list");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_altqp_send_wqe_cache_constructor_end,
                    IBMF_TNF_TRACE, "",
                    "ibmf_altqp_send_wqe_cache_constructor() exit\n");
                return (-1);
        }

        mutex_enter(&wqe_mgt->wqes_mutex);

        send_wqe->send_mem = (caddr_t)((uintptr_t)wqe_mgt->wqes_kmem +
            (uintptr_t)(send_wqe->send_sg_mem - wqe_mgt->wqes_ib_mem));
        bzero(send_wqe->send_mem, IBMF_MEM_PER_WQE);
        send_wqe->send_sg_lkey = wqe_mgt->wqes_ib_lkey;
        send_wqe->send_mem_hdl = wqe_mgt->wqes_ib_mem_hdl;
        send_wqe->send_wqe_flags = 0;

        mutex_exit(&wqe_mgt->wqes_mutex);
        mutex_exit(&qp_ctx->isq_wqe_mutex);

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_altqp_send_wqe_cache_constructor_end, IBMF_TNF_TRACE, "",
            "ibmf_altqp_send_wqe_cache_constructor() exit\n");

        return (0);
}

/*
 * ibmf_altqp_send_wqe_cache_destructor():
 *      Destructor for send WQE kmem cache for alternate QPs
 */
/* ARGSUSED */
void
ibmf_altqp_send_wqe_cache_destructor(void *buf, void *cdrarg)
{
        ibmf_send_wqe_t *send_wqe = (ibmf_send_wqe_t *)buf;
        ibmf_alt_qp_t   *qp_ctx = (ibmf_alt_qp_t *)cdrarg;

        IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_altqp_send_wqe_cache_destructor_start, IBMF_TNF_TRACE, "",
            "ibmf_altqp_send_wqe_cache_destructor() enter, buf = %p, "
            "cdarg = %p\n", tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg);

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqe))

        /* Free the vmem allocated for the WQE */
        vmem_free(qp_ctx->isq_wqe_ib_vmem,
            (void *)(uintptr_t)send_wqe->send_sg_mem, IBMF_MEM_PER_WQE);
        send_wqe->send_mem = NULL;

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_altqp_send_wqe_cache_destructor_end, IBMF_TNF_TRACE, "",
            "ibmf_altqp_send_wqe_cache_destructor() exit\n");
}

/*
 * ibmf_altqp_recv_wqe_cache_constructor():
 *      Constructor for receive WQE kmem cache for alternate QPs
 */
/* ARGSUSED */
int
ibmf_altqp_recv_wqe_cache_constructor(void *buf, void *cdrarg, int kmflags)
{
        ibmf_recv_wqe_t *recv_wqe = (ibmf_recv_wqe_t *)buf;
        ibmf_alt_qp_t   *qp_ctx = (ibmf_alt_qp_t *)cdrarg;
        ibmf_wqe_mgt_t  *wqe_mgt;

        IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_altqp_recv_wqe_cache_constructor_start, IBMF_TNF_TRACE, "",
            "ibmf_altqp_recv_wqe_cache_constructor() enter, buf = %p, "
            "cdarg = %p\n", tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg);

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*recv_wqe))

        /* initialize recv WQE context */
        recv_wqe->recv_sg_mem = (ib_vaddr_t)(uintptr_t)vmem_alloc(
            qp_ctx->isq_wqe_ib_vmem, IBMF_MEM_PER_WQE,
            kmflags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
        if (recv_wqe->recv_sg_mem == 0) {
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_altqp_recv_wqe_cache_constructor_err, IBMF_TNF_ERROR,
                    "", "ibmf_altqp_recv_wqe_cache_constructor(): %s\n",
                    tnf_string, msg,
                    "Failed vmem allocation in recv WQE cache constructor");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_altqp_recv_wqe_cache_constructor_end, IBMF_TNF_TRACE,
                    "", "ibmf_altqp_recv_wqe_cache_constructor() exit\n");
                return (-1);
        }

        mutex_enter(&qp_ctx->isq_wqe_mutex);
        wqe_mgt = qp_ctx->isq_wqe_mgt_list;

        /* Look for the WQE management struct that includes this address */
        while (wqe_mgt != NULL) {
                mutex_enter(&wqe_mgt->wqes_mutex);
                if ((recv_wqe->recv_sg_mem >= wqe_mgt->wqes_ib_mem) &&
                    (recv_wqe->recv_sg_mem < (wqe_mgt->wqes_ib_mem +
                    wqe_mgt->wqes_kmem_sz))) {
                        mutex_exit(&wqe_mgt->wqes_mutex);
                        break;
                }
                mutex_exit(&wqe_mgt->wqes_mutex);
                wqe_mgt = wqe_mgt->wqe_mgt_next;
        }

        if (wqe_mgt == NULL) {
                mutex_exit(&qp_ctx->isq_wqe_mutex);
                vmem_free(qp_ctx->isq_wqe_ib_vmem,
                    (void *)(uintptr_t)recv_wqe->recv_sg_mem, IBMF_MEM_PER_WQE);
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_recv_wqe_cache_constructor_err, IBMF_TNF_ERROR, "",
                    "ibmf_altqp_recv_wqe_cache_constructor(): %s\n",
                    tnf_string, msg, "Address not found in WQE mgt list");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_recv_wqe_cache_constructor_end, IBMF_TNF_TRACE, "",
                    "ibmf_recv_wqe_cache_constructor() exit\n");
                return (-1);
        }

        mutex_enter(&wqe_mgt->wqes_mutex);

        recv_wqe->recv_mem = (caddr_t)((uintptr_t)wqe_mgt->wqes_kmem +
            (uintptr_t)(recv_wqe->recv_sg_mem - wqe_mgt->wqes_ib_mem));
        bzero(recv_wqe->recv_mem, IBMF_MEM_PER_WQE);
        recv_wqe->recv_sg_lkey = wqe_mgt->wqes_ib_lkey;
        recv_wqe->recv_mem_hdl = wqe_mgt->wqes_ib_mem_hdl;
        recv_wqe->recv_wqe_flags = 0;

        mutex_exit(&wqe_mgt->wqes_mutex);
        mutex_exit(&qp_ctx->isq_wqe_mutex);

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_altqp_recv_wqe_cache_constructor_end, IBMF_TNF_TRACE, "",
            "ibmf_altqp_recv_wqe_cache_constructor() exit\n");

        return (0);
}

/*
 * ibmf_altqp_recv_wqe_cache_destructor():
 *      Destructor for receive WQE kmem cache for alternate QPs
 */
/* ARGSUSED */
void
ibmf_altqp_recv_wqe_cache_destructor(void *buf, void *cdrarg)
{
        ibmf_recv_wqe_t *recv_wqe = (ibmf_recv_wqe_t *)buf;
        ibmf_alt_qp_t   *qp_ctx = (ibmf_alt_qp_t *)cdrarg;

        IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_altqp_recv_wqe_cache_destructor_start, IBMF_TNF_TRACE, "",
            "ibmf_altqp_recv_wqe_cache_destructor() enter, buf = %p, "
            "cdarg = %p\n", tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg);

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*recv_wqe))

        /* Free the vmem allocated for the WQE */
        vmem_free(qp_ctx->isq_wqe_ib_vmem,
            (void *)(uintptr_t)recv_wqe->recv_sg_mem, IBMF_MEM_PER_WQE);
        recv_wqe->recv_mem = NULL;

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_altqp_recv_wqe_cache_destructor_end, IBMF_TNF_TRACE, "",
            "ibmf_altqp_recv_wqe_cache_destructor() exit\n");
}

/*
 * ibmf_i_init_wqes():
 *      Create the kmem cache for send and receive WQEs
 */
int
ibmf_i_init_wqes(ibmf_ci_t *cip)
{
        ibt_status_t            status;
        ibt_mr_hdl_t            mem_hdl;
        ibt_mr_desc_t           mem_desc;
        ibt_mr_attr_t           mem_attr;
        ibmf_wqe_mgt_t          *wqe_mgtp;
        char                    string[128];

        IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_wqes_start,
            IBMF_TNF_TRACE, "", "ibmf_i_init_wqes() enter, cip = %p\n",
            tnf_opaque, cip, cip);

        /*
         * Allocate memory for the WQE management structure
         */
        wqe_mgtp = kmem_zalloc(sizeof (ibmf_wqe_mgt_t), KM_SLEEP);
        mutex_init(&wqe_mgtp->wqes_mutex, NULL, MUTEX_DRIVER, NULL);
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqe_mgtp))

        /*
         * Allocate memory for the WQEs to be used by the special QPs on this CI
         * There are two special QPs per CI port
         */
        wqe_mgtp->wqes_kmem_sz = cip->ci_nports * 2 *
            ((IBMF_MEM_PER_WQE * ibmf_send_wqes_per_port) +
            (IBMF_MEM_PER_WQE * ibmf_recv_wqes_per_port));
        wqe_mgtp->wqes_kmem =
            kmem_zalloc(wqe_mgtp->wqes_kmem_sz, KM_SLEEP);

        mem_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)wqe_mgtp->wqes_kmem;
        mem_attr.mr_len = wqe_mgtp->wqes_kmem_sz;
        mem_attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
        mem_attr.mr_as = NULL;

        /* Register the allocated memory */
        status = ibt_register_mr(cip->ci_ci_handle, cip->ci_pd, &mem_attr,
            &mem_hdl, &mem_desc);
        if (status != IBT_SUCCESS) {
                kmem_free(wqe_mgtp->wqes_kmem,
                    wqe_mgtp->wqes_kmem_sz);
                IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_i_init_wqes_err, IBMF_TNF_ERROR, "",
                    "ibmf_i_init_wqes(): %s, status = %d\n", tnf_string, msg,
                    "register of WQE mem failed", tnf_uint, status, status);
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_i_init_wqes_end, IBMF_TNF_TRACE, "",
                    "ibmf_i_init_wqes() exit\n");
                return (IBMF_NO_RESOURCES);
        }

        /* Store the memory registration information */
        wqe_mgtp->wqes_ib_mem = mem_desc.md_vaddr;
        wqe_mgtp->wqes_ib_lkey = mem_desc.md_lkey;
        wqe_mgtp->wqes_ib_mem_hdl = mem_hdl;

        /* Create a vmem arena for the IB virtual address space */
        bzero(string, 128);
        (void) sprintf(string, "ibmf_%016" PRIx64 "_wqes", cip->ci_node_guid);
        cip->ci_wqe_ib_vmem = vmem_create(string,
            (void *)(uintptr_t)wqe_mgtp->wqes_ib_mem, wqe_mgtp->wqes_kmem_sz,
            sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP);

        mutex_enter(&cip->ci_wqe_mutex);
        cip->ci_wqe_mgt_list = wqe_mgtp;
        mutex_exit(&cip->ci_wqe_mutex);

        bzero(string, 128);
        (void) sprintf(string, "ibmf_%016" PRIx64 "_swqe", cip->ci_node_guid);
        /* create a kmem cache for the send WQEs */
        cip->ci_send_wqes_cache = kmem_cache_create(string,
            sizeof (ibmf_send_wqe_t), 0, ibmf_send_wqe_cache_constructor,
            ibmf_send_wqe_cache_destructor, NULL, (void *)cip, NULL, 0);

        bzero(string, 128);
        (void) sprintf(string, "ibmf_%016" PRIx64 "_rwqe", cip->ci_node_guid);
        /* create a kmem cache for the receive WQEs */
        cip->ci_recv_wqes_cache = kmem_cache_create(string,
            sizeof (ibmf_recv_wqe_t), 0, ibmf_recv_wqe_cache_constructor,
            ibmf_recv_wqe_cache_destructor, NULL, (void *)cip, NULL, 0);

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_wqes_end,
            IBMF_TNF_TRACE, "", "ibmf_i_init_wqes() exit\n");

        return (IBMF_SUCCESS);
}

/*
 * ibmf_i_fini_wqes():
 *      Destroy the kmem cache for send and receive WQEs
 */
void
ibmf_i_fini_wqes(ibmf_ci_t *cip)
{
        ibmf_wqe_mgt_t  *wqe_mgt;
        ibt_mr_hdl_t    wqe_ib_mem_hdl;
        void            *wqe_kmem;
        uint64_t        wqe_kmem_sz;

        IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_fini_wqes_start,
            IBMF_TNF_TRACE, "", "ibmf_i_fini_wqes() enter, cip = %p\n",
            tnf_opaque, cip, cip);

        mutex_enter(&cip->ci_wqe_mutex);

        wqe_mgt = cip->ci_wqe_mgt_list;
        while (wqe_mgt != NULL) {
                /* Remove the WQE mgt struct from the list */
                cip->ci_wqe_mgt_list = wqe_mgt->wqe_mgt_next;
                mutex_exit(&cip->ci_wqe_mutex);

                mutex_enter(&wqe_mgt->wqes_mutex);
                wqe_ib_mem_hdl = wqe_mgt->wqes_ib_mem_hdl;
                wqe_kmem = wqe_mgt->wqes_kmem;
                wqe_kmem_sz = wqe_mgt->wqes_kmem_sz;
                mutex_exit(&wqe_mgt->wqes_mutex);

                /* Deregister the memory allocated for the WQEs */
                (void) ibt_deregister_mr(cip->ci_ci_handle, wqe_ib_mem_hdl);

                /* Free the kmem allocated for the WQEs */
                kmem_free(wqe_kmem, wqe_kmem_sz);

                /* Destroy the mutex */
                mutex_destroy(&wqe_mgt->wqes_mutex);

                /* Free the WQE management structure */
                kmem_free(wqe_mgt, sizeof (ibmf_wqe_mgt_t));

                mutex_enter(&cip->ci_wqe_mutex);
                wqe_mgt = cip->ci_wqe_mgt_list;
        }

        mutex_exit(&cip->ci_wqe_mutex);

        /* Destroy the kmem_cache for the send WQE */
        kmem_cache_destroy(cip->ci_send_wqes_cache);
        /* Destroy the kmem_cache for the receive WQE */
        kmem_cache_destroy(cip->ci_recv_wqes_cache);

        /*
         * Destroy the vmem arena for the WQEs
         * This must be done after the kmem_cache_destroy() calls since
         * the cache destructors call vmem_free()
         */
        vmem_destroy((void *)cip->ci_wqe_ib_vmem);

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_fini_wqes_end,
            IBMF_TNF_TRACE, "", "ibmf_i_fini_wqes() exit\n");
}

/*
 * ibmf_i_init_altqp_wqes():
 *      Create the kmem cache for send and receive WQEs used by alternate QPs
 */
int
ibmf_i_init_altqp_wqes(ibmf_alt_qp_t *qp_ctx)
{
        ibt_status_t            status;
        ibt_mr_hdl_t            mem_hdl;
        ibt_mr_desc_t           mem_desc;
        ibt_mr_attr_t           mem_attr;
        ibmf_wqe_mgt_t          *wqe_mgtp;
        char                    string[128];

        IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_altqp_wqes_start,
            IBMF_TNF_TRACE, "", "ibmf_i_init_altqp_wqes() enter, qp_ctx = %p\n",
            tnf_opaque, qp, qp_ctx);

        /*
         * Allocate memory for the WQE management structure
         */
        wqe_mgtp = kmem_zalloc(sizeof (ibmf_wqe_mgt_t), KM_SLEEP);
        mutex_init(&wqe_mgtp->wqes_mutex, NULL, MUTEX_DRIVER, NULL);
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqe_mgtp))

        /*
         * Allocate memory for all the WQEs to be used by this alternate QP
         */
        wqe_mgtp->wqes_kmem_sz = (IBMF_MEM_PER_WQE * ibmf_send_wqes_per_port) +
            (IBMF_MEM_PER_WQE * ibmf_recv_wqes_per_port);
        wqe_mgtp->wqes_kmem = kmem_zalloc(wqe_mgtp->wqes_kmem_sz, KM_SLEEP);

        mem_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)wqe_mgtp->wqes_kmem;
        mem_attr.mr_len = wqe_mgtp->wqes_kmem_sz;
        mem_attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
        mem_attr.mr_as = NULL;

        /* Register the allocated memory */
        status = ibt_register_mr(qp_ctx->isq_client_hdl->ic_myci->ci_ci_handle,
            qp_ctx->isq_client_hdl->ic_myci->ci_pd, &mem_attr, &mem_hdl,
            &mem_desc);
        if (status != IBT_SUCCESS) {
                kmem_free(wqe_mgtp->wqes_kmem, wqe_mgtp->wqes_kmem_sz);
                IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_i_init_altqp_wqes_err, IBMF_TNF_ERROR, "",
                    "ibmf_i_init_altqp_wqes(): %s, status = %d\n",
                    tnf_string, msg,
                    "register of WQE mem failed", tnf_uint, status, status);
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_i_init_altqp_wqes_end, IBMF_TNF_TRACE, "",
                    "ibmf_i_init_altqp_wqes() exit\n");
                return (IBMF_NO_RESOURCES);
        }

        /* Store the memory registration information */
        wqe_mgtp->wqes_ib_mem = mem_desc.md_vaddr;
        wqe_mgtp->wqes_ib_lkey = mem_desc.md_lkey;
        wqe_mgtp->wqes_ib_mem_hdl = mem_hdl;

        /* Create a vmem arena for the IB virtual address space */
        bzero(string, 128);
        (void) sprintf(string, "ibmf_%016" PRIx64 "_%x_wqes",
            qp_ctx->isq_client_hdl->ic_client_info.ci_guid, qp_ctx->isq_qpn);
        qp_ctx->isq_wqe_ib_vmem = vmem_create(string,
            (void *)(uintptr_t)wqe_mgtp->wqes_ib_mem, wqe_mgtp->wqes_kmem_sz,
            sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP);

        bzero(string, 128);
        /*
         * CAUTION: Do not exceed 32 characters for the kmem cache name, else,
         * mdb does not exit (bug 4878751). There is some connection between
         * mdb walkers and kmem_caches with the limitation likely to be in the
         * mdb code.
         */
        (void) sprintf(string, "ibmf%016" PRIx64 "_%xs",
            qp_ctx->isq_client_hdl->ic_client_info.ci_guid, qp_ctx->isq_qpn);
        /* create a kmem cache for the send WQEs */
        qp_ctx->isq_send_wqes_cache = kmem_cache_create(string,
            sizeof (ibmf_send_wqe_t), 0, ibmf_altqp_send_wqe_cache_constructor,
            ibmf_altqp_send_wqe_cache_destructor, NULL, (void *)qp_ctx,
            NULL, 0);

        bzero(string, 128);
        (void) sprintf(string, "ibmf%016" PRIx64 "_%xr",
            qp_ctx->isq_client_hdl->ic_client_info.ci_guid, qp_ctx->isq_qpn);
        /* create a kmem cache for the receive WQEs */
        qp_ctx->isq_recv_wqes_cache = kmem_cache_create(string,
            sizeof (ibmf_recv_wqe_t), 0, ibmf_altqp_recv_wqe_cache_constructor,
            ibmf_altqp_recv_wqe_cache_destructor, NULL, (void *)qp_ctx,
            NULL, 0);

        mutex_enter(&qp_ctx->isq_wqe_mutex);
        qp_ctx->isq_wqe_mgt_list = wqe_mgtp;
        mutex_exit(&qp_ctx->isq_wqe_mutex);

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_altqp_wqes_end,
            IBMF_TNF_TRACE, "", "ibmf_i_init_altqp_wqes() exit\n");

        return (IBMF_SUCCESS);
}

/*
 * ibmf_i_fini_altqp_wqes():
 *      Destroy the kmem cache for send and receive WQEs for alternate QPs
 */
void
ibmf_i_fini_altqp_wqes(ibmf_alt_qp_t *qp_ctx)
{
        ibmf_wqe_mgt_t  *wqe_mgt;
        ibt_mr_hdl_t    wqe_ib_mem_hdl;
        void            *wqe_kmem;
        uint64_t        wqe_kmem_sz;

        IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_fini_wqes_start,
            IBMF_TNF_TRACE, "", "ibmf_i_fini_wqes() enter, qp_ctx = %p\n",
            tnf_opaque, qp, qp_ctx);

        mutex_enter(&qp_ctx->isq_wqe_mutex);
        wqe_mgt = qp_ctx->isq_wqe_mgt_list;
        while (wqe_mgt != NULL) {
                /* Remove the WQE mgt struct from the list */
                qp_ctx->isq_wqe_mgt_list = wqe_mgt->wqe_mgt_next;
                mutex_exit(&qp_ctx->isq_wqe_mutex);

                mutex_enter(&wqe_mgt->wqes_mutex);
                wqe_ib_mem_hdl = wqe_mgt->wqes_ib_mem_hdl;
                wqe_kmem = wqe_mgt->wqes_kmem;
                wqe_kmem_sz = wqe_mgt->wqes_kmem_sz;
                mutex_exit(&wqe_mgt->wqes_mutex);

                /* Deregister the memory allocated for the WQEs */
                (void) ibt_deregister_mr(
                    qp_ctx->isq_client_hdl->ic_myci->ci_ci_handle,
                    wqe_ib_mem_hdl);

                /* Free the kmem allocated for the WQEs */
                kmem_free(wqe_kmem, wqe_kmem_sz);

                /* Destroy the WQE mgt struct mutex */
                mutex_destroy(&wqe_mgt->wqes_mutex);

                /* Free the WQE management structure */
                kmem_free(wqe_mgt, sizeof (ibmf_wqe_mgt_t));

                mutex_enter(&qp_ctx->isq_wqe_mutex);
                wqe_mgt = qp_ctx->isq_wqe_mgt_list;
        }

        mutex_exit(&qp_ctx->isq_wqe_mutex);

        /* Destroy the kmem_cache for the send WQE */
        kmem_cache_destroy(qp_ctx->isq_send_wqes_cache);
        /* Destroy the kmem_cache for the receive WQE */
        kmem_cache_destroy(qp_ctx->isq_recv_wqes_cache);

        /*
         * Destroy the vmem arena for the WQEs
         * This must be done after the kmem_cache_destroy() calls since
         * the cache destructors call vmem_free()
         */
        vmem_destroy((void *)qp_ctx->isq_wqe_ib_vmem);

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_fini_wqes_end,
            IBMF_TNF_TRACE, "", "ibmf_i_fini_wqes() exit\n");
}

/*
 * ibmf_i_init_send_wqe():
 *      Initialize a send WQE
 */
/* ARGSUSED */
void
ibmf_i_init_send_wqe(ibmf_client_t *clientp, ibmf_msg_impl_t *msgimplp,
    ibt_wr_ds_t *sglp, ibmf_send_wqe_t *wqep, ibt_ud_dest_hdl_t ud_dest,
    ibt_qp_hdl_t ibt_qp_handle, ibmf_qp_handle_t ibmf_qp_handle)
{
        ibmf_msg_bufs_t *ipbufs = &msgimplp->im_msgbufs_send;
        ibmf_msg_bufs_t *hdr_ipbufs;
        ib_mad_hdr_t    *ibmadhdrp;
        ibmf_rmpp_ctx_t *rmpp_ctx = &msgimplp->im_rmpp_ctx;
        ibmf_rmpp_hdr_t *rmpp_hdr;
        ibt_send_wr_t   *swrp;
        uchar_t         *buf;
        size_t          data_sz, offset;
        uint32_t        cl_hdr_sz, cl_hdr_off;

        IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_send_wqe_start,
            IBMF_TNF_TRACE, "", "ibmf_i_init_send_wqe() enter, "
            "clientp = %p, msg = %p, sglp = %p , wqep = %p, qp_hdl = %p\n",
            tnf_opaque, clientp, clientp, tnf_opaque, msg, msgimplp,
            tnf_opaque, sglp, sglp, tnf_opaque, wqep, wqep,
            tnf_opaque, qp_hdl, ibmf_qp_handle);

        _NOTE(ASSUMING_PROTECTED(*wqep))
        _NOTE(ASSUMING_PROTECTED(*sglp))
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*swrp))

        swrp = &wqep->send_wr;
        /* use send wqe pointer as the WR ID */
        IBMF_ADDR_TO_SEND_WR_ID(wqep, swrp->wr_id);
        ASSERT(swrp->wr_id != 0);
        swrp->wr_flags = IBT_WR_NO_FLAGS;
        swrp->wr_opcode = IBT_WRC_SEND;
        swrp->wr_trans = IBT_UD_SRV;
        wqep->send_client = clientp;
        wqep->send_msg = msgimplp;

        IBMF_INIT_SG_ELEMENT(sglp[0], wqep->send_mem, wqep->send_sg_lkey,
            IBMF_MAD_SIZE);

        bzero(wqep->send_mem, IBMF_MAD_SIZE);
        if (msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP) {
                buf = (uchar_t *)ipbufs->im_bufs_cl_data +
                    (rmpp_ctx->rmpp_ns - 1) * rmpp_ctx->rmpp_pkt_data_sz;
                data_sz = (rmpp_ctx->rmpp_ns == rmpp_ctx->rmpp_num_pkts) ?
                    rmpp_ctx->rmpp_last_pkt_sz : rmpp_ctx->rmpp_pkt_data_sz;
        } else {
                buf = ipbufs->im_bufs_cl_data;
                data_sz = ipbufs->im_bufs_cl_data_len;
        }

        /*
         * We pick the correct msgbuf based on the nature of the transaction.
         * Where the send msgbuf is available, we pick it to provide the
         * context of the outgoing MAD. Note that if this is a termination
         * context, then  the send buffer is invalid even if the sequenced
         * flags is set because the termination message only has a receive
         * buffer set up.
         */
        if ((msgimplp->im_flags & IBMF_MSG_FLAGS_SEQUENCED) &&
            ((msgimplp->im_flags & IBMF_MSG_FLAGS_TERMINATION) == 0)) {
                hdr_ipbufs = &msgimplp->im_msgbufs_send;
        } else if (msgimplp->im_flags & IBMF_MSG_FLAGS_RECV_RMPP) {
                hdr_ipbufs = &msgimplp->im_msgbufs_recv;
        } else if (msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP) {
                hdr_ipbufs = &msgimplp->im_msgbufs_send;
        } else {
                if (msgimplp->im_unsolicited == B_TRUE) {
                        hdr_ipbufs = &msgimplp->im_msgbufs_recv;
                } else {
                        hdr_ipbufs = &msgimplp->im_msgbufs_send;
                }
        }

        bcopy((void *)hdr_ipbufs->im_bufs_mad_hdr,
            (void *)wqep->send_mem, sizeof (ib_mad_hdr_t));

        /*
         * For unsolicited messages, we only have the sender's MAD at hand.
         * So, we must flip the response bit in the method for the outgoing MAD.
         */
        ibmadhdrp = (ib_mad_hdr_t *)wqep->send_mem;
        if (msgimplp->im_unsolicited == B_TRUE) {
                ibmadhdrp->R_Method = IBMF_FLIP_RESP_BIT(ibmadhdrp->R_Method);
        }

        offset = sizeof (ib_mad_hdr_t);

        if ((msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP) ||
            (msgimplp->im_flags & IBMF_MSG_FLAGS_RECV_RMPP)) {

                rmpp_hdr = (ibmf_rmpp_hdr_t *)
                    ((uintptr_t)wqep->send_mem + offset);

                _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rmpp_hdr));

                IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_init_send_wqe,
                    IBMF_TNF_TRACE, "",
                    "ibmf_init_send_wqe: msgimplp = %p, rmpp_type = %d,"
                    " next_seg = %d, num_pkts = %d\n",
                    tnf_opaque, msgimplp, msgimplp,
                    tnf_opaque, rmpp_type, rmpp_ctx->rmpp_type,
                    tnf_opaque, next_seg, rmpp_ctx->rmpp_ns,
                    tnf_opaque, num_pkts, rmpp_ctx->rmpp_num_pkts);

                /*
                 * Initialize the RMPP header
                 */
                rmpp_ctx->rmpp_flags = IBMF_RMPP_FLAGS_ACTIVE;

                /* first, last packet flags set only for type DATA */
                if (rmpp_ctx->rmpp_type == IBMF_RMPP_TYPE_DATA) {

                        if (rmpp_ctx->rmpp_ns == 1)
                                rmpp_ctx->rmpp_flags |=
                                    IBMF_RMPP_FLAGS_FIRST_PKT;
                        else
                                rmpp_ctx->rmpp_respt = IBMF_RMPP_DEFAULT_RRESPT;

                        if (rmpp_ctx->rmpp_ns == rmpp_ctx->rmpp_num_pkts)
                                rmpp_ctx->rmpp_flags |=
                                    IBMF_RMPP_FLAGS_LAST_PKT;
                } else {
                        data_sz = 0;
                        rmpp_ctx->rmpp_respt = IBMF_RMPP_TERM_RRESPT;
                }

                IBMF_INIT_RMPP_HDR(rmpp_hdr,
                    IBMF_RMPP_VERSION, rmpp_ctx->rmpp_type,
                    rmpp_ctx->rmpp_respt, rmpp_ctx->rmpp_flags,
                    rmpp_ctx->rmpp_status, rmpp_ctx->rmpp_word3,
                    rmpp_ctx->rmpp_word4)

                IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_init_send_wqe,
                    IBMF_TNF_TRACE, "",
                    "ibmf_init_send_wqe: msgimplp = %p, rmpp_type = %d,"
                    " rmpp_flags = 0x%x, rmpp_segnum = %d, pyld_nwl = %d\n",
                    tnf_opaque, msgimplp, msgimplp,
                    tnf_opaque, rmpp_type, rmpp_hdr->rmpp_type,
                    tnf_opaque, rmpp_flags, rmpp_hdr->rmpp_flags,
                    tnf_opaque, rmpp_segnum, b2h32(rmpp_hdr->rmpp_segnum),
                    tnf_opaque, pyld_nwl, b2h32(rmpp_hdr->rmpp_pyldlen_nwl));

                _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(rmpp_hdr));
        }

        /* determine offset to start class header */
        ibmf_i_mgt_class_to_hdr_sz_off(
            hdr_ipbufs->im_bufs_mad_hdr->MgmtClass,
            &cl_hdr_sz, &cl_hdr_off);
        offset += cl_hdr_off;
        if (hdr_ipbufs->im_bufs_cl_hdr != NULL) {
                bcopy((void *)hdr_ipbufs->im_bufs_cl_hdr,
                    (void *)((uintptr_t)wqep->send_mem + offset),
                    hdr_ipbufs->im_bufs_cl_hdr_len);
                offset += hdr_ipbufs->im_bufs_cl_hdr_len;
        }
        bcopy((void *)buf, (void *)((uintptr_t)wqep->send_mem + offset),
            data_sz);
        swrp->wr_sgl = sglp;
        swrp->wr_nds = 1;
        swrp->wr.ud.udwr_dest = ud_dest;
        wqep->send_port_num = clientp->ic_client_info.port_num;
        wqep->send_qp_handle = ibt_qp_handle;
        wqep->send_ibmf_qp_handle = ibmf_qp_handle;

        _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*swrp))

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_send_wqe_end,
            IBMF_TNF_TRACE, "", "ibmf_i_init_send_wqe() exit\n");
}

/*
 * ibmf_i_init_recv_wqe():
 *      Initialize a receive WQE
 */
void
ibmf_i_init_recv_wqe(ibmf_qp_t *qpp, ibt_wr_ds_t *sglp,
    ibmf_recv_wqe_t *wqep, ibt_qp_hdl_t ibt_qp_handle,
    ibmf_qp_handle_t ibmf_qp_handle)
{
        ibt_recv_wr_t           *rwrp;

        IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_recv_wqe_start,
            IBMF_TNF_TRACE, "", "ibmf_i_init_recv_wqe() enter, "
            "qpp = %p, sglp = %p , wqep = %p, ud_dest = %p, qp_hdl = %p\n",
            tnf_opaque, qpp, qpp, tnf_opaque, sglp, sglp, tnf_opaque,
            wqep, wqep, tnf_opaque, qp_hdl, ibmf_qp_handle);

        _NOTE(ASSUMING_PROTECTED(*wqep))
        _NOTE(ASSUMING_PROTECTED(*sglp))
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rwrp))

        rwrp = &wqep->recv_wr;

        /*
         * we set a bit in the WR ID to be able to easily distinguish
         * between send completions and recv completions
         */
        IBMF_ADDR_TO_RECV_WR_ID(wqep, rwrp->wr_id);

        IBMF_INIT_SG_ELEMENT(sglp[0], wqep->recv_mem, wqep->recv_sg_lkey,
            sizeof (ib_grh_t) + IBMF_MAD_SIZE);

        rwrp->wr_sgl = sglp;
        rwrp->wr_nds = IBMF_MAX_RQ_WR_SGL_ELEMENTS;
        if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
                wqep->recv_port_num = qpp->iq_port_num;
        } else {
                ibmf_alt_qp_t   *altqp = (ibmf_alt_qp_t *)ibmf_qp_handle;
                wqep->recv_port_num = altqp->isq_port_num;
        }
        wqep->recv_qpp = qpp;
        wqep->recv_qp_handle = ibt_qp_handle;
        wqep->recv_ibmf_qp_handle = ibmf_qp_handle;

        _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*rwrp))

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_recv_wqe_end,
            IBMF_TNF_TRACE, "", "ibmf_i_init_recv_wqe() exit\n");
}

/*
 * ibmf_i_extend_wqe_cache():
 *      Extend the kmem WQE cache
 */
int
ibmf_i_extend_wqe_cache(ibmf_ci_t *cip, ibmf_qp_handle_t ibmf_qp_handle,
    boolean_t block)
{
        ibmf_wqe_mgt_t          *wqe_mgt;

        IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_extend_wqe_cache_start, IBMF_TNF_TRACE, "",
            "ibmf_i_extend_wqe_cache() enter, cip = %p, qp_hdl = %p, "
            " block = %d\n", tnf_opaque, cip, cip, tnf_opaque, qp_hdl,
            ibmf_qp_handle, tnf_uint, block, block);

        /*
         * Allocate memory for the WQE management structure
         */
        wqe_mgt = kmem_zalloc(sizeof (ibmf_wqe_mgt_t),
            (block == B_TRUE ? KM_SLEEP : KM_NOSLEEP));
        if (wqe_mgt == NULL) {
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_i_extend_wqe_cache_err, IBMF_TNF_ERROR, "",
                    "ibmf_i_extend_wqe_cache(): %s\n",
                    tnf_string, msg, "wqe mgt alloc failed");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_i_extend_wqe_cache_end, IBMF_TNF_TRACE, "",
                    "ibmf_i_extend_wqe_cache() exit\n");
                return (IBMF_NO_RESOURCES);
        }
        mutex_init(&wqe_mgt->wqes_mutex, NULL, MUTEX_DRIVER, NULL);
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqe_mgt))

        /* Allocate and register more WQE memory */
        if (ibmf_i_extend_wqe_mem(cip, ibmf_qp_handle, wqe_mgt,
            block) != IBMF_SUCCESS) {
                mutex_destroy(&wqe_mgt->wqes_mutex);
                kmem_free(wqe_mgt, sizeof (ibmf_wqe_mgt_t));
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_i_extend_wqe_cache_err, IBMF_TNF_ERROR, "",
                    "ibmf_i_extend_wqe_cache(): %s\n",
                    tnf_string, msg, "extension of WQE pool failed");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_i_extend_wqe_cache_end, IBMF_TNF_TRACE, "",
                    "ibmf_i_extend_wqe_cache() exit\n");
                return (IBMF_NO_RESOURCES);
        }

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_extend_wqe_cache_end, IBMF_TNF_TRACE, "",
            "ibmf_i_extend_wqe_cache() exit\n");

        return (IBMF_SUCCESS);
}

/*
 * ibmf_i_extend_wqe_mem():
 *      Allocate and register more WQE memory, and expand the VMEM arena
 */
static int
ibmf_i_extend_wqe_mem(ibmf_ci_t *cip, ibmf_qp_handle_t ibmf_qp_handle,
    ibmf_wqe_mgt_t *wqe_mgt, boolean_t block)
{
        ibt_status_t            status;
        ibt_mr_hdl_t            mem_hdl;
        ibt_mr_desc_t           mem_desc;
        ibt_mr_attr_t           mem_attr;
        ibmf_alt_qp_t           *qp_ctx;
        ibmf_wqe_mgt_t          *pwqe_mgt;
        vmem_t                  *wqe_vmem_arena;

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqe_mgt))

        IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_extend_wqe_cache_start, IBMF_TNF_TRACE, "",
            "ibmf_i_extend_wqe_cache() enter, cip = %p, qp_hdl = %p"
            "wqe_mgt = %p, block = %d\n",
            tnf_opaque, cip, cip, tnf_opaque, qp_hdl, ibmf_qp_handle,
            tnf_opaque, wqe_mgt, wqe_mgt, tnf_uint, block, block);

        /*
         * Allocate more memory for the WQEs to be used by the
         * specified QP
         */
        wqe_mgt->wqes_kmem_sz = cip->ci_nports * 2 *
            ((IBMF_MEM_PER_WQE * ibmf_send_wqes_per_port) +
            (IBMF_MEM_PER_WQE * ibmf_recv_wqes_per_port));
        wqe_mgt->wqes_kmem = kmem_zalloc(wqe_mgt->wqes_kmem_sz,
            (block == B_TRUE ? KM_SLEEP : KM_NOSLEEP));
        if (wqe_mgt->wqes_kmem == NULL) {
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_i_extend_wqe_mem_err, IBMF_TNF_ERROR, "",
                    "ibmf_i_extend_wqe_mem(): %s\n",
                    tnf_string, msg, "extension of WQE pool failed");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_i_extend_wqe_mem_end, IBMF_TNF_TRACE, "",
                    "ibmf_i_extend_wqe_mem() exit\n");
                return (IBMF_NO_RESOURCES);
        }

        mem_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)wqe_mgt->wqes_kmem;
        mem_attr.mr_len = wqe_mgt->wqes_kmem_sz;
        mem_attr.mr_flags = (block == B_TRUE ? IBT_MR_SLEEP : IBT_MR_NOSLEEP)
            | IBT_MR_ENABLE_LOCAL_WRITE;
        mem_attr.mr_as = NULL;

        /* Register the allocated memory */
        status = ibt_register_mr(cip->ci_ci_handle, cip->ci_pd,
            &mem_attr, &mem_hdl, &mem_desc);
        if (status != IBT_SUCCESS) {
                kmem_free(wqe_mgt->wqes_kmem, wqe_mgt->wqes_kmem_sz);
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_i_extend_wqe_mem_err, IBMF_TNF_ERROR, "",
                    "ibmf_i_extend_wqe_mem(): %s\n",
                    tnf_string, msg, "wqe extension MR failed");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_i_extend_wqe_mem_end, IBMF_TNF_TRACE, "",
                    "ibmf_i_extend_wqe_mem() exit\n");
                return (IBMF_NO_RESOURCES);
        }

        /* Store the memory registration information */
        wqe_mgt->wqes_ib_mem = mem_desc.md_vaddr;
        wqe_mgt->wqes_ib_lkey = mem_desc.md_lkey;
        wqe_mgt->wqes_ib_mem_hdl = mem_hdl;

        /* Get the VMEM arena based on the QP type */
        if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
                wqe_vmem_arena = cip->ci_wqe_ib_vmem;
        } else {
                qp_ctx = (ibmf_alt_qp_t *)ibmf_qp_handle;
                wqe_vmem_arena = qp_ctx->isq_wqe_ib_vmem;
        }

        /* Add these addresses to the vmem arena */
        if (vmem_add(wqe_vmem_arena, (void *)(uintptr_t)wqe_mgt->wqes_ib_mem,
            wqe_mgt->wqes_kmem_sz,
            (block == B_TRUE ? VM_SLEEP : VM_NOSLEEP)) == NULL) {
                (void) ibt_deregister_mr(cip->ci_ci_handle,
                    wqe_mgt->wqes_ib_mem_hdl);
                kmem_free(wqe_mgt->wqes_kmem, wqe_mgt->wqes_kmem_sz);
                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_i_extend_wqe_mem_err, IBMF_TNF_ERROR, "",
                    "ibmf_i_extend_wqe_mem(): %s\n",
                    tnf_string, msg, "wqe extension vmem_add failed");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_i_extend_wqe_mem_end, IBMF_TNF_TRACE, "",
                    "ibmf_i_extend_wqe_mem() exit\n");
                return (IBMF_NO_RESOURCES);
        }

        /* Get the WQE management pointers based on the QP type */
        if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
                mutex_enter(&cip->ci_wqe_mutex);
                pwqe_mgt = cip->ci_wqe_mgt_list;

                /* Add the new wqe management struct to the end of the list */
                while (pwqe_mgt->wqe_mgt_next != NULL)
                        pwqe_mgt = pwqe_mgt->wqe_mgt_next;
                pwqe_mgt->wqe_mgt_next = wqe_mgt;

                mutex_exit(&cip->ci_wqe_mutex);
        } else {
                mutex_enter(&qp_ctx->isq_wqe_mutex);
                pwqe_mgt = qp_ctx->isq_wqe_mgt_list;

                /* Add the new wqe management struct to the end of the list */
                while (pwqe_mgt->wqe_mgt_next != NULL)
                        pwqe_mgt = pwqe_mgt->wqe_mgt_next;
                pwqe_mgt->wqe_mgt_next = wqe_mgt;

                mutex_exit(&qp_ctx->isq_wqe_mutex);
        }

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_extend_wqe_mem_end,
            IBMF_TNF_TRACE, "", "ibmf_i_extend_wqe_mem() exit\n");

        return (IBMF_SUCCESS);
}

/*
 * ibmf_i_alloc_send_resources():
 *      Allocate send resources (the send WQE)
 */
int
ibmf_i_alloc_send_resources(ibmf_ci_t *cip, ibmf_msg_impl_t *msgimplp,
    boolean_t block, ibmf_send_wqe_t **swqepp)
{
        ibmf_send_wqe_t         *send_wqep;
        struct kmem_cache       *kmem_cachep;
        ibmf_qp_handle_t        ibmf_qp_handle = msgimplp->im_qp_hdl;
        ibmf_alt_qp_t           *altqp;

        IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_alloc_send_resources_start, IBMF_TNF_TRACE, "",
            "ibmf_i_alloc_send_resources() enter, cip = %p, msg = %p, "
            " block = %d\n", tnf_opaque, cip, cip, tnf_opaque, msg,
            msgimplp, tnf_uint, block, block);

        /* Get the WQE kmem cache pointer based on the QP type */
        if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT)
                kmem_cachep = cip->ci_send_wqes_cache;
        else {
                altqp = (ibmf_alt_qp_t *)ibmf_qp_handle;
                kmem_cachep = altqp->isq_send_wqes_cache;
        }

        /*
         * Allocate a send WQE from the send WQE kmem cache
         * Do not block here as we are holding the msgimpl mutex.
         */
        send_wqep = kmem_cache_alloc(kmem_cachep, KM_NOSLEEP);
        if (send_wqep == NULL) {
                /*
                 * Attempt to extend the cache and then retry the
                 * kmem_cache_alloc()
                 * The block argument (third) is set to B_FALSE.
                 */
                if (ibmf_i_extend_wqe_cache(cip, ibmf_qp_handle, B_FALSE) ==
                    IBMF_NO_RESOURCES) {
                        mutex_enter(&cip->ci_mutex);
                        IBMF_ADD32_PORT_KSTATS(cip, swqe_allocs_failed, 1);
                        mutex_exit(&cip->ci_mutex);
                        IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                            ibmf_i_alloc_send_resources_err, IBMF_TNF_ERROR, "",
                            "ibmf_i_alloc_send_resources(): %s\n",
                            tnf_string, msg, "alloc send_wqe failed");
                        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                            ibmf_i_alloc_send_resources_end, IBMF_TNF_TRACE, "",
                            "ibmf_i_alloc_send_resources() exit\n");
                        return (IBMF_NO_RESOURCES);
                } else {
                        send_wqep = kmem_cache_alloc(kmem_cachep, KM_NOSLEEP);
                        if (send_wqep == NULL) {
                                /* Allocation failed again. Give up here. */
                                mutex_enter(&cip->ci_mutex);
                                IBMF_ADD32_PORT_KSTATS(cip, swqe_allocs_failed,
                                    1);
                                mutex_exit(&cip->ci_mutex);
                                IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
                                    ibmf_i_alloc_send_resources_err,
                                    IBMF_TNF_ERROR, "",
                                    "ibmf_i_alloc_send_resources(): %s\n",
                                    tnf_string, msg, "alloc send_wqe failed");
                                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                                    ibmf_i_alloc_send_resources_end,
                                    IBMF_TNF_TRACE, "",
                                    "ibmf_i_alloc_send_resources() exit\n");
                                return (IBMF_NO_RESOURCES);
                        }
                }
        }

        mutex_enter(&cip->ci_mutex);
        IBMF_ADD32_PORT_KSTATS(cip, send_wqes_alloced, 1);
        mutex_exit(&cip->ci_mutex);
        if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
                mutex_enter(&cip->ci_mutex);
                cip->ci_wqes_alloced++;
                mutex_exit(&cip->ci_mutex);
        } else {
                mutex_enter(&altqp->isq_mutex);
                altqp->isq_wqes_alloced++;
                mutex_exit(&altqp->isq_mutex);
        }

        send_wqep->send_msg = msgimplp;
        *swqepp = send_wqep;

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_alloc_send_resources_end, IBMF_TNF_TRACE, "",
            "ibmf_i_alloc_send_resources() exit\n");

        return (IBMF_SUCCESS);
}

/*
 * ibmf_i_free_send_resources():
 *      Free send resources (just the send WQE)
 */
/* ARGSUSED */
void
ibmf_i_free_send_resources(ibmf_ci_t *cip, ibmf_msg_impl_t *msgimplp,
    ibmf_send_wqe_t *swqep)
{
        struct kmem_cache       *kmem_cachep;
        ibmf_qp_handle_t        ibmf_qp_handle = msgimplp->im_qp_hdl;
        ibmf_alt_qp_t           *altqp;

        IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_free_send_resources_start, IBMF_TNF_TRACE, "",
            "ibmf_i_free_send_resources() enter, cip = %p, msg = %p, "
            " swqep = %p\n", tnf_opaque, cip, cip, tnf_opaque, msg,
            msgimplp, tnf_opaque, swqep, swqep);

        /* Get the WQE kmem cache pointer based on the QP type */
        if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT)
                kmem_cachep = cip->ci_send_wqes_cache;
        else {
                altqp = (ibmf_alt_qp_t *)ibmf_qp_handle;
                kmem_cachep = altqp->isq_send_wqes_cache;
        }

        /* return the send WQE to the kmem cache */
        kmem_cache_free(kmem_cachep, swqep);

        mutex_enter(&cip->ci_mutex);
        IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1);
        mutex_exit(&cip->ci_mutex);
        if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
                mutex_enter(&cip->ci_mutex);
                cip->ci_wqes_alloced--;
                if (cip->ci_wqes_alloced == 0)
                        cv_signal(&cip->ci_wqes_cv);
                mutex_exit(&cip->ci_mutex);
        } else {
                mutex_enter(&altqp->isq_mutex);
                altqp->isq_wqes_alloced--;
                if (altqp->isq_wqes_alloced == 0)
                        cv_signal(&altqp->isq_wqes_cv);
                mutex_exit(&altqp->isq_mutex);
        }

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_free_send_resources_end, IBMF_TNF_TRACE, "",
            "ibmf_i_free_send_resources() exit\n");
}