/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * This file implements the MAD send logic in IBMF.
 */

#include <sys/ib/mgt/ibmf/ibmf_impl.h>

#define IBMF_SEND_WR_ID_TO_ADDR(id, ptr)                \
        (ptr) = (void *)(uintptr_t)(id)

extern int ibmf_trace_level;

static void ibmf_i_do_send_cb(void *taskq_arg);
static void ibmf_i_do_send_compl(ibmf_handle_t ibmf_handle,
    ibmf_msg_impl_t *msgimplp, ibmf_send_wqe_t *send_wqep);

/*
 * ibmf_i_issue_pkt():
 *      Post an IB packet on the specified QP's send queue
 */
int
ibmf_i_issue_pkt(ibmf_client_t *clientp, ibmf_msg_impl_t *msgimplp,
    ibmf_qp_handle_t ibmf_qp_handle, ibmf_send_wqe_t *send_wqep)
{
        int                     ret;
        ibt_status_t            status;
        ibt_wr_ds_t             sgl[1];
        ibt_qp_hdl_t            ibt_qp_handle;

        _NOTE(ASSUMING_PROTECTED(*send_wqep))
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep))

        IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_issue_pkt_start, IBMF_TNF_TRACE, "",
            "ibmf_i_issue_pkt() enter, clientp = %p, msg = %p, "
            "qp_hdl = %p,  swqep = %p\n", tnf_opaque, clientp, clientp,
            tnf_opaque, msg, msgimplp, tnf_opaque, ibmf_qp_handle,
            ibmf_qp_handle, tnf_opaque, send_wqep, send_wqep);

        ASSERT(MUTEX_HELD(&msgimplp->im_mutex));
        ASSERT(MUTEX_NOT_HELD(&clientp->ic_mutex));

        /*
         * if the qp handle provided in ibmf_send_pkt()
         * is not the default qp handle for this client,
         * then the wqe must be sent on this qp,
         * else use the default qp handle set up during ibmf_register()
         */
        if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
                ibt_qp_handle = clientp->ic_qp->iq_qp_handle;
        } else {
                ibt_qp_handle =
                    ((ibmf_alt_qp_t *)ibmf_qp_handle)->isq_qp_handle;
        }

        /* initialize the send WQE */
        ibmf_i_init_send_wqe(clientp, msgimplp, sgl, send_wqep,
            msgimplp->im_ud_dest, ibt_qp_handle, ibmf_qp_handle);

        _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep))

        /*
         * Issue the wqe to the transport.
         * NOTE: ibt_post_send() will not block, so, it is ok
         * to hold the msgimpl mutex across this call.
         */
        status = ibt_post_send(send_wqep->send_qp_handle, &send_wqep->send_wr,
            1, NULL);
        if (status != IBT_SUCCESS) {
                mutex_enter(&clientp->ic_kstat_mutex);
                IBMF_ADD32_KSTATS(clientp, send_pkt_failed, 1);
                mutex_exit(&clientp->ic_kstat_mutex);
                IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_i_issue_pkt_err, IBMF_TNF_TRACE, "",
                    "ibmf_i_issue_pkt(): %s, status = %d\n",
                    tnf_string, msg, "post send failure",
                    tnf_uint, ibt_status, status);
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_issue_pkt_end,
                    IBMF_TNF_TRACE, "", "ibmf_i_issue_pkt(() exit\n");
                return (IBMF_TRANSPORT_FAILURE);
        }

        ret = IBMF_SUCCESS;

        /* bump the number of active sends */
        if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
                mutex_enter(&clientp->ic_mutex);
                clientp->ic_sends_active++;
                mutex_exit(&clientp->ic_mutex);
                mutex_enter(&clientp->ic_kstat_mutex);
                IBMF_ADD32_KSTATS(clientp, sends_active, 1);
                mutex_exit(&clientp->ic_kstat_mutex);
        } else {
                ibmf_alt_qp_t *qpp = (ibmf_alt_qp_t *)ibmf_qp_handle;
                mutex_enter(&qpp->isq_mutex);
                qpp->isq_sends_active++;
                mutex_exit(&qpp->isq_mutex);
                mutex_enter(&clientp->ic_kstat_mutex);
                IBMF_ADD32_KSTATS(clientp, sends_active, 1);
                mutex_exit(&clientp->ic_kstat_mutex);
        }

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_issue_pkt_end,
            IBMF_TNF_TRACE, "", "ibmf_i_issue_pkt() exit\n");
        return (ret);
}

/*
 * ibmf_i_send_pkt()
 *      Send an IB packet after allocating send resources
 */
int
ibmf_i_send_pkt(ibmf_client_t *clientp, ibmf_qp_handle_t ibmf_qp_handle,
    ibmf_msg_impl_t *msgimplp, int block)
{
        ibmf_send_wqe_t *send_wqep;
        int             status;

        IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_start,
            IBMF_TNF_TRACE, "",
            "ibmf_i_send_pkt(): clientp = 0x%p, qp_hdl = 0x%p, "
            "msgp = 0x%p, block = %d\n", tnf_opaque, clientp, clientp,
            tnf_opaque, qp_hdl, ibmf_qp_handle, tnf_opaque, msg, msgimplp,
            tnf_uint, block, block);

        ASSERT(MUTEX_HELD(&msgimplp->im_mutex));

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep))

        /*
         * Reset send_done to indicate we have not received the completion
         * for this send yet.
         */
        msgimplp->im_trans_state_flags &= ~IBMF_TRANS_STATE_FLAG_SEND_DONE;

        /*
         * Allocate resources needed to send a UD packet including the
         * send WQE context
         */
        status = ibmf_i_alloc_send_resources(clientp->ic_myci,
            msgimplp, block, &send_wqep);
        if (status != IBMF_SUCCESS) {
                IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_send_pkt_err,
                    IBMF_TNF_ERROR, "", "ibmf_i_send_pkt(): %s, status = %d\n",
                    tnf_string, msg, "unable to allocate send resources",
                    tnf_uint, status, status);
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_end,
                    IBMF_TNF_TRACE, "", "ibmf_i_send_pkt() exit\n");
                return (status);
        }

        /* Set the segment number in the send WQE context */
        if (msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP)
                send_wqep->send_rmpp_segment = msgimplp->im_rmpp_ctx.rmpp_ns;

        _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep))

        /*
         * Increment the count of pending send completions.
         * Only when this count is zero should the client be notified
         * of completion of the transaction.
         */
        msgimplp->im_pending_send_compls += 1;

        /* Send the packet */
        status = ibmf_i_issue_pkt(clientp, msgimplp, ibmf_qp_handle, send_wqep);
        if (status != IBMF_SUCCESS) {
                ibmf_i_free_send_resources(clientp->ic_myci, msgimplp,
                    send_wqep);
                IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_send_pkt_err,
                    IBMF_TNF_ERROR, "", "ibmf_i_send_pkt(): %s, status = %d\n",
                    tnf_string, msg, "unable to issue packet",
                    tnf_uint, status, status);
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_end,
                    IBMF_TNF_TRACE, "", "ibmf_i_send_pkt() exit\n");
                return (status);
        }

        IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_end,
            IBMF_TNF_TRACE, "", "ibmf_i_send_pkt() exit, status = %d\n",
            tnf_uint, status, status);

        return (IBMF_SUCCESS);
}

/*
 * ibmf_i_send_single_pkt():
 *      Send a single IB packet.  Only used to send non-RMPP packets.
 */
int
ibmf_i_send_single_pkt(ibmf_client_t *clientp, ibmf_qp_handle_t ibmf_qp_handle,
    ibmf_msg_impl_t *msgimplp, int block)
{
        int     status;

        IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_single_pkt_start,
            IBMF_TNF_TRACE, "",
            "ibmf_i_send_single_pkt(): clientp = 0x%p, qp_hdl = 0x%p, "
            "msgp = 0x%p, block = %d\n", tnf_opaque, clientp, clientp,
            tnf_opaque, qp_hdl, ibmf_qp_handle, tnf_opaque, msg, msgimplp,
            tnf_uint, block, block);

        ASSERT(MUTEX_HELD(&msgimplp->im_mutex));

        status = ibmf_i_send_pkt(clientp, ibmf_qp_handle, msgimplp, block);
        if (status != IBMF_SUCCESS) {
                IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
                    ibmf_i_send_single_pkt_err, IBMF_TNF_ERROR, "",
                    "ibmf_i_send_single_pkt(): %s, msgp = 0x%p\n",
                    tnf_string, msg, "unable to send packet",
                    tnf_uint, status, status);
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_i_send_single_pkt_end, IBMF_TNF_TRACE, "",
                    "ibmf_i_send_single_pkt() exit\n");
                return (status);
        }

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_single_pkt_end,
            IBMF_TNF_TRACE, "", "ibmf_i_send_single_pkt() exit\n");
        return (IBMF_SUCCESS);
}

/*
 * ibmf_i_handle_send_completion():
 *      Process the WQE from the SQ identified in the work completion entry.
 */
/* ARGSUSED */
void
ibmf_i_handle_send_completion(ibmf_ci_t *cip, ibt_wc_t *wcp)
{
        ibmf_client_t           *clientp, *cclientp;
        ibmf_send_wqe_t         *send_wqep;
        ibmf_qp_handle_t        ibmf_qp_handle;
        ibmf_alt_qp_t           *qpp;
        int                     ret;

        IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_handle_send_completion_start, IBMF_TNF_TRACE, "",
            "ibmf_i_handle_send_completion() enter, cip = %p, wcp = %p\n",
            tnf_opaque, cip, cip, tnf_opaque, wcp, wcp);

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep))

        ASSERT(wcp->wc_id != 0);

        ASSERT(IBMF_IS_SEND_WR_ID(wcp->wc_id));

        /* get the IBMF send WQE context */
        IBMF_SEND_WR_ID_TO_ADDR(wcp->wc_id, send_wqep);

        ASSERT(send_wqep != NULL);

        /* get the client context */
        cclientp =  clientp = send_wqep->send_client;

        /* Check if this is a completion for a BUSY MAD sent by IBMF */
        if (clientp == NULL) {
                ibmf_msg_impl_t         *msgimplp;

                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3,
                    ibmf_i_handle_send_completion, IBMF_TNF_TRACE, "",
                    "ibmf_i_handle_send_completion(): NULL client\n");

                msgimplp = send_wqep->send_msg;

                /*
                 * Deregister registered memory and free it, and
                 * free up the send WQE context
                 */
                (void) ibt_deregister_mr(cip->ci_ci_handle,
                    send_wqep->send_mem_hdl);
                kmem_free(send_wqep->send_mem, IBMF_MEM_PER_WQE);
                kmem_free(send_wqep, sizeof (ibmf_send_wqe_t));

                /* Free up the message context */
                ibmf_i_put_ud_dest(cip, msgimplp->im_ibmf_ud_dest);
                ibmf_i_clean_ud_dest_list(cip, B_FALSE);
                kmem_free(msgimplp, sizeof (ibmf_msg_impl_t));
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "",
                    "ibmf_i_handle_send_completion() exit\n");
                return;
        }

        /* get the QP handle */
        ibmf_qp_handle = send_wqep->send_ibmf_qp_handle;
        qpp = (ibmf_alt_qp_t *)ibmf_qp_handle;

        ASSERT(clientp != NULL);

        /* decrement the number of active sends */
        if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
                mutex_enter(&clientp->ic_mutex);
                clientp->ic_sends_active--;
                mutex_exit(&clientp->ic_mutex);
        } else {
                mutex_enter(&qpp->isq_mutex);
                qpp->isq_sends_active--;
                mutex_exit(&qpp->isq_mutex);
        }

        mutex_enter(&clientp->ic_kstat_mutex);
        IBMF_SUB32_KSTATS(clientp, sends_active, 1);
        mutex_exit(&clientp->ic_kstat_mutex);

        send_wqep->send_status = ibmf_i_ibt_wc_to_ibmf_status(wcp->wc_status);

        /*
         * issue the callback using taskq. If no taskq or if the
         * dispatch fails, we do the send processing in the callback context
         * which is the interrupt context
         */
        if (cclientp->ic_send_taskq == NULL) {
                /* Do the processing in callback context */
                mutex_enter(&clientp->ic_kstat_mutex);
                IBMF_ADD32_KSTATS(clientp, send_cb_active, 1);
                mutex_exit(&clientp->ic_kstat_mutex);
                ibmf_i_do_send_cb((void *)send_wqep);
                mutex_enter(&clientp->ic_kstat_mutex);
                IBMF_SUB32_KSTATS(clientp, send_cb_active, 1);
                mutex_exit(&clientp->ic_kstat_mutex);
                IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_i_handle_send_err, IBMF_TNF_ERROR, "",
                    "ibmf_i_handle_send_completion(): %s\n",
                    tnf_string, msg, "ci_send_taskq == NULL");
                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "",
                    "ibmf_i_handle_send_completion() exit\n");
                return;
        }

        mutex_enter(&clientp->ic_kstat_mutex);
        IBMF_ADD32_KSTATS(clientp, send_cb_active, 1);
        mutex_exit(&clientp->ic_kstat_mutex);

        /* Use taskq for processing if the IBMF_REG_FLAG_NO_OFFLOAD isn't set */
        if ((clientp->ic_reg_flags & IBMF_REG_FLAG_NO_OFFLOAD) == 0) {
                ret = taskq_dispatch(cclientp->ic_send_taskq, ibmf_i_do_send_cb,
                    send_wqep, TQ_NOSLEEP);
                if (ret == TASKQID_INVALID) {
                        IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4,
                            ibmf_i_handle_send_err, IBMF_TNF_ERROR, "",
                            "ibmf_i_handle_send_completion(): %s\n",
                            tnf_string, msg, "send: dispatch failed");
                        ibmf_i_do_send_cb((void *)send_wqep);
                }
        } else {
                ibmf_i_do_send_cb((void *)send_wqep);
        }

        mutex_enter(&clientp->ic_kstat_mutex);
        IBMF_SUB32_KSTATS(clientp, send_cb_active, 1);
        mutex_exit(&clientp->ic_kstat_mutex);

        _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep))

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "",
            "ibmf_i_handle_send_completion() exit\n");
}

/*
 * ibmf_i_do_send_cb():
 *      Do the send completion processing
 */
static void
ibmf_i_do_send_cb(void *taskq_arg)
{
        ibmf_ci_t               *cip;
        ibmf_msg_impl_t         *msgimplp;
        ibmf_client_t           *clientp;
        ibmf_send_wqe_t         *send_wqep;
        boolean_t               found;
        int                     msg_trans_state_flags, msg_flags;
        uint_t                  ref_cnt;
        ibmf_qp_handle_t        ibmf_qp_handle;
        struct kmem_cache       *kmem_cachep;
        timeout_id_t            msg_rp_unset_id, msg_tr_unset_id;
        timeout_id_t            msg_rp_set_id, msg_tr_set_id;
        ibmf_alt_qp_t           *altqp;
        boolean_t               inc_refcnt;

        send_wqep = taskq_arg;

        IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_do_send_cb_start, IBMF_TNF_TRACE, "",
            "ibmf_i_do_send_cb() enter, send_wqep = %p\n",
            tnf_opaque, send_wqep, send_wqep);

        clientp = send_wqep->send_client;
        cip = clientp->ic_myci;
        msgimplp = send_wqep->send_msg;

        /* get the QP handle */
        ibmf_qp_handle = send_wqep->send_ibmf_qp_handle;

        /* Get the WQE kmem cache pointer based on the QP type */
        if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT)
                kmem_cachep = cip->ci_send_wqes_cache;
        else {
                altqp = (ibmf_alt_qp_t *)ibmf_qp_handle;
                kmem_cachep = altqp->isq_send_wqes_cache;
        }

        /* Look for a message in the client's message list */
        inc_refcnt = B_TRUE;
        found = ibmf_i_find_msg_client(clientp, msgimplp, inc_refcnt);

        /*
         * If the message context was not found, then it's likely
         * been freed up. So, do nothing in this timeout handler
         */
        if (found == B_FALSE) {
                kmem_cache_free(kmem_cachep, send_wqep);
                mutex_enter(&cip->ci_mutex);
                IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1);
                mutex_exit(&cip->ci_mutex);
                if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
                        mutex_enter(&cip->ci_mutex);
                        cip->ci_wqes_alloced--;
                        if (cip->ci_wqes_alloced == 0)
                                cv_signal(&cip->ci_wqes_cv);
                        mutex_exit(&cip->ci_mutex);
                } else {
                        mutex_enter(&altqp->isq_mutex);
                        altqp->isq_wqes_alloced--;
                        if (altqp->isq_wqes_alloced == 0)
                                cv_signal(&altqp->isq_wqes_cv);
                        mutex_exit(&altqp->isq_mutex);
                }
                IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
                    ibmf_i_do_send_cb, IBMF_TNF_TRACE, "",
                    "ibmf_i_do_send_cb(): %s\n", tnf_string, msg,
                    "Message not found, return without processing send cb");
                return;
        }

        /* Grab the message context lock */
        mutex_enter(&msgimplp->im_mutex);

        /*
         * Decrement the count of pending send completions for
         * this transaction
         */
        msgimplp->im_pending_send_compls -= 1;

        /*
         * If the pending send completions is not zero, then we must
         * not attempt to notify the client of a transaction completion
         * in this instance of the send completion handler. Notification
         * of transaction completion should be provided only by the
         * last send completion so that all send completions are accounted
         * for before the client is notified and subsequently attempts to
         * reuse the message for an other transaction.
         * If this is not done, the message may be reused while the
         * send WR from the old transaction is still active in the QP's WQ.
         * This could result in an attempt to modify the address handle with
         * information for the new transaction which could be potentially
         * incompatible, such as an incorrect port number. Such an
         * incompatible modification of the address handle of the old
         * transaction could result in a QP error.
         */
        if (msgimplp->im_pending_send_compls != 0) {
                IBMF_MSG_DECR_REFCNT(msgimplp);
                mutex_exit(&msgimplp->im_mutex);
                kmem_cache_free(kmem_cachep, send_wqep);
                mutex_enter(&cip->ci_mutex);
                IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1);
                mutex_exit(&cip->ci_mutex);
                if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
                        mutex_enter(&cip->ci_mutex);
                        cip->ci_wqes_alloced--;
                        if (cip->ci_wqes_alloced == 0)
                                cv_signal(&cip->ci_wqes_cv);
                        mutex_exit(&cip->ci_mutex);
                } else {
                        mutex_enter(&altqp->isq_mutex);
                        altqp->isq_wqes_alloced--;
                        if (altqp->isq_wqes_alloced == 0)
                                cv_signal(&altqp->isq_wqes_cv);
                        mutex_exit(&altqp->isq_mutex);
                }
                IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
                    ibmf_i_do_send_cb, IBMF_TNF_TRACE, "",
                    "ibmf_i_do_send_cb(): %s\n", tnf_string, msg,
                    "Message found with pending send completions, "
                    "return without processing send cb");
                return;
        }

        /*
         * If the message has been marked unitialized or done
         * release the message mutex and return
         */
        if ((msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_UNINIT) ||
            (msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE)) {
                IBMF_MSG_DECR_REFCNT(msgimplp);
                msg_trans_state_flags = msgimplp->im_trans_state_flags;
                msg_flags = msgimplp->im_flags;
                ref_cnt = msgimplp->im_ref_count;
                mutex_exit(&msgimplp->im_mutex);
                /*
                 * This thread may notify the client only if the
                 * transaction is done, the message has been removed
                 * from the client's message list, and the message
                 * reference count is 0.
                 * If the transaction is done, and the message reference
                 * count = 0, there is still a possibility that a
                 * packet could arrive for the message and its reference
                 * count increased if the message is still on the list.
                 * If the message is still on the list, it will be
                 * removed by a call to ibmf_i_client_rem_msg() at
                 * the completion point of the transaction.
                 * So, the reference count should be checked after the
                 * message has been removed.
                 */
                if ((msg_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) &&
                    !(msg_flags & IBMF_MSG_FLAGS_ON_LIST) &&
                    (ref_cnt == 0)) {

                        ibmf_i_notify_sequence(clientp, msgimplp, msg_flags);

                }
                kmem_cache_free(kmem_cachep, send_wqep);
                mutex_enter(&cip->ci_mutex);
                IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1);
                mutex_exit(&cip->ci_mutex);
                if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
                        mutex_enter(&cip->ci_mutex);
                        cip->ci_wqes_alloced--;
                        if (cip->ci_wqes_alloced == 0)
                                cv_signal(&cip->ci_wqes_cv);
                        mutex_exit(&cip->ci_mutex);
                } else {
                        mutex_enter(&altqp->isq_mutex);
                        altqp->isq_wqes_alloced--;
                        if (altqp->isq_wqes_alloced == 0)
                                cv_signal(&altqp->isq_wqes_cv);
                        mutex_exit(&altqp->isq_mutex);
                }
                IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
                    ibmf_i_do_send_cb, IBMF_TNF_TRACE, "",
                    "ibmf_i_do_send_cb(): %s, msg = %p\n", tnf_string, msg,
                    "Message marked for removal, return without processing "
                    "send cb", tnf_opaque, msgimplp, msgimplp);
                return;
        }

        /* Perform send completion processing of the message context */
        ibmf_i_do_send_compl((ibmf_handle_t)clientp, msgimplp, send_wqep);

        msg_rp_unset_id = msg_tr_unset_id = msg_rp_set_id = msg_tr_set_id = 0;

        /* Save the message flags before releasing the mutex */
        msg_trans_state_flags = msgimplp->im_trans_state_flags;
        msg_flags = msgimplp->im_flags;
        msg_rp_unset_id = msgimplp->im_rp_unset_timeout_id;
        msg_tr_unset_id = msgimplp->im_tr_unset_timeout_id;
        msgimplp->im_rp_unset_timeout_id = 0;
        msgimplp->im_tr_unset_timeout_id = 0;

        /*
         * Decrement the message reference count
         * This count was inceremented when the message was found on the
         * client's message list
         */
        IBMF_MSG_DECR_REFCNT(msgimplp);

        if (msg_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) {
                if (msgimplp->im_rp_timeout_id != 0) {
                        msg_rp_set_id = msgimplp->im_rp_timeout_id;
                        msgimplp->im_rp_timeout_id = 0;
                }
                if (msgimplp->im_tr_timeout_id != 0) {
                        msg_tr_set_id = msgimplp->im_tr_timeout_id;
                        msgimplp->im_tr_timeout_id = 0;
                }
        }

        mutex_exit(&msgimplp->im_mutex);

        if (msg_rp_unset_id != 0) {
                (void) untimeout(msg_rp_unset_id);
        }

        if (msg_tr_unset_id != 0) {
                (void) untimeout(msg_tr_unset_id);
        }

        if (msg_rp_set_id != 0) {
                (void) untimeout(msg_rp_set_id);
        }

        if (msg_tr_set_id != 0) {
                (void) untimeout(msg_tr_set_id);
        }

        IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
            ibmf_i_do_send_cb, IBMF_TNF_TRACE, "",
            "ibmf_i_do_send_cb(): %s, msg = %p\n",
            tnf_string, msg, "Send callback done.  Dec ref count",
            tnf_opaque, msgimplp, msgimplp);

        /*
         * If the transaction is done, signal the block thread if the
         * transaction is blocking, or call the client's transaction done
         * notification callback
         */
        if (msg_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) {

                /* Remove the message from the client's message list */
                ibmf_i_client_rem_msg(clientp, msgimplp, &ref_cnt);

                /*
                 * Notify the client if the message reference count is zero.
                 * At this point, we know that the transaction is done and
                 * the message has been removed from the client's message list.
                 * So, we only need to make sure the reference count is zero
                 * before notifying the client.
                 */
                if (ref_cnt == 0) {

                        ibmf_i_notify_sequence(clientp, msgimplp, msg_flags);

                }
        }

        kmem_cache_free(kmem_cachep, send_wqep);
        mutex_enter(&cip->ci_mutex);
        IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1);
        mutex_exit(&cip->ci_mutex);
        if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) {
                mutex_enter(&cip->ci_mutex);
                cip->ci_wqes_alloced--;
                if (cip->ci_wqes_alloced == 0)
                        cv_signal(&cip->ci_wqes_cv);
                mutex_exit(&cip->ci_mutex);
        } else {
                mutex_enter(&altqp->isq_mutex);
                altqp->isq_wqes_alloced--;
                if (altqp->isq_wqes_alloced == 0)
                        cv_signal(&altqp->isq_wqes_cv);
                mutex_exit(&altqp->isq_mutex);
        }

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
            ibmf_i_do_send_cb_end, IBMF_TNF_TRACE, "",
            "ibmf_i_do_send_cb() exit\n");
}

/*
 * ibmf_i_do_send_compl():
 *      Determine if the transaction is complete
 */
/* ARGSUSED */
static void
ibmf_i_do_send_compl(ibmf_handle_t ibmf_handle, ibmf_msg_impl_t *msgimplp,
    ibmf_send_wqe_t *send_wqep)
{
        IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_compl_start,
            IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl(): ibmf_hdl = 0x%p "
            "msgp = %p, send_wqep = 0x%p, msg_flags = 0x%x\n",
            tnf_opaque, ibmf_hdl, ibmf_handle, tnf_opaque, msgimplp, msgimplp,
            tnf_opaque, send_wqep, send_wqep,
            tnf_opaque, msg_flags, msgimplp->im_flags);

        ASSERT(MUTEX_HELD(&msgimplp->im_mutex));

        /*
         * For RMPP transactions, we only care about the final packet of the
         * transaction.  For others, the code does not need to wait for the send
         * completion (although bad things can happen if it never occurs).
         * The final packets of a transaction are sent when the state is either
         * ABORT or RECEVR_TERMINATE.
         * Don't mark the transaction as send_done if there are still more
         * packets to be sent, including doing the second part of a double-sided
         * transaction.
         */
        if ((msgimplp->im_flags & IBMF_MSG_FLAGS_RECV_RMPP) ||
            (msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP)) {

                IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
                    ibmf_i_do_send_compl, IBMF_TNF_TRACE, "",
                    "ibmf_i_do_send_compl(): %s msgp = %p, rmpp_state = 0x%x\n",
                    tnf_string, msg, "Received send callback for RMPP trans",
                    tnf_opaque, msg, msgimplp,
                    tnf_opaque, rmpp_state, msgimplp->im_rmpp_ctx.rmpp_state);

                /*
                 * For ABORT state, we should not return control to
                 * the client from the send completion handler.
                 * Control should be returned in the error timeout handler.
                 *
                 * The exception is when the IBMF_TRANS_STATE_FLAG_RECV_DONE
                 * flag has already been set. This flag is set when
                 * ibmf_i_terminate_transaction is called from one of the
                 * three timeout handlers. In this case return control from
                 * here.
                 */
                if (msgimplp->im_rmpp_ctx.rmpp_state == IBMF_RMPP_STATE_ABORT) {
                        msgimplp->im_trans_state_flags |=
                            IBMF_TRANS_STATE_FLAG_SEND_DONE;
                        if (msgimplp->im_trans_state_flags &
                            IBMF_TRANS_STATE_FLAG_RECV_DONE) {
                                msgimplp->im_trans_state_flags |=
                                    IBMF_TRANS_STATE_FLAG_DONE;
                        }
                }

                if ((msgimplp->im_rmpp_ctx.rmpp_state ==
                    IBMF_RMPP_STATE_RECEVR_TERMINATE) ||
                    (msgimplp->im_rmpp_ctx.rmpp_state ==
                    IBMF_RMPP_STATE_DONE)) {
                        msgimplp->im_trans_state_flags |=
                            IBMF_TRANS_STATE_FLAG_SEND_DONE;
                        if (msgimplp->im_trans_state_flags  &
                            IBMF_TRANS_STATE_FLAG_RECV_DONE) {
                                msgimplp->im_trans_state_flags |=
                                    IBMF_TRANS_STATE_FLAG_DONE;
                        }
                }

                /*
                 * If the transaction is a send-only RMPP, then
                 * set the SEND_DONE flag on every send completion
                 * as long as there are no outstanding ones.
                 * This is needed so that the transaction can return
                 * in the receive path, where ibmf_i_terminate_transaction
                 * is called from ibmf_i_rmpp_sender_active_flow,
                 * after checking if the SEND_DONE flag is set.
                 * When a new MAD is sent as part of the RMPP transaction,
                 * the SEND_DONE flag will get reset.
                 * The RECV_DONE indicates that the last ACK was received.
                 */
                if ((msgimplp->im_flags & IBMF_MSG_FLAGS_SEQUENCED) == 0) {
                        if (msgimplp->im_pending_send_compls == 0) {
                                msgimplp->im_trans_state_flags |=
                                    IBMF_TRANS_STATE_FLAG_SEND_DONE;
                                if (msgimplp->im_trans_state_flags  &
                                    IBMF_TRANS_STATE_FLAG_RECV_DONE) {
                                        msgimplp->im_trans_state_flags |=
                                            IBMF_TRANS_STATE_FLAG_DONE;
                                }
                        }
                }

                IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                    ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "",
                    "ibmf_i_do_send_compl() exit\n");
                return;
        }

        /*
         * Only non-RMPP send completion gets here.
         * If the send is a single-packet send that does not use RMPP, and if
         * the transaction is not a sequenced transaction, call the transaction
         * callback handler after flagging the transaction as done.  If the
         * message is sequenced, start a timer to bound the wait for the first
         * data packet of the response.
         */
        if (msgimplp->im_flags & IBMF_MSG_FLAGS_SEQUENCED) {

                IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
                    ibmf_i_do_send_compl, IBMF_TNF_TRACE, "",
                    "ibmf_i_do_send_compl(): %s msgp = %p\n", tnf_string, msg,
                    "Sequenced transaction, setting response timer",
                    tnf_opaque, msg, msgimplp);

                /*
                 * Check if the send completion already occured,
                 * which could imply that this is a send completion
                 * for some previous transaction that has come in very late.
                 * In this case exit here.
                 */
                if (msgimplp->im_trans_state_flags  &
                    IBMF_TRANS_STATE_FLAG_SEND_DONE) {
                        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                            ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "",
                            "ibmf_i_do_send_compl() exit, "
                            "Duplicate SEND completion\n");
                        return;
                }

                /* mark as send_compl happened */
                msgimplp->im_trans_state_flags |=
                    IBMF_TRANS_STATE_FLAG_SEND_DONE;

                if (msgimplp->im_trans_state_flags  &
                    IBMF_TRANS_STATE_FLAG_RECV_DONE) {
                        msgimplp->im_trans_state_flags |=
                            IBMF_TRANS_STATE_FLAG_DONE;
                        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
                            ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "",
                            "ibmf_i_do_send_compl() exit, RECV_DONE\n");
                        return;
                }

                /*
                 * check if response was received before send
                 * completion
                 */
                if (((msgimplp->im_trans_state_flags &
                    IBMF_TRANS_STATE_FLAG_DONE) == 0) &&
                    ((msgimplp->im_trans_state_flags &
                    IBMF_TRANS_STATE_FLAG_RECV_ACTIVE) == 0)) {
                        /* set timer for first packet of response */
                        ibmf_i_set_timer(ibmf_i_send_timeout, msgimplp,
                            IBMF_RESP_TIMER);
                }
        } else {
                msgimplp->im_msg_status = IBMF_SUCCESS;
                msgimplp->im_trans_state_flags |=
                    IBMF_TRANS_STATE_FLAG_SEND_DONE;
                msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_DONE;
        }

        IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_compl_end,
            IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl() exit\n");
}