/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */

#include <sys/types.h>
#include <sys/kmem.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/ksynch.h>

#include <sys/ib/clients/eoib/eib_impl.h>

/*
 * Declarations private to this file
 */
static int eib_rsrc_setup_txbufs(eib_t *, int *);
static int eib_rsrc_setup_rxbufs(eib_t *, int *);
static int eib_rsrc_setup_lsobufs(eib_t *, int *);
static void eib_rsrc_init_wqe_pool(eib_t *, eib_wqe_pool_t **,
    ib_memlen_t, int);
static void eib_rsrc_fini_wqe_pool(eib_t *, eib_wqe_pool_t **);
static boolean_t eib_rsrc_ok_to_free_pool(eib_t *, eib_wqe_pool_t *, boolean_t);
static int eib_rsrc_grab_wqes(eib_t *, eib_wqe_pool_t *, eib_wqe_t **, uint_t,
    uint_t *, int);
static void eib_rsrc_return_wqes(eib_t *, eib_wqe_pool_t *, eib_wqe_t **,
    uint_t);

static void eib_rb_rsrc_setup_txbufs(eib_t *, boolean_t);
static void eib_rb_rsrc_setup_rxbufs(eib_t *, boolean_t);
static void eib_rb_rsrc_setup_lsobufs(eib_t *, boolean_t);

/*
 * Definitions private to this file
 */
static uint_t eib_lso_num_bufs = EIB_LSO_NUM_BUFS;      /* tunable? */

int
eib_rsrc_setup_bufs(eib_t *ss, int *err)
{
        if (eib_rsrc_setup_txbufs(ss, err) != EIB_E_SUCCESS)
                return (EIB_E_FAILURE);

        if (ss->ei_caps->cp_lso_maxlen && ss->ei_caps->cp_cksum_flags &&
            ss->ei_caps->cp_resv_lkey_capab) {
                if (eib_rsrc_setup_lsobufs(ss, err) != EIB_E_SUCCESS) {
                        eib_rb_rsrc_setup_txbufs(ss, B_FALSE);
                        return (EIB_E_FAILURE);
                }
        }

        if (eib_rsrc_setup_rxbufs(ss, err) != EIB_E_SUCCESS) {
                eib_rb_rsrc_setup_lsobufs(ss, B_FALSE);
                eib_rb_rsrc_setup_txbufs(ss, B_FALSE);
                return (EIB_E_FAILURE);
        }

        return (EIB_E_SUCCESS);
}

int
eib_rsrc_grab_swqes(eib_t *ss, eib_wqe_t **wqes, uint_t n_req, uint_t *actual,
    int pri)
{
        eib_wqe_t *wqe;
        uint32_t *encap_hdr;
        int ret;
        int i;

        ASSERT(ss->ei_tx != NULL);

        ret = eib_rsrc_grab_wqes(ss, ss->ei_tx, wqes, n_req, actual, pri);
        if (ret != EIB_E_SUCCESS)
                return (EIB_E_FAILURE);

        /*
         * See note for eib_rsrc_grab_swqe()
         */
        for (i = 0; i < (*actual); i++) {
                wqe = wqes[i];
                wqe->qe_wr.send.wr_flags = IBT_WR_NO_FLAGS;
                wqe->qe_wr.send.wr.ud.udwr_dest = wqe->qe_dest;
                wqe->qe_wr.send.wr_opcode = IBT_WRC_SEND;
                wqe->qe_wr.send.wr_nds = 1;
                wqe->qe_wr.send.wr_sgl = &wqe->qe_sgl;
                wqe->qe_nxt_post = NULL;
                wqe->qe_iov_hdl = NULL;

                encap_hdr = (uint32_t *)(void *)wqe->qe_payload_hdr;
                *encap_hdr = htonl(EIB_TX_ENCAP_HDR);
        }

        return (EIB_E_SUCCESS);
}

int
eib_rsrc_grab_rwqes(eib_t *ss, eib_wqe_t **wqes, uint_t n_req, uint_t *actual,
    int pri)
{
        ASSERT(ss->ei_rx != NULL);

        return (eib_rsrc_grab_wqes(ss, ss->ei_rx, wqes, n_req, actual, pri));
}

int
eib_rsrc_grab_lsobufs(eib_t *ss, uint_t req_sz, ibt_wr_ds_t *sgl, uint32_t *nds)
{
        eib_lsobkt_t *bkt = ss->ei_lso;
        eib_lsobuf_t *elem;
        eib_lsobuf_t *nxt;
        uint_t frag_sz;
        uint_t num_needed;
        int i;

        ASSERT(req_sz != 0);
        ASSERT(sgl != NULL);
        ASSERT(nds != NULL);

        /*
         * Determine how many bufs we'd need for the size requested
         */
        num_needed = req_sz / EIB_LSO_BUFSZ;
        if ((frag_sz = req_sz % EIB_LSO_BUFSZ) != 0)
                num_needed++;

        if (bkt == NULL)
                return (EIB_E_FAILURE);

        /*
         * If we don't have enough lso bufs, return failure
         */
        mutex_enter(&bkt->bk_lock);
        if (bkt->bk_nfree < num_needed) {
                mutex_exit(&bkt->bk_lock);
                return (EIB_E_FAILURE);
        }

        /*
         * Pick the first "num_needed" bufs from the free list
         */
        elem = bkt->bk_free_head;
        for (i = 0; i < num_needed; i++) {
                ASSERT(elem->lb_isfree != 0);
                ASSERT(elem->lb_buf != NULL);

                nxt = elem->lb_next;

                sgl[i].ds_va = (ib_vaddr_t)(uintptr_t)elem->lb_buf;
                sgl[i].ds_key = bkt->bk_lkey;
                sgl[i].ds_len = EIB_LSO_BUFSZ;

                elem->lb_isfree = 0;
                elem->lb_next = NULL;

                elem = nxt;
        }
        bkt->bk_free_head = elem;

        /*
         * If the requested size is not a multiple of EIB_LSO_BUFSZ, we need
         * to adjust the last sgl entry's length. Since we know we need atleast
         * one, the i-1 use below is ok.
         */
        if (frag_sz) {
                sgl[i-1].ds_len = frag_sz;
        }

        /*
         * Update nfree count and return
         */
        bkt->bk_nfree -= num_needed;

        mutex_exit(&bkt->bk_lock);

        *nds = num_needed;

        return (EIB_E_SUCCESS);
}

eib_wqe_t *
eib_rsrc_grab_swqe(eib_t *ss, int pri)
{
        eib_wqe_t *wqe = NULL;
        uint32_t *encap_hdr;

        ASSERT(ss->ei_tx != NULL);
        (void) eib_rsrc_grab_wqes(ss, ss->ei_tx, &wqe, 1, NULL, pri);

        /*
         * Let's reset the swqe basic wr parameters to default. We need
         * to do this because this swqe could've previously been used
         * for a checksum offload (when the flags would've been set)
         * or for an LSO send (in which case the opcode would've been set
         * to a different value), or been iov mapped (in which case the
         * sgl/nds could've been set to different values).  We'll make
         * it easy and initialize it here, so simple transactions can
         * go through without any special effort by the caller.
         *
         * Note that even though the wqe structure is common for both
         * send and recv, they're in two independent pools and the wqe
         * type remains the same throughout its lifetime. So we don't
         * have to worry about resetting any other field.
         */
        if (wqe) {
                wqe->qe_wr.send.wr_flags = IBT_WR_NO_FLAGS;
                wqe->qe_wr.send.wr.ud.udwr_dest = wqe->qe_dest;
                wqe->qe_wr.send.wr_opcode = IBT_WRC_SEND;
                wqe->qe_wr.send.wr_nds = 1;
                wqe->qe_wr.send.wr_sgl = &wqe->qe_sgl;
                wqe->qe_nxt_post = NULL;
                wqe->qe_iov_hdl = NULL;

                encap_hdr = (uint32_t *)(void *)wqe->qe_payload_hdr;
                *encap_hdr = htonl(EIB_TX_ENCAP_HDR);
        }

        return (wqe);
}

eib_wqe_t *
eib_rsrc_grab_rwqe(eib_t *ss, int pri)
{
        eib_wqe_t *wqe = NULL;

        ASSERT(ss->ei_rx != NULL);
        (void) eib_rsrc_grab_wqes(ss, ss->ei_rx, &wqe, 1, NULL, pri);

        return (wqe);
}

void
eib_rsrc_return_swqe(eib_t *ss, eib_wqe_t *wqe, eib_chan_t *chan)
{
        ASSERT(ss->ei_tx != NULL);

        eib_rsrc_return_wqes(ss, ss->ei_tx, &wqe, 1);
        if (chan) {
                eib_rsrc_decr_posted_swqe(ss, chan);
        }
}


void
eib_rsrc_return_rwqe(eib_t *ss, eib_wqe_t *wqe, eib_chan_t *chan)
{
        ASSERT(ss->ei_rx != NULL);

        eib_rsrc_return_wqes(ss, ss->ei_rx, &wqe, 1);
        if (chan) {
                eib_rsrc_decr_posted_rwqe(ss, chan);
        }
}

void
eib_rsrc_return_lsobufs(eib_t *ss, ibt_wr_ds_t *sgl_p, uint32_t nds)
{
        eib_lsobkt_t *bkt = ss->ei_lso;
        eib_lsobuf_t *elem;
        uint8_t *va;
        ptrdiff_t ndx;
        int i;

        /*
         * Nowhere to return the buffers to ??
         */
        if (bkt == NULL)
                return;

        mutex_enter(&bkt->bk_lock);

        for (i = 0; i < nds; i++) {
                va = (uint8_t *)(uintptr_t)sgl_p[i].ds_va;

                ASSERT(va >= bkt->bk_mem);
                ASSERT(va < (bkt->bk_mem + bkt->bk_nelem * EIB_LSO_BUFSZ));

                /*
                 * Figure out the buflist element this sgl buffer corresponds
                 * to and put it back at the head
                 */
                ndx = ((uintptr_t)va - (uintptr_t)bkt->bk_mem) / EIB_LSO_BUFSZ;
                elem = bkt->bk_bufl + ndx;

                ASSERT(elem->lb_isfree == 0);
                ASSERT(elem->lb_buf == va);

                elem->lb_isfree = 1;
                elem->lb_next = bkt->bk_free_head;
                bkt->bk_free_head = elem;
        }
        bkt->bk_nfree += nds;

        /*
         * If the number of available lso buffers just crossed the
         * threshold, wakeup anyone who may be sleeping on the event.
         */
        if (((bkt->bk_nfree - nds) < EIB_LSO_FREE_BUFS_THRESH) &&
            (bkt->bk_nfree >= EIB_LSO_FREE_BUFS_THRESH)) {
                cv_broadcast(&bkt->bk_cv);
        }

        mutex_exit(&bkt->bk_lock);
}

/*ARGSUSED*/
void
eib_rsrc_decr_posted_swqe(eib_t *ss, eib_chan_t *chan)
{
        ASSERT(chan != NULL);

        mutex_enter(&chan->ch_tx_lock);

        chan->ch_tx_posted--;
        if ((chan->ch_tear_down) && (chan->ch_tx_posted == 0)) {
                cv_signal(&chan->ch_tx_cv);
        }

        mutex_exit(&chan->ch_tx_lock);
}

void
eib_rsrc_decr_posted_rwqe(eib_t *ss, eib_chan_t *chan)
{
        eib_chan_t *tail;
        boolean_t queue_for_refill = B_FALSE;

        ASSERT(chan != NULL);

        /*
         * Decrement the ch_rx_posted count. If we are tearing this channel
         * down, signal the waiter when the count reaches 0.  If we aren't
         * tearing the channel down, see if the count has gone below the low
         * water mark.  If it has, and if this channel isn't already being
         * refilled, queue the channel up with the service thread for a
         * rwqe refill.
         */
        mutex_enter(&chan->ch_rx_lock);
        chan->ch_rx_posted--;
        if (chan->ch_tear_down) {
                if (chan->ch_rx_posted == 0)
                        cv_signal(&chan->ch_rx_cv);
        } else if (chan->ch_rx_posted < chan->ch_lwm_rwqes) {
                if (chan->ch_rx_refilling == B_FALSE) {
                        chan->ch_rx_refilling = B_TRUE;
                        queue_for_refill = B_TRUE;
                }
        }
        mutex_exit(&chan->ch_rx_lock);

        if (queue_for_refill) {
                mutex_enter(&ss->ei_rxpost_lock);

                chan->ch_rxpost_next = NULL;
                for (tail = ss->ei_rxpost; tail; tail = tail->ch_rxpost_next) {
                        if (tail->ch_rxpost_next == NULL)
                                break;
                }
                if (tail) {
                        tail->ch_rxpost_next = chan;
                } else {
                        ss->ei_rxpost = chan;
                }

                cv_signal(&ss->ei_rxpost_cv);
                mutex_exit(&ss->ei_rxpost_lock);
        }
}

void
eib_rsrc_txwqes_needed(eib_t *ss)
{
        eib_wqe_pool_t *wp = ss->ei_tx;

        EIB_INCR_COUNTER(&ss->ei_stats->st_noxmitbuf);

        mutex_enter(&wp->wp_lock);
        if ((wp->wp_status & EIB_TXWQE_SHORT) == 0) {
                wp->wp_status |= EIB_TXWQE_SHORT;
                cv_broadcast(&wp->wp_cv);
        }
        mutex_exit(&wp->wp_lock);
}

void
eib_rsrc_lsobufs_needed(eib_t *ss)
{
        eib_lsobkt_t *bkt = ss->ei_lso;

        EIB_INCR_COUNTER(&ss->ei_stats->st_noxmitbuf);

        if (bkt == NULL) {
                EIB_DPRINTF_WARN(ss->ei_instance,
                    "eib_rsrc_lsobufs_needed: "
                    "lso bufs seem to be needed even though "
                    "LSO support was not advertised");
                return;
        }

        mutex_enter(&bkt->bk_lock);
        if ((bkt->bk_status & EIB_LBUF_SHORT) == 0) {
                bkt->bk_status |= EIB_LBUF_SHORT;
                cv_broadcast(&bkt->bk_cv);
        }
        mutex_exit(&bkt->bk_lock);
}

boolean_t
eib_rsrc_rxpool_low(eib_wqe_t *wqe)
{
        eib_wqe_pool_t *wp = wqe->qe_pool;
        boolean_t ret = B_FALSE;

        /*
         * Set the EIB_RXWQE_SHORT flag when the number of free wqes
         * in the rx pool falls below the low threshold for rwqes and
         * clear it only when the number of free wqes gets back above
         * the high water mark.
         */
        mutex_enter(&wp->wp_lock);

        if (wp->wp_nfree <= EIB_NFREE_RWQES_LOW) {
                wp->wp_status |= (EIB_RXWQE_SHORT);
        } else if (wp->wp_nfree >= EIB_NFREE_RWQES_HWM) {
                wp->wp_status &= (~EIB_RXWQE_SHORT);
        }

        if ((wp->wp_status & EIB_RXWQE_SHORT) == EIB_RXWQE_SHORT)
                ret = B_TRUE;

        mutex_exit(&wp->wp_lock);

        return (ret);
}

void
eib_rb_rsrc_setup_bufs(eib_t *ss, boolean_t force)
{
        eib_rb_rsrc_setup_rxbufs(ss, force);
        eib_rb_rsrc_setup_lsobufs(ss, force);
        eib_rb_rsrc_setup_txbufs(ss, force);
}

static int
eib_rsrc_setup_txbufs(eib_t *ss, int *err)
{
        eib_wqe_pool_t *tx;
        eib_wqe_t *wqe;
        ibt_ud_dest_hdl_t dest;
        ibt_mr_attr_t attr;
        ibt_mr_desc_t desc;
        ibt_status_t ret;
        kthread_t *kt;
        uint32_t *encap_hdr;
        uint8_t *buf;
        uint_t mtu = ss->ei_props->ep_mtu;
        uint_t tx_bufsz;
        uint_t blk;
        uint_t ndx;
        uint_t i;
        int lso_enabled;

        /*
         * Try to allocate and initialize the tx wqe pool
         */
        if (ss->ei_tx != NULL)
                return (EIB_E_SUCCESS);

        /*
         * If we keep the tx buffers as mtu-sized, then potentially every
         * LSO request that cannot be satisfactorily mapped, will use up
         * the 8K large (default size) lso buffers. This may be inadvisable
         * given that lso buffers are a scarce resource.  Instead, we'll
         * slightly raise the size of the copy buffers in the send wqes
         * (say to EIB_TX_COPY_THRESH) so that requests that cannot be
         * mapped could still avoid using the 8K LSO buffers if they're
         * less than the copy threshold size.
         */
        lso_enabled = ss->ei_caps->cp_lso_maxlen &&
            ss->ei_caps->cp_cksum_flags && ss->ei_caps->cp_resv_lkey_capab;
        tx_bufsz = ((lso_enabled) && (EIB_TX_COPY_THRESH > mtu)) ?
            EIB_TX_COPY_THRESH : mtu;

        eib_rsrc_init_wqe_pool(ss, &ss->ei_tx, tx_bufsz, EIB_WP_TYPE_TX);
        tx = ss->ei_tx;

        /*
         * Register the TX memory region with IBTF for use
         */
        attr.mr_vaddr = tx->wp_vaddr;
        attr.mr_len = tx->wp_memsz;
        attr.mr_as = NULL;
        attr.mr_flags = IBT_MR_SLEEP;

        ret = ibt_register_mr(ss->ei_hca_hdl, ss->ei_pd_hdl, &attr,
            &tx->wp_mr, &desc);
        if (ret != IBT_SUCCESS) {
                EIB_DPRINTF_ERR(ss->ei_instance, "eib_rsrc_setup_txbufs: "
                    "ibt_register_mr() failed for tx "
                    "region (0x%llx, 0x%llx) with ret=%d",
                    attr.mr_vaddr, attr.mr_len, ret);

                *err = EINVAL;
                goto rsrc_setup_txbufs_fail;
        }
        tx->wp_lkey = desc.md_lkey;

        /*
         * Now setup the tx wqes
         */
        buf = (uint8_t *)(uintptr_t)(tx->wp_vaddr);
        for (i = 0, blk = 0; blk < EIB_BLKS_PER_POOL; blk++) {
                for (ndx = 0; ndx < EIB_WQES_PER_BLK; ndx++, i++) {
                        wqe = &tx->wp_wqe[i];
                        /*
                         * Allocate a UD destination handle
                         */
                        ret = ibt_alloc_ud_dest(ss->ei_hca_hdl,
                            IBT_UD_DEST_NO_FLAGS, ss->ei_pd_hdl, &dest);
                        if (ret != IBT_SUCCESS) {
                                EIB_DPRINTF_ERR(ss->ei_instance,
                                    "eib_rsrc_setup_txbufs: "
                                    "ibt_alloc_ud_dest(hca_hdl=0x%llx) "
                                    "failed, ret=%d", ss->ei_hca_hdl, ret);

                                *err = ENOMEM;
                                goto rsrc_setup_txbufs_fail;
                        }

                        /*
                         * These parameters should remain fixed throughout the
                         * lifetime of this wqe.
                         */
                        wqe->qe_pool = tx;
                        wqe->qe_cpbuf = buf;
                        wqe->qe_bufsz = tx_bufsz;

                        /*
                         * The qe_dest and qe_payload_hdr are specific to tx
                         * only, but remain unchanged throughout the lifetime
                         * of the wqe.
                         *
                         * The payload header is normally used when we have an
                         * LSO packet to send.  Since the EoIB encapsulation
                         * header won't be part of the message we get from the
                         * network layer, we'll need to copy the lso header into
                         * a new buffer every time before we hand over the LSO
                         * send request to the hca driver.
                         */
                        wqe->qe_dest = dest;
                        wqe->qe_payload_hdr =
                            kmem_zalloc(EIB_MAX_PAYLOAD_HDR_SZ, KM_SLEEP);

                        /*
                         * The encapsulation header is at the start of the
                         * payload header and is initialized to the default
                         * encapsulation header we use (no multiple segments,
                         * no FCS). This part of the header is not expected
                         * to change.
                         */
                        encap_hdr = (uint32_t *)(void *)wqe->qe_payload_hdr;
                        *encap_hdr = htonl(EIB_TX_ENCAP_HDR);

                        /*
                         * The parameter set below are used in tx and rx paths.
                         * These parameters (except ds_key) are reset to these
                         * default values in eib_rsrc_return_wqes().
                         */
                        wqe->qe_sgl.ds_key = tx->wp_lkey;
                        wqe->qe_sgl.ds_va = (ib_vaddr_t)(uintptr_t)buf;
                        wqe->qe_sgl.ds_len = wqe->qe_bufsz;
                        wqe->qe_mp = NULL;
                        wqe->qe_info =
                            ((blk & EIB_WQEBLK_MASK) << EIB_WQEBLK_SHIFT) |
                            ((ndx & EIB_WQENDX_MASK) << EIB_WQENDX_SHIFT) |
                            ((uint_t)EIB_WQE_TX << EIB_WQETYP_SHIFT);

                        /*
                         * These tx-specific parameters (except wr_id and
                         * wr_trans) are reset in eib_rsrc_grab_swqes() to make
                         * sure any freshly acquired swqe from the pool has
                         * these default settings for the caller.
                         */
                        wqe->qe_wr.send.wr_id = (ibt_wrid_t)(uintptr_t)wqe;
                        wqe->qe_wr.send.wr_trans = IBT_UD_SRV;
                        wqe->qe_wr.send.wr_flags = IBT_WR_NO_FLAGS;
                        wqe->qe_wr.send.wr.ud.udwr_dest = wqe->qe_dest;
                        wqe->qe_wr.send.wr_opcode = IBT_WRC_SEND;
                        wqe->qe_wr.send.wr_nds = 1;
                        wqe->qe_wr.send.wr_sgl = &wqe->qe_sgl;
                        wqe->qe_nxt_post = NULL;
                        wqe->qe_iov_hdl = NULL;

                        buf += wqe->qe_bufsz;
                }
        }

        /*
         * Before returning, create a kernel thread to monitor the status
         * of wqes in the tx wqe pool.  Note that this thread cannot be
         * created from eib_state_init() during attach(), since the thread
         * expects the wqe pool to be allocated and ready when it starts,
         * and the tx bufs initialization only happens during eib_m_start().
         */
        kt = thread_create(NULL, 0, eib_monitor_tx_wqes, ss, 0,
            &p0, TS_RUN, minclsyspri);
        ss->ei_txwqe_monitor = kt->t_did;

        return (EIB_E_SUCCESS);

rsrc_setup_txbufs_fail:
        eib_rb_rsrc_setup_txbufs(ss, B_FALSE);
        return (EIB_E_FAILURE);
}

static int
eib_rsrc_setup_rxbufs(eib_t *ss, int *err)
{
        eib_wqe_pool_t *rx;
        eib_wqe_t *wqe;
        ibt_mr_attr_t attr;
        ibt_mr_desc_t desc;
        ibt_status_t ret;
        uint8_t *buf;
        uint_t mtu = ss->ei_props->ep_mtu;
        uint_t blk;
        uint_t ndx;
        uint_t i;

        /*
         * Try to allocate and initialize the wqe pool. When this is called
         * during a plumb via the mac m_start callback, we need to make
         * sure there is a need to allocate a wqe pool afresh.  If during a
         * previous unplumb we didn't free the wqe pool because the nw layer
         * was holding on to some rx buffers, we don't need to allocate new
         * pool and set up the buffers again; we'll just start re-using the
         * previous one.
         */
        if (ss->ei_rx != NULL)
                return (EIB_E_SUCCESS);

        /*
         * The receive buffer has to work for all channels, specifically the
         * data qp of the vnics.  This means that the buffer must be large
         * enough to hold MTU sized IB payload (including the EoIB and ethernet
         * headers) plus the GRH. In addition, because the ethernet header is
         * either 14 or 18 bytes (tagless or vlan tagged), we should have the
         * buffer filled in such a way that the IP header starts at atleast a
         * 4-byte aligned address.  In order to do this, we need to have some
         * additional room.
         */
        eib_rsrc_init_wqe_pool(ss, &ss->ei_rx,
            mtu + EIB_GRH_SZ + EIB_IPHDR_ALIGN_ROOM, EIB_WP_TYPE_RX);
        rx = ss->ei_rx;

        /*
         * Register the RX memory region with IBTF for use
         */
        attr.mr_vaddr = rx->wp_vaddr;
        attr.mr_len = rx->wp_memsz;
        attr.mr_as = NULL;
        attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE;

        ret = ibt_register_mr(ss->ei_hca_hdl, ss->ei_pd_hdl, &attr,
            &rx->wp_mr, &desc);
        if (ret != IBT_SUCCESS) {
                EIB_DPRINTF_ERR(ss->ei_instance, "eib_rsrc_setup_rxbufs: "
                    "ibt_register_mr() failed for rx "
                    "region (0x%llx, 0x%llx) with ret=%d",
                    attr.mr_vaddr, attr.mr_len, ret);

                *err = EINVAL;
                goto rsrc_setup_rxbufs_fail;
        }
        rx->wp_lkey = desc.md_lkey;

        /*
         * Now setup the rx wqes
         */
        buf = (uint8_t *)(uintptr_t)(rx->wp_vaddr);
        for (i = 0, blk = 0; blk < EIB_BLKS_PER_POOL; blk++) {
                for (ndx = 0; ndx < EIB_WQES_PER_BLK; ndx++, i++) {
                        wqe = &rx->wp_wqe[i];

                        /*
                         * These parameters should remain fixed throughout the
                         * lifetime of this recv wqe. The qe_frp will only be
                         * used by the data channel of vnics and will remain
                         * unused by other channels.
                         */
                        wqe->qe_pool = rx;
                        wqe->qe_cpbuf = buf;
                        wqe->qe_bufsz = mtu + EIB_GRH_SZ + EIB_IPHDR_ALIGN_ROOM;
                        wqe->qe_wr.recv.wr_id = (ibt_wrid_t)(uintptr_t)wqe;
                        wqe->qe_wr.recv.wr_nds = 1;
                        wqe->qe_wr.recv.wr_sgl = &wqe->qe_sgl;
                        wqe->qe_frp.free_func = eib_data_rx_recycle;
                        wqe->qe_frp.free_arg = (caddr_t)wqe;

                        /*
                         * The parameter set below are used in tx and rx paths.
                         * These parameters (except ds_key) are reset to these
                         * default values in eib_rsrc_return_wqes().
                         */
                        wqe->qe_sgl.ds_key = rx->wp_lkey;
                        wqe->qe_sgl.ds_va = (ib_vaddr_t)(uintptr_t)buf;
                        wqe->qe_sgl.ds_len = wqe->qe_bufsz;
                        wqe->qe_mp = NULL;
                        wqe->qe_info =
                            ((blk & EIB_WQEBLK_MASK) << EIB_WQEBLK_SHIFT) |
                            ((ndx & EIB_WQENDX_MASK) << EIB_WQENDX_SHIFT) |
                            ((uint_t)EIB_WQE_RX << EIB_WQETYP_SHIFT);

                        /*
                         * These rx-specific parameters are also reset to
                         * these default values in eib_rsrc_return_wqes().
                         */
                        wqe->qe_chan = NULL;
                        wqe->qe_vnic_inst = -1;

                        buf += (mtu + EIB_GRH_SZ + EIB_IPHDR_ALIGN_ROOM);
                }
        }

        return (EIB_E_SUCCESS);

rsrc_setup_rxbufs_fail:
        eib_rb_rsrc_setup_rxbufs(ss, B_FALSE);
        return (EIB_E_FAILURE);
}

static int
eib_rsrc_setup_lsobufs(eib_t *ss, int *err)
{
        eib_lsobkt_t *bkt;
        eib_lsobuf_t *elem;
        eib_lsobuf_t *tail;
        ibt_mr_attr_t attr;
        ibt_mr_desc_t desc;
        kthread_t *kt;

        uint8_t *lsomem;
        uint8_t *memp;
        ibt_status_t ret;
        int i;

        /*
         * Allocate the lso bucket and space for buffers
         */
        bkt = kmem_zalloc(sizeof (eib_lsobkt_t), KM_SLEEP);
        lsomem = kmem_zalloc(eib_lso_num_bufs * EIB_LSO_BUFSZ, KM_SLEEP);

        /*
         * Register lso memory and save the lkey
         */
        attr.mr_vaddr = (uint64_t)(uintptr_t)lsomem;
        attr.mr_len = eib_lso_num_bufs * EIB_LSO_BUFSZ;
        attr.mr_as = NULL;
        attr.mr_flags = IBT_MR_SLEEP;

        ret = ibt_register_mr(ss->ei_hca_hdl, ss->ei_pd_hdl, &attr,
            &bkt->bk_mr_hdl, &desc);
        if (ret != IBT_SUCCESS) {
                *err = EINVAL;
                EIB_DPRINTF_ERR(ss->ei_instance, "eib_rsrc_setup_lsobufs: "
                    "ibt_register_mr() failed for LSO "
                    "region (0x%llx, 0x%llx) with ret=%d",
                    attr.mr_vaddr, attr.mr_len, ret);

                kmem_free(lsomem, eib_lso_num_bufs * EIB_LSO_BUFSZ);
                kmem_free(bkt, sizeof (eib_lsobkt_t));

                return (EIB_E_FAILURE);
        }
        bkt->bk_lkey = desc.md_lkey;

        /*
         * Now allocate the buflist.  Note that the elements in the buflist and
         * the buffers in the lso memory have a permanent 1-1 relation, so we
         * can always derive the address of a buflist entry from the address of
         * an lso buffer.
         */
        bkt->bk_bufl = kmem_zalloc(eib_lso_num_bufs * sizeof (eib_lsobuf_t),
            KM_SLEEP);

        /*
         * Set up the lso buf chain
         */
        memp = lsomem;
        elem = bkt->bk_bufl;
        for (i = 0; i < eib_lso_num_bufs; i++) {
                elem->lb_isfree = 1;
                elem->lb_buf = memp;
                elem->lb_next = elem + 1;

                tail = elem;

                memp += EIB_LSO_BUFSZ;
                elem++;
        }
        tail->lb_next = NULL;

        /*
         * Set up the LSO buffer information in eib state
         */
        bkt->bk_free_head = bkt->bk_bufl;
        bkt->bk_mem = lsomem;
        bkt->bk_nelem = eib_lso_num_bufs;
        bkt->bk_nfree = bkt->bk_nelem;

        mutex_init(&bkt->bk_lock, NULL, MUTEX_DRIVER, NULL);
        cv_init(&bkt->bk_cv, NULL, CV_DEFAULT, NULL);

        ss->ei_lso = bkt;

        /*
         * Before returning, create a kernel thread to monitor the status
         * of lso bufs
         */
        kt = thread_create(NULL, 0, eib_monitor_lso_bufs, ss, 0,
            &p0, TS_RUN, minclsyspri);
        ss->ei_lsobufs_monitor = kt->t_did;

        return (EIB_E_SUCCESS);
}

static void
eib_rsrc_init_wqe_pool(eib_t *ss, eib_wqe_pool_t **wpp, ib_memlen_t bufsz,
    int wp_type)
{
        eib_wqe_pool_t *wp;
        uint_t wp_wqesz;
        int i;

        ASSERT(wpp != NULL);
        ASSERT(*wpp == NULL);

        /*
         * Allocate the wqe pool, wqes and bufs
         */
        wp = kmem_zalloc(sizeof (eib_wqe_pool_t), KM_SLEEP);
        wp_wqesz = EIB_WQES_PER_POOL * sizeof (eib_wqe_t);
        wp->wp_wqe = (eib_wqe_t *)kmem_zalloc(wp_wqesz, KM_SLEEP);
        wp->wp_memsz = EIB_WQES_PER_POOL * bufsz;
        wp->wp_vaddr = (ib_vaddr_t)(uintptr_t)kmem_zalloc(wp->wp_memsz,
            KM_SLEEP);
        wp->wp_ss = ss;
        wp->wp_type = wp_type;
        wp->wp_nfree_lwm = (wp_type == EIB_WP_TYPE_TX) ?
            EIB_NFREE_SWQES_LWM : EIB_NFREE_RWQES_LWM;

        /*
         * Initialize the lock and bitmaps: everything is available at first,
         * but note that if the number of blocks per pool is less than 64, we
         * need to initialize those extra bits as "unavailable" - these will
         * remain unavailable throughout.
         */
        mutex_init(&wp->wp_lock, NULL, MUTEX_DRIVER, NULL);
        cv_init(&wp->wp_cv, NULL, CV_DEFAULT, NULL);

        wp->wp_nfree = EIB_WQES_PER_POOL;
        wp->wp_free_blks = (EIB_BLKS_PER_POOL >= 64) ? (~0) :
            (((uint64_t)1 << EIB_BLKS_PER_POOL) - 1);
        for (i = 0; i < EIB_BLKS_PER_POOL; i++)
                wp->wp_free_wqes[i] = ~0;

        *wpp = wp;
}

/*ARGSUSED*/
static void
eib_rsrc_fini_wqe_pool(eib_t *ss, eib_wqe_pool_t **wpp)
{
        eib_wqe_pool_t *wp;

        ASSERT(wpp != NULL);

        wp = *wpp;
        ASSERT(*wpp != NULL);

        cv_destroy(&wp->wp_cv);
        mutex_destroy(&wp->wp_lock);

        kmem_free((void *)(uintptr_t)(wp->wp_vaddr), wp->wp_memsz);
        kmem_free(wp->wp_wqe, EIB_WQES_PER_POOL * sizeof (eib_wqe_t));
        kmem_free(wp, sizeof (eib_wqe_pool_t));

        *wpp = NULL;
}

/*ARGSUSED*/
static boolean_t
eib_rsrc_ok_to_free_pool(eib_t *ss, eib_wqe_pool_t *wp, boolean_t force)
{
        uint64_t free_blks;
        int i;

        /*
         * See if we can release all memory allocated for buffers, wqes and
         * the pool.  Note that in the case of data channel rx buffers, some
         * of the buffers may not be free if the nw layer is holding on to
         * them still.  If this is the case, we cannot free the wqe pool now
         * or a subsequent access by the nw layer to the buffers will cause
         * a panic.
         */
        ASSERT(wp != NULL);

        /*
         * If force-free flag is set, we can always release the memory.
         * Note that this flag is unused currently, and should be removed.
         */
        if (force == B_TRUE)
                return (B_TRUE);

        mutex_enter(&wp->wp_lock);

        /*
         * If a whole block remains allocated, obviously we cannot free
         * the pool
         */
        free_blks = (EIB_BLKS_PER_POOL >= 64) ? (~0) :
            (((uint64_t)1 << EIB_BLKS_PER_POOL) - 1);
        if (wp->wp_free_blks != free_blks) {
                mutex_exit(&wp->wp_lock);
                return (B_FALSE);
        }

        /*
         * If even a single wqe within any one block remains in-use, we
         * cannot free the pool
         */
        for (i = 0; i < EIB_BLKS_PER_POOL; i++) {
                if (wp->wp_free_wqes[i] != (~0)) {
                        mutex_exit(&wp->wp_lock);
                        return (B_FALSE);
                }
        }

        mutex_exit(&wp->wp_lock);

        return (B_TRUE);
}

/*ARGSUSED*/
static int
eib_rsrc_grab_wqes(eib_t *ss, eib_wqe_pool_t *wp, eib_wqe_t **wqes,
    uint_t n_req, uint_t *actual, int pri)
{
        uint_t n_allocd = 0;
        int blk;
        int ndx;
        int wqe_ndx;

        ASSERT(wp != NULL);
        ASSERT(wqes != NULL);

        mutex_enter(&wp->wp_lock);

        /*
         * If this is a low priority request, adjust the number requested
         * so we don't allocate beyond the low-water-mark
         */
        if (pri == EIB_WPRI_LO) {
                if (wp->wp_nfree <= wp->wp_nfree_lwm)
                        n_req = 0;
                else if ((wp->wp_nfree - n_req) < wp->wp_nfree_lwm)
                        n_req = wp->wp_nfree - wp->wp_nfree_lwm;
        }

        for (n_allocd = 0;  n_allocd < n_req; n_allocd++) {
                /*
                 * If the entire pool is unavailable, quit
                 */
                if (wp->wp_free_blks == 0)
                        break;

                /*
                 * Find the first wqe that's available
                 */
                blk = EIB_FIND_LSB_SET(wp->wp_free_blks);
                ASSERT(blk != -1);
                ndx = EIB_FIND_LSB_SET(wp->wp_free_wqes[blk]);
                ASSERT(ndx != -1);

                /*
                 * Mark the wqe as allocated
                 */
                wp->wp_free_wqes[blk] &= (~((uint64_t)1 << ndx));

                /*
                 * If this was the last free wqe in this block, mark
                 * the block itself as unavailable
                 */
                if (wp->wp_free_wqes[blk] == 0)
                        wp->wp_free_blks &= (~((uint64_t)1 << blk));

                /*
                 * Return this wqe to the caller
                 */
                wqe_ndx = blk * EIB_WQES_PER_BLK + ndx;
                wqes[n_allocd] = &(wp->wp_wqe[wqe_ndx]);
        }

        wp->wp_nfree -= n_allocd;

        mutex_exit(&wp->wp_lock);

        if (n_allocd == 0)
                return (EIB_E_FAILURE);

        if (actual) {
                *actual = n_allocd;
        }

        return (EIB_E_SUCCESS);
}

/*ARGSUSED*/
static void
eib_rsrc_return_wqes(eib_t *ss, eib_wqe_pool_t *wp, eib_wqe_t **wqes,
    uint_t n_wqes)
{
        eib_wqe_t *wqe;
        uint_t n_freed = 0;
        uint_t blk;
        uint_t ndx;

        ASSERT(wp != NULL);
        ASSERT(wqes != NULL);

        mutex_enter(&wp->wp_lock);
        for (n_freed = 0;  n_freed < n_wqes; n_freed++) {
                wqe = wqes[n_freed];

                /*
                 * This wqe is being returned back to the pool, so clear
                 * any wqe flags and reset buffer address and size in the
                 * single segment sgl back to what they were initially.
                 * Also erase any mblk pointer and callback function ptrs.
                 */
                wqe->qe_sgl.ds_va = (ib_vaddr_t)(uintptr_t)wqe->qe_cpbuf;
                wqe->qe_sgl.ds_len = wqe->qe_bufsz;
                wqe->qe_mp = NULL;
                wqe->qe_chan = NULL;
                wqe->qe_vnic_inst = -1;
                wqe->qe_info &= (~EIB_WQEFLGS_MASK);

                /*
                 * Mark the wqe free in its block
                 */
                blk = EIB_WQE_BLK(wqe->qe_info);
                ndx = EIB_WQE_NDX(wqe->qe_info);

                wp->wp_free_wqes[blk] |= ((uint64_t)1 << ndx);

                /*
                 * This block now has atleast one wqe free, so mark
                 * the block itself as available and move on to the
                 * next wqe to free
                 */
                wp->wp_free_blks |= ((uint64_t)1 << blk);
        }

        wp->wp_nfree += n_freed;

        /*
         * If the number of available wqes in the pool has just crossed
         * the high-water-mark, wakeup anyone who may be sleeping on it.
         */
        if ((wp->wp_type == EIB_WP_TYPE_TX) &&
            ((wp->wp_nfree - n_freed) < EIB_NFREE_SWQES_HWM) &&
            (wp->wp_nfree >= EIB_NFREE_SWQES_HWM)) {
                cv_broadcast(&wp->wp_cv);
        }

        mutex_exit(&wp->wp_lock);
}

static void
eib_rb_rsrc_setup_txbufs(eib_t *ss, boolean_t force)
{
        eib_wqe_pool_t *wp = ss->ei_tx;
        eib_wqe_t *wqe;
        ibt_ud_dest_hdl_t dest;
        ibt_status_t ret;
        uint8_t *plhdr;
        int i;

        if (wp == NULL)
                return;

        /*
         * Check if it's ok to free the tx wqe pool (i.e. all buffers have
         * been reclaimed) and if so, stop the txwqe monitor thread (and wait
         * for it to die), release the UD destination handles, deregister
         * memory and fini the wqe pool.
         */
        if (eib_rsrc_ok_to_free_pool(ss, wp, force)) {
                eib_stop_monitor_tx_wqes(ss);

                for (i = 0; i < EIB_WQES_PER_POOL; i++) {
                        wqe = &wp->wp_wqe[i];
                        if ((plhdr = wqe->qe_payload_hdr) != NULL) {
                                kmem_free(plhdr, EIB_MAX_PAYLOAD_HDR_SZ);
                        }
                        if ((dest = wqe->qe_dest) != NULL) {
                                ret = ibt_free_ud_dest(dest);
                                if (ret != IBT_SUCCESS) {
                                        EIB_DPRINTF_WARN(ss->ei_instance,
                                            "eib_rb_rsrc_setup_txbufs: "
                                            "ibt_free_ud_dest() failed, ret=%d",
                                            ret);
                                }
                        }
                }
                if (wp->wp_mr) {
                        if ((ret = ibt_deregister_mr(ss->ei_hca_hdl,
                            wp->wp_mr)) != IBT_SUCCESS) {
                                EIB_DPRINTF_WARN(ss->ei_instance,
                                    "eib_rb_rsrc_setup_txbufs: "
                                    "ibt_deregister_mr() failed, ret=%d", ret);
                        }
                        wp->wp_mr = NULL;
                }
                eib_rsrc_fini_wqe_pool(ss, &ss->ei_tx);
        }
}

void
eib_rb_rsrc_setup_rxbufs(eib_t *ss, boolean_t force)
{
        eib_wqe_pool_t *rx = ss->ei_rx;
        ibt_status_t ret;

        if (rx == NULL)
                return;

        /*
         * Check if it's ok to free the rx wqe pool (i.e. all buffers have
         * been reclaimed) and if so, deregister memory and fini the wqe pool.
         */
        if (eib_rsrc_ok_to_free_pool(ss, rx, force)) {
                if (rx->wp_mr) {
                        if ((ret = ibt_deregister_mr(ss->ei_hca_hdl,
                            rx->wp_mr)) != IBT_SUCCESS) {
                                EIB_DPRINTF_WARN(ss->ei_instance,
                                    "eib_rb_rsrc_setup_rxbufs: "
                                    "ibt_deregister_mr() failed, ret=%d", ret);
                        }
                        rx->wp_mr = NULL;
                }

                eib_rsrc_fini_wqe_pool(ss, &ss->ei_rx);
        }
}

static void
eib_rb_rsrc_setup_lsobufs(eib_t *ss, boolean_t force)
{
        eib_lsobkt_t *bkt;
        ibt_status_t ret;

        /*
         * Remove the lso bucket from the state
         */
        if ((bkt = ss->ei_lso) == NULL)
                return;

        /*
         * Try to stop the lso bufs monitor thread. If we fail, we simply
         * return.  We'll have another shot at it later from detach() with
         * the force flag set.
         */
        if (eib_stop_monitor_lso_bufs(ss, force) != EIB_E_SUCCESS)
                return;

        /*
         * Free the buflist
         */
        if (bkt->bk_bufl) {
                kmem_free(bkt->bk_bufl, bkt->bk_nelem * sizeof (eib_lsobuf_t));
                bkt->bk_bufl = NULL;
        }

        /*
         * Deregister LSO memory and free it
         */
        if (bkt->bk_mr_hdl) {
                if ((ret = ibt_deregister_mr(ss->ei_hca_hdl,
                    bkt->bk_mr_hdl)) != IBT_SUCCESS) {
                        EIB_DPRINTF_WARN(ss->ei_instance,
                            "eib_rb_rsrc_setup_lsobufs: "
                            "ibt_deregister_mr() failed, ret=%d", ret);
                }
                bkt->bk_mr_hdl = NULL;
        }
        if (bkt->bk_mem) {
                kmem_free(bkt->bk_mem, bkt->bk_nelem * EIB_LSO_BUFSZ);
                bkt->bk_mem = NULL;
        }

        /*
         * Destroy the mutex and condvar
         */
        cv_destroy(&bkt->bk_cv);
        mutex_destroy(&bkt->bk_lock);

        /*
         * Finally, free the lso bucket itself
         */
        kmem_free(bkt, sizeof (eib_lsobkt_t));
        ss->ei_lso = NULL;
}