/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/mac_provider.h>
#include <sys/nxge/nxge_impl.h>
#include <sys/nxge/nxge_hio.h>
#include <npi_tx_wr64.h>

/* Software LSO required header files */
#include <netinet/tcp.h>
#include <inet/ip_impl.h>
#include <inet/tcp.h>

extern uint64_t mac_pkt_hash(uint_t, mblk_t *mp, uint8_t policy,
    boolean_t is_outbound);

static mblk_t *nxge_lso_eliminate(mblk_t *);
static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss);
static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *);
static void nxge_hcksum_retrieve(mblk_t *,
    uint32_t *, uint32_t *, uint32_t *,
    uint32_t *, uint32_t *);
static uint32_t nxge_csgen(uint16_t *, int);

extern uint32_t         nxge_reclaim_pending;
extern uint32_t         nxge_bcopy_thresh;
extern uint32_t         nxge_dvma_thresh;
extern uint32_t         nxge_dma_stream_thresh;
extern uint32_t         nxge_tx_minfree;
extern uint32_t         nxge_tx_intr_thres;
extern uint32_t         nxge_tx_max_gathers;
extern uint32_t         nxge_tx_tiny_pack;
extern uint32_t         nxge_tx_use_bcopy;
extern nxge_tx_mode_t   nxge_tx_scheme;
uint32_t                nxge_lso_kick_cnt = 2;


void
nxge_tx_ring_task(void *arg)
{
        p_tx_ring_t     ring = (p_tx_ring_t)arg;

        ASSERT(ring->tx_ring_handle != NULL);

        MUTEX_ENTER(&ring->lock);
        (void) nxge_txdma_reclaim(ring->nxgep, ring, 0);
        MUTEX_EXIT(&ring->lock);

        if (!ring->tx_ring_offline) {
                mac_tx_ring_update(ring->nxgep->mach, ring->tx_ring_handle);
        }
}

static void
nxge_tx_ring_dispatch(p_tx_ring_t ring)
{
        /*
         * Kick the ring task to reclaim some buffers.
         */
        (void) ddi_taskq_dispatch(ring->taskq,
            nxge_tx_ring_task, (void *)ring, DDI_SLEEP);
}

mblk_t *
nxge_tx_ring_send(void *arg, mblk_t *mp)
{
        p_nxge_ring_handle_t    nrhp = (p_nxge_ring_handle_t)arg;
        p_nxge_t                nxgep;
        p_tx_ring_t             tx_ring_p;
        int                     status, channel;

        ASSERT(nrhp != NULL);
        nxgep = nrhp->nxgep;
        channel = nxgep->pt_config.hw_config.tdc.start + nrhp->index;
        tx_ring_p = nxgep->tx_rings->rings[channel];

        /*
         * We may be in a transition from offlined DMA to onlined
         * DMA.
         */
        if (tx_ring_p == NULL) {
                ASSERT(tx_ring_p != NULL);
                freemsg(mp);
                return ((mblk_t *)NULL);
        }

        /*
         * Valid DMA?
         */
        ASSERT(nxgep == tx_ring_p->nxgep);

        /*
         * Make sure DMA is not offlined.
         */
        if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline) {
                ASSERT(!tx_ring_p->tx_ring_offline);
                freemsg(mp);
                return ((mblk_t *)NULL);
        }

        /*
         * Transmit the packet.
         */
        status = nxge_start(nxgep, tx_ring_p, mp);
        if (status) {
                nxge_tx_ring_dispatch(tx_ring_p);
                return (mp);
        }

        return ((mblk_t *)NULL);
}

int
nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp)
{
        int                     dma_status, status = 0;
        p_tx_desc_t             tx_desc_ring_vp;
        npi_handle_t            npi_desc_handle;
        nxge_os_dma_handle_t    tx_desc_dma_handle;
        p_tx_desc_t             tx_desc_p;
        p_tx_msg_t              tx_msg_ring;
        p_tx_msg_t              tx_msg_p = NULL;
        tx_desc_t               tx_desc, *tmp_desc_p;
        tx_desc_t               sop_tx_desc, *sop_tx_desc_p;
        p_tx_pkt_header_t       hdrp = NULL;
        tx_pkt_hdr_all_t        tmp_hdrp;
        p_tx_pkt_hdr_all_t      pkthdrp;
        uint8_t                 npads = 0;
        uint64_t                dma_ioaddr;
        uint32_t                dma_flags;
        int                     last_bidx;
        uint8_t                 *b_rptr;
        caddr_t                 kaddr;
        uint32_t                nmblks;
        uint32_t                ngathers;
        uint32_t                clen;
        int                     len;
        uint32_t                pkt_len, pack_len, min_len;
        uint32_t                bcopy_thresh;
        int                     i, cur_index, sop_index;
        uint16_t                tail_index;
        boolean_t               tail_wrap = B_FALSE;
        nxge_dma_common_t       desc_area;
        nxge_os_dma_handle_t    dma_handle;
        ddi_dma_cookie_t        dma_cookie;
        npi_handle_t            npi_handle;
        p_mblk_t                nmp;
        p_mblk_t                t_mp;
        uint32_t                ncookies;
        boolean_t               good_packet;
        boolean_t               mark_mode = B_FALSE;
        p_nxge_stats_t          statsp;
        p_nxge_tx_ring_stats_t tdc_stats;
        t_uscalar_t             start_offset = 0;
        t_uscalar_t             stuff_offset = 0;
        t_uscalar_t             end_offset = 0;
        t_uscalar_t             value = 0;
        t_uscalar_t             cksum_flags = 0;
        boolean_t               cksum_on = B_FALSE;
        uint32_t                boff = 0;
        uint64_t                tot_xfer_len = 0;
        boolean_t               header_set = B_FALSE;
#ifdef NXGE_DEBUG
        p_tx_desc_t             tx_desc_ring_pp;
        p_tx_desc_t             tx_desc_pp;
        tx_desc_t               *save_desc_p;
        int                     dump_len;
        int                     sad_len;
        uint64_t                sad;
        int                     xfer_len;
        uint32_t                msgsize;
#endif
        p_mblk_t                mp_chain = NULL;
        boolean_t               is_lso = B_FALSE;
        boolean_t               lso_again;
        int                     cur_index_lso;
        p_mblk_t                nmp_lso_save = NULL;
        uint32_t                lso_ngathers;
        boolean_t               lso_tail_wrap = B_FALSE;

        NXGE_DEBUG_MSG((nxgep, TX_CTL,
            "==> nxge_start: tx dma channel %d", tx_ring_p->tdc));
        NXGE_DEBUG_MSG((nxgep, TX_CTL,
            "==> nxge_start: Starting tdc %d desc pending %d",
            tx_ring_p->tdc, tx_ring_p->descs_pending));

        statsp = nxgep->statsp;

        if (!isLDOMguest(nxgep)) {
                switch (nxgep->mac.portmode) {
                default:
                        if (nxgep->statsp->port_stats.lb_mode ==
                            nxge_lb_normal) {
                                if (!statsp->mac_stats.link_up) {
                                        freemsg(mp);
                                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                            "==> nxge_start: "
                                            "link not up"));
                                        goto nxge_start_fail1;
                                }
                        }
                        break;
                case PORT_10G_FIBER:
                        /*
                         * For the following modes, check the link status
                         * before sending the packet out:
                         * nxge_lb_normal,
                         * nxge_lb_ext10g,
                         * nxge_lb_ext1000,
                         * nxge_lb_ext100,
                         * nxge_lb_ext10.
                         */
                        if (nxgep->statsp->port_stats.lb_mode <
                            nxge_lb_phy10g) {
                                if (!statsp->mac_stats.link_up) {
                                        freemsg(mp);
                                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                            "==> nxge_start: "
                                            "link not up"));
                                        goto nxge_start_fail1;
                                }
                        }
                        break;
                }
        }

        if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
            (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
                NXGE_DEBUG_MSG((nxgep, TX_CTL,
                    "==> nxge_start: hardware not initialized or stopped"));
                freemsg(mp);
                goto nxge_start_fail1;
        }

        if (nxgep->soft_lso_enable) {
                mp_chain = nxge_lso_eliminate(mp);
                NXGE_DEBUG_MSG((nxgep, TX_CTL,
                    "==> nxge_start(0): LSO mp $%p mp_chain $%p",
                    mp, mp_chain));
                if (mp_chain == NULL) {
                        NXGE_ERROR_MSG((nxgep, TX_CTL,
                            "==> nxge_send(0): NULL mp_chain $%p != mp $%p",
                            mp_chain, mp));
                        goto nxge_start_fail1;
                }
                if (mp_chain != mp) {
                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                            "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p",
                            mp_chain, mp));
                        is_lso = B_TRUE;
                        mp = mp_chain;
                        mp_chain = mp_chain->b_next;
                        mp->b_next = NULL;
                }
        }

        mac_hcksum_get(mp, &start_offset, &stuff_offset, &end_offset,
            &value, &cksum_flags);
        if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) {
                start_offset += sizeof (ether_header_t);
                stuff_offset += sizeof (ether_header_t);
        } else {
                start_offset += sizeof (struct ether_vlan_header);
                stuff_offset += sizeof (struct ether_vlan_header);
        }

        if (cksum_flags & HCK_PARTIALCKSUM) {
                NXGE_DEBUG_MSG((nxgep, TX_CTL,
                    "==> nxge_start: mp $%p len %d "
                    "cksum_flags 0x%x (partial checksum) ",
                    mp, MBLKL(mp), cksum_flags));
                cksum_on = B_TRUE;
        }

        pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp;
        pkthdrp->reserved = 0;
        tmp_hdrp.pkthdr.value = 0;
        nxge_fill_tx_hdr(mp, B_FALSE, cksum_on,
            0, 0, pkthdrp,
            start_offset, stuff_offset);

        lso_again = B_FALSE;
        lso_ngathers = 0;

        MUTEX_ENTER(&tx_ring_p->lock);

        if (isLDOMservice(nxgep)) {
                tx_ring_p->tx_ring_busy = B_TRUE;
                if (tx_ring_p->tx_ring_offline) {
                        freemsg(mp);
                        tx_ring_p->tx_ring_busy = B_FALSE;
                        (void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
                            NXGE_TX_RING_OFFLINED);
                        MUTEX_EXIT(&tx_ring_p->lock);
                        return (status);
                }
        }

        cur_index_lso = tx_ring_p->wr_index;
        lso_tail_wrap = tx_ring_p->wr_index_wrap;
start_again:
        ngathers = 0;
        sop_index = tx_ring_p->wr_index;
#ifdef  NXGE_DEBUG
        if (tx_ring_p->descs_pending) {
                NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
                    "desc pending %d ", tx_ring_p->descs_pending));
        }

        dump_len = (int)(MBLKL(mp));
        dump_len = (dump_len > 128) ? 128: dump_len;

        NXGE_DEBUG_MSG((nxgep, TX_CTL,
            "==> nxge_start: tdc %d: dumping ...: b_rptr $%p "
            "(Before header reserve: ORIGINAL LEN %d)",
            tx_ring_p->tdc,
            mp->b_rptr,
            dump_len));

        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets "
            "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr,
            nxge_dump_packet((char *)mp->b_rptr, dump_len)));
#endif

        tdc_stats = tx_ring_p->tdc_stats;
        mark_mode = (tx_ring_p->descs_pending &&
            (((int)tx_ring_p->tx_ring_size - (int)tx_ring_p->descs_pending) <
            (int)nxge_tx_minfree));

        NXGE_DEBUG_MSG((nxgep, TX_CTL,
            "TX Descriptor ring is channel %d mark mode %d",
            tx_ring_p->tdc, mark_mode));

        if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) {
                if (!nxge_txdma_reclaim(nxgep, tx_ring_p,
                    (nxge_tx_minfree + lso_ngathers))) {
                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                            "TX Descriptor ring is full: channel %d",
                            tx_ring_p->tdc));
                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                            "TX Descriptor ring is full: channel %d",
                            tx_ring_p->tdc));
                        if (is_lso) {
                                /*
                                 * free the current mp and mp_chain if not FULL.
                                 */
                                tdc_stats->tx_no_desc++;
                                NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                    "LSO packet: TX Descriptor ring is full: "
                                    "channel %d",
                                    tx_ring_p->tdc));
                                goto nxge_start_fail_lso;
                        } else {
                                (void) atomic_cas_32(
                                    (uint32_t *)&tx_ring_p->queueing, 0, 1);
                                tdc_stats->tx_no_desc++;

                                if (isLDOMservice(nxgep)) {
                                        tx_ring_p->tx_ring_busy = B_FALSE;
                                        if (tx_ring_p->tx_ring_offline) {
                                                (void) atomic_swap_32(
                                                    &tx_ring_p->tx_ring_offline,
                                                    NXGE_TX_RING_OFFLINED);
                                        }
                                }

                                MUTEX_EXIT(&tx_ring_p->lock);
                                status = 1;
                                goto nxge_start_fail1;
                        }
                }
        }

        nmp = mp;
        i = sop_index = tx_ring_p->wr_index;
        nmblks = 0;
        ngathers = 0;
        pkt_len = 0;
        pack_len = 0;
        clen = 0;
        last_bidx = -1;
        good_packet = B_TRUE;

        desc_area = tx_ring_p->tdc_desc;
        npi_handle = desc_area.npi_handle;
        npi_desc_handle.regh = (nxge_os_acc_handle_t)
            DMA_COMMON_ACC_HANDLE(desc_area);
        tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
        tx_desc_dma_handle = (nxge_os_dma_handle_t)
            DMA_COMMON_HANDLE(desc_area);
        tx_msg_ring = tx_ring_p->tx_msg_ring;

        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d",
            sop_index, i));

#ifdef  NXGE_DEBUG
        msgsize = msgdsize(nmp);
        NXGE_DEBUG_MSG((nxgep, TX_CTL,
            "==> nxge_start(1): wr_index %d i %d msgdsize %d",
            sop_index, i, msgsize));
#endif
        /*
         * The first 16 bytes of the premapped buffer are reserved
         * for header. No padding will be used.
         */
        pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE;
        if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) {
                bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE);
        } else {
                bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE);
        }
        while (nmp) {
                good_packet = B_TRUE;
                b_rptr = nmp->b_rptr;
                len = MBLKL(nmp);
                if (len <= 0) {
                        nmp = nmp->b_cont;
                        continue;
                }
                nmblks++;

                NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d "
                    "len %d pkt_len %d pack_len %d",
                    nmblks, len, pkt_len, pack_len));
                /*
                 * Hardware limits the transfer length to 4K for NIU and
                 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just
                 * use TX_MAX_TRANSFER_LENGTH as the limit for both.
                 * If len is longer than the limit, then we break nmp into
                 * two chunks: Make the first chunk equal to the limit and
                 * the second chunk for the remaining data. If the second
                 * chunk is still larger than the limit, then it will be
                 * broken into two in the next pass.
                 */
                if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) {
                        if ((t_mp = dupb(nmp)) != NULL) {
                                nmp->b_wptr = nmp->b_rptr +
                                    (TX_MAX_TRANSFER_LENGTH
                                    - TX_PKT_HEADER_SIZE);
                                t_mp->b_rptr = nmp->b_wptr;
                                t_mp->b_cont = nmp->b_cont;
                                nmp->b_cont = t_mp;
                                len = MBLKL(nmp);
                        } else {
                                if (is_lso) {
                                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                            "LSO packet: dupb failed: "
                                            "channel %d",
                                            tx_ring_p->tdc));
                                        mp = nmp;
                                        goto nxge_start_fail_lso;
                                } else {
                                        good_packet = B_FALSE;
                                        goto nxge_start_fail2;
                                }
                        }
                }
                tx_desc.value = 0;
                tx_desc_p = &tx_desc_ring_vp[i];
#ifdef  NXGE_DEBUG
                tx_desc_pp = &tx_desc_ring_pp[i];
#endif
                tx_msg_p = &tx_msg_ring[i];
                npi_desc_handle.regp = (uint64_t)tx_desc_p;
                if (!header_set &&
                    ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) ||
                    (len >= bcopy_thresh))) {
                        header_set = B_TRUE;
                        bcopy_thresh += TX_PKT_HEADER_SIZE;
                        boff = 0;
                        pack_len = 0;
                        kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
                        hdrp = (p_tx_pkt_header_t)kaddr;
                        clen = pkt_len;
                        dma_handle = tx_msg_p->buf_dma_handle;
                        dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
                        (void) ddi_dma_sync(dma_handle,
                            i * nxge_bcopy_thresh, nxge_bcopy_thresh,
                            DDI_DMA_SYNC_FORDEV);

                        tx_msg_p->flags.dma_type = USE_BCOPY;
                        goto nxge_start_control_header_only;
                }

                pkt_len += len;
                pack_len += len;

                NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): "
                    "desc entry %d "
                    "DESC IOADDR $%p "
                    "desc_vp $%p tx_desc_p $%p "
                    "desc_pp $%p tx_desc_pp $%p "
                    "len %d pkt_len %d pack_len %d",
                    i,
                    DMA_COMMON_IOADDR(desc_area),
                    tx_desc_ring_vp, tx_desc_p,
                    tx_desc_ring_pp, tx_desc_pp,
                    len, pkt_len, pack_len));

                if (len < bcopy_thresh) {
                        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): "
                            "USE BCOPY: "));
                        if (nxge_tx_tiny_pack) {
                                uint32_t blst =
                                    TXDMA_DESC_NEXT_INDEX(i, -1,
                                    tx_ring_p->tx_wrap_mask);
                                NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                    "==> nxge_start(5): pack"));
                                if ((pack_len <= bcopy_thresh) &&
                                    (last_bidx == blst)) {
                                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                            "==> nxge_start: pack(6) "
                                            "(pkt_len %d pack_len %d)",
                                            pkt_len, pack_len));
                                        i = blst;
                                        tx_desc_p = &tx_desc_ring_vp[i];
#ifdef  NXGE_DEBUG
                                        tx_desc_pp = &tx_desc_ring_pp[i];
#endif
                                        tx_msg_p = &tx_msg_ring[i];
                                        boff = pack_len - len;
                                        ngathers--;
                                } else if (pack_len > bcopy_thresh &&
                                    header_set) {
                                        pack_len = len;
                                        boff = 0;
                                        bcopy_thresh = nxge_bcopy_thresh;
                                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                            "==> nxge_start(7): > max NEW "
                                            "bcopy thresh %d "
                                            "pkt_len %d pack_len %d(next)",
                                            bcopy_thresh,
                                            pkt_len, pack_len));
                                }
                                last_bidx = i;
                        }
                        kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
                        if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) {
                                hdrp = (p_tx_pkt_header_t)kaddr;
                                header_set = B_TRUE;
                                NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                    "==> nxge_start(7_x2): "
                                    "pkt_len %d pack_len %d (new hdrp $%p)",
                                    pkt_len, pack_len, hdrp));
                        }
                        tx_msg_p->flags.dma_type = USE_BCOPY;
                        kaddr += boff;
                        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): "
                            "USE BCOPY: before bcopy "
                            "DESC IOADDR $%p entry %d "
                            "bcopy packets %d "
                            "bcopy kaddr $%p "
                            "bcopy ioaddr (SAD) $%p "
                            "bcopy clen %d "
                            "bcopy boff %d",
                            DMA_COMMON_IOADDR(desc_area), i,
                            tdc_stats->tx_hdr_pkts,
                            kaddr,
                            dma_ioaddr,
                            clen,
                            boff));
                        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
                            "1USE BCOPY: "));
                        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
                            "2USE BCOPY: "));
                        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
                            "last USE BCOPY: copy from b_rptr $%p "
                            "to KADDR $%p (len %d offset %d",
                            b_rptr, kaddr, len, boff));

                        bcopy(b_rptr, kaddr, len);

#ifdef  NXGE_DEBUG
                        dump_len = (len > 128) ? 128: len;
                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                            "==> nxge_start: dump packets "
                            "(After BCOPY len %d)"
                            "(b_rptr $%p): %s", len, nmp->b_rptr,
                            nxge_dump_packet((char *)nmp->b_rptr,
                            dump_len)));
#endif

                        dma_handle = tx_msg_p->buf_dma_handle;
                        dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
                        (void) ddi_dma_sync(dma_handle,
                            i * nxge_bcopy_thresh, nxge_bcopy_thresh,
                            DDI_DMA_SYNC_FORDEV);
                        clen = len + boff;
                        tdc_stats->tx_hdr_pkts++;
                        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): "
                            "USE BCOPY: "
                            "DESC IOADDR $%p entry %d "
                            "bcopy packets %d "
                            "bcopy kaddr $%p "
                            "bcopy ioaddr (SAD) $%p "
                            "bcopy clen %d "
                            "bcopy boff %d",
                            DMA_COMMON_IOADDR(desc_area),
                            i,
                            tdc_stats->tx_hdr_pkts,
                            kaddr,
                            dma_ioaddr,
                            clen,
                            boff));
                } else {
                        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): "
                            "USE DVMA: len %d", len));
                        tx_msg_p->flags.dma_type = USE_DMA;
                        dma_flags = DDI_DMA_WRITE;
                        if (len < nxge_dma_stream_thresh) {
                                dma_flags |= DDI_DMA_CONSISTENT;
                        } else {
                                dma_flags |= DDI_DMA_STREAMING;
                        }

                        dma_handle = tx_msg_p->dma_handle;
                        dma_status = ddi_dma_addr_bind_handle(dma_handle, NULL,
                            (caddr_t)b_rptr, len, dma_flags,
                            DDI_DMA_DONTWAIT, NULL,
                            &dma_cookie, &ncookies);
                        if (dma_status == DDI_DMA_MAPPED) {
                                dma_ioaddr = dma_cookie.dmac_laddress;
                                len = (int)dma_cookie.dmac_size;
                                clen = (uint32_t)dma_cookie.dmac_size;
                                NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                    "==> nxge_start(12_1): "
                                    "USE DVMA: len %d clen %d "
                                    "ngathers %d",
                                    len, clen,
                                    ngathers));
                                npi_desc_handle.regp = (uint64_t)tx_desc_p;
                                while (ncookies > 1) {
                                        ngathers++;
                                        /*
                                         * this is the fix for multiple
                                         * cookies, which are basically
                                         * a descriptor entry, we don't set
                                         * SOP bit as well as related fields
                                         */

                                        (void) npi_txdma_desc_gather_set(
                                            npi_desc_handle,
                                            &tx_desc,
                                            (ngathers -1),
                                            mark_mode,
                                            ngathers,
                                            dma_ioaddr,
                                            clen);

                                        tx_msg_p->tx_msg_size = clen;
                                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                            "==> nxge_start:  DMA "
                                            "ncookie %d "
                                            "ngathers %d "
                                            "dma_ioaddr $%p len %d"
                                            "desc $%p descp $%p (%d)",
                                            ncookies,
                                            ngathers,
                                            dma_ioaddr, clen,
                                            *tx_desc_p, tx_desc_p, i));

                                        ddi_dma_nextcookie(dma_handle,
                                            &dma_cookie);
                                        dma_ioaddr =
                                            dma_cookie.dmac_laddress;

                                        len = (int)dma_cookie.dmac_size;
                                        clen = (uint32_t)dma_cookie.dmac_size;
                                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                            "==> nxge_start(12_2): "
                                            "USE DVMA: len %d clen %d ",
                                            len, clen));

                                        i = TXDMA_DESC_NEXT_INDEX(i, 1,
                                            tx_ring_p->tx_wrap_mask);
                                        tx_desc_p = &tx_desc_ring_vp[i];

                                        npi_desc_handle.regp =
                                            (uint64_t)tx_desc_p;
                                        tx_msg_p = &tx_msg_ring[i];
                                        tx_msg_p->flags.dma_type = USE_NONE;
                                        tx_desc.value = 0;

                                        ncookies--;
                                }
                                tdc_stats->tx_ddi_pkts++;
                                NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:"
                                    "DMA: ddi packets %d",
                                    tdc_stats->tx_ddi_pkts));
                        } else {
                                NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
                                    "dma mapping failed for %d "
                                    "bytes addr $%p flags %x (%d)",
                                    len, b_rptr, status, status));
                                good_packet = B_FALSE;
                                tdc_stats->tx_dma_bind_fail++;
                                tx_msg_p->flags.dma_type = USE_NONE;
                                if (is_lso) {
                                        mp = nmp;
                                        goto nxge_start_fail_lso;
                                } else {
                                        status = 1;
                                        goto nxge_start_fail2;
                                }
                        }
                } /* ddi dvma */

                if (is_lso) {
                        nmp_lso_save = nmp;
                }
                nmp = nmp->b_cont;
nxge_start_control_header_only:
                npi_desc_handle.regp = (uint64_t)tx_desc_p;
                ngathers++;

                if (ngathers == 1) {
#ifdef  NXGE_DEBUG
                        save_desc_p = &sop_tx_desc;
#endif
                        sop_tx_desc_p = &sop_tx_desc;
                        sop_tx_desc_p->value = 0;
                        sop_tx_desc_p->bits.hdw.tr_len = clen;
                        sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
                        sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
                } else {
#ifdef  NXGE_DEBUG
                        save_desc_p = &tx_desc;
#endif
                        tmp_desc_p = &tx_desc;
                        tmp_desc_p->value = 0;
                        tmp_desc_p->bits.hdw.tr_len = clen;
                        tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
                        tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;

                        tx_desc_p->value = tmp_desc_p->value;
                }

                NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): "
                    "Desc_entry %d ngathers %d "
                    "desc_vp $%p tx_desc_p $%p "
                    "len %d clen %d pkt_len %d pack_len %d nmblks %d "
                    "dma_ioaddr (SAD) $%p mark %d",
                    i, ngathers,
                    tx_desc_ring_vp, tx_desc_p,
                    len, clen, pkt_len, pack_len, nmblks,
                    dma_ioaddr, mark_mode));

#ifdef NXGE_DEBUG
                npi_desc_handle.nxgep = nxgep;
                npi_desc_handle.function.function = nxgep->function_num;
                npi_desc_handle.function.instance = nxgep->instance;
                sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK);
                xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >>
                    TX_PKT_DESC_TR_LEN_SHIFT);


                NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
                    "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t"
                    "mark %d sop %d\n",
                    save_desc_p->value,
                    sad,
                    save_desc_p->bits.hdw.tr_len,
                    xfer_len,
                    save_desc_p->bits.hdw.num_ptr,
                    save_desc_p->bits.hdw.mark,
                    save_desc_p->bits.hdw.sop));

                npi_txdma_dump_desc_one(npi_desc_handle, NULL, i);
#endif

                tx_msg_p->tx_msg_size = clen;
                i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask);
                if (ngathers > nxge_tx_max_gathers) {
                        good_packet = B_FALSE;
                        mac_hcksum_get(mp, &start_offset,
                            &stuff_offset, &end_offset, &value,
                            &cksum_flags);

                        NXGE_DEBUG_MSG((NULL, TX_CTL,
                            "==> nxge_start(14): pull msg - "
                            "len %d pkt_len %d ngathers %d",
                            len, pkt_len, ngathers));

                        /*
                         * Just give up on this packet.
                         */
                        if (is_lso) {
                                mp = nmp_lso_save;
                                goto nxge_start_fail_lso;
                        }
                        status = 0;
                        goto nxge_start_fail2;
                }
        } /* while (nmp) */

        tx_msg_p->tx_message = mp;
        tx_desc_p = &tx_desc_ring_vp[sop_index];
        npi_desc_handle.regp = (uint64_t)tx_desc_p;

        pkthdrp = (p_tx_pkt_hdr_all_t)hdrp;
        pkthdrp->reserved = 0;
        hdrp->value = 0;
        bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t));

        if (pkt_len > NXGE_MTU_DEFAULT_MAX) {
                tdc_stats->tx_jumbo_pkts++;
        }

        min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2));
        if (pkt_len < min_len) {
                /* Assume we use bcopy to premapped buffers */
                kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "==> nxge_start(14-1): < (msg_min + 16)"
                    "len %d pkt_len %d min_len %d bzero %d ngathers %d",
                    len, pkt_len, min_len, (min_len - pkt_len), ngathers));
                bzero((kaddr + pkt_len), (min_len - pkt_len));
                pkt_len = tx_msg_p->tx_msg_size = min_len;

                sop_tx_desc_p->bits.hdw.tr_len = min_len;

                NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
                tx_desc_p->value = sop_tx_desc_p->value;

                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "==> nxge_start(14-2): < msg_min - "
                    "len %d pkt_len %d min_len %d ngathers %d",
                    len, pkt_len, min_len, ngathers));
        }

        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ",
            cksum_flags));
        {
                uint64_t        tmp_len;

                /* pkt_len already includes 16 + paddings!! */
                /* Update the control header length */
                tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE);
                tmp_len = hdrp->value |
                    (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);

                NXGE_DEBUG_MSG((nxgep, TX_CTL,
                    "==> nxge_start(15_x1): setting SOP "
                    "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len "
                    "0x%llx hdrp->value 0x%llx",
                    tot_xfer_len, tot_xfer_len, pkt_len,
                    tmp_len, hdrp->value));
#if defined(_BIG_ENDIAN)
                hdrp->value = ddi_swap64(tmp_len);
#else
                hdrp->value = tmp_len;
#endif
                NXGE_DEBUG_MSG((nxgep,
                    TX_CTL, "==> nxge_start(15_x2): setting SOP "
                    "after SWAP: tot_xfer_len 0x%llx pkt_len %d "
                    "tmp_len 0x%llx hdrp->value 0x%llx",
                    tot_xfer_len, pkt_len,
                    tmp_len, hdrp->value));
        }

        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP "
            "wr_index %d "
            "tot_xfer_len (%d) pkt_len %d npads %d",
            sop_index,
            tot_xfer_len, pkt_len,
            npads));

        sop_tx_desc_p->bits.hdw.sop = 1;
        sop_tx_desc_p->bits.hdw.mark = mark_mode;
        sop_tx_desc_p->bits.hdw.num_ptr = ngathers;

        NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);

        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done"));

#ifdef NXGE_DEBUG
        npi_desc_handle.nxgep = nxgep;
        npi_desc_handle.function.function = nxgep->function_num;
        npi_desc_handle.function.instance = nxgep->instance;

        NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
            "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n",
            save_desc_p->value,
            sad,
            save_desc_p->bits.hdw.tr_len,
            xfer_len,
            save_desc_p->bits.hdw.num_ptr,
            save_desc_p->bits.hdw.mark,
            save_desc_p->bits.hdw.sop));
        (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index);

        dump_len = (pkt_len > 128) ? 128: pkt_len;
        NXGE_DEBUG_MSG((nxgep, TX_CTL,
            "==> nxge_start: dump packets(17) (after sop set, len "
            " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n"
            "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len,
            (char *)hdrp,
            nxge_dump_packet((char *)hdrp, dump_len)));
        NXGE_DEBUG_MSG((nxgep, TX_CTL,
            "==> nxge_start(18): TX desc sync: sop_index %d",
            sop_index));
#endif

        if ((ngathers == 1) || tx_ring_p->wr_index < i) {
                (void) ddi_dma_sync(tx_desc_dma_handle,
                    sop_index * sizeof (tx_desc_t),
                    ngathers * sizeof (tx_desc_t),
                    DDI_DMA_SYNC_FORDEV);

                NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 "
                    "cs_off = 0x%02X cs_s_off = 0x%02X "
                    "pkt_len %d ngathers %d sop_index %d\n",
                    stuff_offset, start_offset,
                    pkt_len, ngathers, sop_index));
        } else { /* more than one descriptor and wrap around */
                uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index;
                (void) ddi_dma_sync(tx_desc_dma_handle,
                    sop_index * sizeof (tx_desc_t),
                    nsdescs * sizeof (tx_desc_t),
                    DDI_DMA_SYNC_FORDEV);
                NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 "
                    "cs_off = 0x%02X cs_s_off = 0x%02X "
                    "pkt_len %d ngathers %d sop_index %d\n",
                    stuff_offset, start_offset,
                    pkt_len, ngathers, sop_index));

                (void) ddi_dma_sync(tx_desc_dma_handle,
                    0,
                    (ngathers - nsdescs) * sizeof (tx_desc_t),
                    DDI_DMA_SYNC_FORDEV);
                NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 "
                    "cs_off = 0x%02X cs_s_off = 0x%02X "
                    "pkt_len %d ngathers %d sop_index %d\n",
                    stuff_offset, start_offset,
                    pkt_len, ngathers, sop_index));
        }

        tail_index = tx_ring_p->wr_index;
        tail_wrap = tx_ring_p->wr_index_wrap;

        tx_ring_p->wr_index = i;
        if (tx_ring_p->wr_index <= tail_index) {
                tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ?
                    B_FALSE : B_TRUE);
        }

        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: "
            "channel %d wr_index %d wrap %d ngathers %d desc_pend %d",
            tx_ring_p->tdc,
            tx_ring_p->wr_index,
            tx_ring_p->wr_index_wrap,
            ngathers,
            tx_ring_p->descs_pending));

        if (is_lso) {
                lso_ngathers += ngathers;
                if (mp_chain != NULL) {
                        mp = mp_chain;
                        mp_chain = mp_chain->b_next;
                        mp->b_next = NULL;
                        if (nxge_lso_kick_cnt == lso_ngathers) {
                                tx_ring_p->descs_pending += lso_ngathers;
                                {
                                        tx_ring_kick_t          kick;

                                        kick.value = 0;
                                        kick.bits.ldw.wrap =
                                            tx_ring_p->wr_index_wrap;
                                        kick.bits.ldw.tail =
                                            (uint16_t)tx_ring_p->wr_index;

                                        /* Kick the Transmit kick register */
                                        TXDMA_REG_WRITE64(
                                            NXGE_DEV_NPI_HANDLE(nxgep),
                                            TX_RING_KICK_REG,
                                            (uint8_t)tx_ring_p->tdc,
                                            kick.value);
                                        tdc_stats->tx_starts++;

                                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                            "==> nxge_start: more LSO: "
                                            "LSO_CNT %d",
                                            lso_ngathers));
                                }
                                lso_ngathers = 0;
                                ngathers = 0;
                                cur_index_lso = sop_index = tx_ring_p->wr_index;
                                lso_tail_wrap = tx_ring_p->wr_index_wrap;
                        }
                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                            "==> nxge_start: lso again: "
                            "lso_gathers %d ngathers %d cur_index_lso %d "
                            "wr_index %d sop_index %d",
                            lso_ngathers, ngathers, cur_index_lso,
                            tx_ring_p->wr_index, sop_index));

                        NXGE_DEBUG_MSG((nxgep, TX_CTL,
                            "==> nxge_start: next : count %d",
                            lso_ngathers));
                        lso_again = B_TRUE;
                        goto start_again;
                }
                ngathers = lso_ngathers;
        }

        NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: "));

        {
                tx_ring_kick_t          kick;

                kick.value = 0;
                kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap;
                kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index;

                /* Kick start the Transmit kick register */
                TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep),
                    TX_RING_KICK_REG,
                    (uint8_t)tx_ring_p->tdc,
                    kick.value);
        }

        tx_ring_p->descs_pending += ngathers;
        tdc_stats->tx_starts++;

        if (isLDOMservice(nxgep)) {
                tx_ring_p->tx_ring_busy = B_FALSE;
                if (tx_ring_p->tx_ring_offline) {
                        (void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
                            NXGE_TX_RING_OFFLINED);
                }
        }

        MUTEX_EXIT(&tx_ring_p->lock);

        NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
        return (status);

nxge_start_fail_lso:
        status = 0;
        good_packet = B_FALSE;
        if (mp != NULL)
                freemsg(mp);
        if (mp_chain != NULL)
                freemsgchain(mp_chain);

        if (!lso_again && !ngathers) {
                if (isLDOMservice(nxgep)) {
                        tx_ring_p->tx_ring_busy = B_FALSE;
                        if (tx_ring_p->tx_ring_offline) {
                                (void) atomic_swap_32(
                                    &tx_ring_p->tx_ring_offline,
                                    NXGE_TX_RING_OFFLINED);
                        }
                }

                MUTEX_EXIT(&tx_ring_p->lock);
                NXGE_DEBUG_MSG((nxgep, TX_CTL,
                    "==> nxge_start: lso exit (nothing changed)"));
                goto nxge_start_fail1;
        }

        NXGE_DEBUG_MSG((nxgep, TX_CTL,
            "==> nxge_start (channel %d): before lso "
            "lso_gathers %d ngathers %d cur_index_lso %d "
            "wr_index %d sop_index %d lso_again %d",
            tx_ring_p->tdc,
            lso_ngathers, ngathers, cur_index_lso,
            tx_ring_p->wr_index, sop_index, lso_again));

        if (lso_again) {
                lso_ngathers += ngathers;
                ngathers = lso_ngathers;
                sop_index = cur_index_lso;
                tx_ring_p->wr_index = sop_index;
                tx_ring_p->wr_index_wrap = lso_tail_wrap;
        }

        NXGE_DEBUG_MSG((nxgep, TX_CTL,
            "==> nxge_start (channel %d): after lso "
            "lso_gathers %d ngathers %d cur_index_lso %d "
            "wr_index %d sop_index %d lso_again %d",
            tx_ring_p->tdc,
            lso_ngathers, ngathers, cur_index_lso,
            tx_ring_p->wr_index, sop_index, lso_again));

nxge_start_fail2:
        if (good_packet == B_FALSE) {
                cur_index = sop_index;
                NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up"));
                for (i = 0; i < ngathers; i++) {
                        tx_desc_p = &tx_desc_ring_vp[cur_index];
                        npi_handle.regp = (uint64_t)tx_desc_p;
                        tx_msg_p = &tx_msg_ring[cur_index];
                        (void) npi_txdma_desc_set_zero(npi_handle, 1);
                        if (tx_msg_p->flags.dma_type == USE_DVMA) {
                                NXGE_DEBUG_MSG((nxgep, TX_CTL,
                                    "tx_desc_p = %X index = %d",
                                    tx_desc_p, tx_ring_p->rd_index));
                                (void) dvma_unload(tx_msg_p->dvma_handle,
                                    0, -1);
                                tx_msg_p->dvma_handle = NULL;
                                if (tx_ring_p->dvma_wr_index ==
                                    tx_ring_p->dvma_wrap_mask)
                                        tx_ring_p->dvma_wr_index = 0;
                                else
                                        tx_ring_p->dvma_wr_index++;
                                tx_ring_p->dvma_pending--;
                        } else if (tx_msg_p->flags.dma_type == USE_DMA) {
                                if (ddi_dma_unbind_handle(
                                    tx_msg_p->dma_handle)) {
                                        cmn_err(CE_WARN, "!nxge_start: "
                                            "ddi_dma_unbind_handle failed");
                                }
                        }
                        tx_msg_p->flags.dma_type = USE_NONE;
                        cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1,
                            tx_ring_p->tx_wrap_mask);

                }
        }

        if (isLDOMservice(nxgep)) {
                tx_ring_p->tx_ring_busy = B_FALSE;
                if (tx_ring_p->tx_ring_offline) {
                        (void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
                            NXGE_TX_RING_OFFLINED);
                }
        }

        MUTEX_EXIT(&tx_ring_p->lock);

nxge_start_fail1:
        /* Add FMA to check the access handle nxge_hregh */

        NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
        return (status);
}

/* Software LSO starts here */
static void
nxge_hcksum_retrieve(mblk_t *mp,
    uint32_t *start, uint32_t *stuff, uint32_t *end,
    uint32_t *value, uint32_t *flags)
{
        if (mp->b_datap->db_type == M_DATA) {
                if (flags != NULL) {
                        *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM |
                            HCK_PARTIALCKSUM | HCK_FULLCKSUM |
                            HCK_FULLCKSUM_OK);
                        if ((*flags & (HCK_PARTIALCKSUM |
                            HCK_FULLCKSUM)) != 0) {
                                if (value != NULL)
                                        *value = (uint32_t)DB_CKSUM16(mp);
                                if ((*flags & HCK_PARTIALCKSUM) != 0) {
                                        if (start != NULL)
                                                *start =
                                                    (uint32_t)DB_CKSUMSTART(mp);
                                        if (stuff != NULL)
                                                *stuff =
                                                    (uint32_t)DB_CKSUMSTUFF(mp);
                                        if (end != NULL)
                                                *end =
                                                    (uint32_t)DB_CKSUMEND(mp);
                                }
                        }
                }
        }
}

static void
nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags)
{
        ASSERT(DB_TYPE(mp) == M_DATA);

        if (mss == NULL || flags == NULL)
                return;

        *mss = 0;
        if (flags != NULL) {
                *flags = DB_CKSUMFLAGS(mp) & HW_LSO;
                if (*flags != 0) {
                        *mss = (uint32_t)DB_LSOMSS(mp);
                }
                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x",
                    *mss, *flags));
        }

        NXGE_DEBUG_MSG((NULL, TX_CTL,
            "<== nxge_lso_info_get: mss %d", *mss));
}

/*
 * Do Soft LSO on the oversized packet.
 *
 * 1. Create a chain of message for headers.
 * 2. Fill up header messages with proper information.
 * 3. Copy Eithernet, IP, and TCP headers from the original message to
 *    each new message with necessary adjustments.
 *    * Unchange the ethernet header for DIX frames. (by default)
 *    * IP Total Length field is updated to MSS or less(only for the last one).
 *    * IP Identification value is incremented by one for each packet.
 *    * TCP sequence Number is recalculated according to the payload length.
 *    * Set FIN and/or PSH flags for the *last* packet if applied.
 *    * TCP partial Checksum
 * 4. Update LSO information in the first message header.
 * 5. Release the original message header.
 */
static mblk_t *
nxge_do_softlso(mblk_t *mp, uint32_t mss)
{
        uint32_t        hckflags;
        int             pktlen;
        int             hdrlen;
        int             segnum;
        int             i;
        struct ether_vlan_header *evh;
        int             ehlen, iphlen, tcphlen;
        struct ip       *oiph, *niph;
        struct tcphdr *otcph, *ntcph;
        int             available, len, left;
        uint16_t        ip_id;
        uint32_t        tcp_seq;
#ifdef __sparc
        uint32_t        tcp_seq_tmp;
#endif
        mblk_t          *datamp;
        uchar_t         *rptr;
        mblk_t          *nmp;
        mblk_t          *cmp;
        mblk_t          *mp_chain;
        boolean_t do_cleanup = B_FALSE;
        t_uscalar_t start_offset = 0;
        t_uscalar_t stuff_offset = 0;
        t_uscalar_t value = 0;
        uint16_t        l4_len;
        ipaddr_t        src, dst;
        uint32_t        cksum, sum, l4cksum;

        NXGE_DEBUG_MSG((NULL, TX_CTL,
            "==> nxge_do_softlso"));
        /*
         * check the length of LSO packet payload and calculate the number of
         * segments to be generated.
         */
        pktlen = msgsize(mp);
        evh = (struct ether_vlan_header *)mp->b_rptr;

        /* VLAN? */
        if (evh->ether_tpid == htons(ETHERTYPE_VLAN))
                ehlen = sizeof (struct ether_vlan_header);
        else
                ehlen = sizeof (struct ether_header);
        oiph = (struct ip *)(mp->b_rptr + ehlen);
        iphlen = oiph->ip_hl * 4;
        otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen);
        tcphlen = otcph->th_off * 4;

        l4_len = pktlen - ehlen - iphlen;

        NXGE_DEBUG_MSG((NULL, TX_CTL,
            "==> nxge_do_softlso: mss %d oiph $%p "
            "original ip_sum oiph->ip_sum 0x%x "
            "original tcp_sum otcph->th_sum 0x%x "
            "oiph->ip_len %d pktlen %d ehlen %d "
            "l4_len %d (0x%x) ip_len - iphlen %d ",
            mss,
            oiph,
            oiph->ip_sum,
            otcph->th_sum,
            ntohs(oiph->ip_len), pktlen,
            ehlen,
            l4_len,
            l4_len,
            ntohs(oiph->ip_len) - iphlen));

        /* IPv4 + TCP */
        if (!(oiph->ip_v == IPV4_VERSION)) {
                NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
                    "<== nxge_do_softlso: not IPV4 "
                    "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
                    ntohs(oiph->ip_len), pktlen, ehlen,
                    tcphlen));
                freemsg(mp);
                return (NULL);
        }

        if (!(oiph->ip_p == IPPROTO_TCP)) {
                NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
                    "<== nxge_do_softlso: not TCP "
                    "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
                    ntohs(oiph->ip_len), pktlen, ehlen,
                    tcphlen));
                freemsg(mp);
                return (NULL);
        }

        if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) {
                NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
                    "<== nxge_do_softlso: len not matched  "
                    "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
                    ntohs(oiph->ip_len), pktlen, ehlen,
                    tcphlen));
                freemsg(mp);
                return (NULL);
        }

        otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen);
        tcphlen = otcph->th_off * 4;

        /* TCP flags can not include URG, RST, or SYN */
        VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0);

        hdrlen = ehlen + iphlen + tcphlen;

        VERIFY(MBLKL(mp) >= hdrlen);

        if (MBLKL(mp) > hdrlen) {
                datamp = mp;
                rptr = mp->b_rptr + hdrlen;
        } else { /* = */
                datamp = mp->b_cont;
                rptr = datamp->b_rptr;
        }

        NXGE_DEBUG_MSG((NULL, TX_CTL,
            "nxge_do_softlso: otcph $%p pktlen: %d, "
            "hdrlen %d ehlen %d iphlen %d tcphlen %d "
            "mblkl(mp): %d, mblkl(datamp): %d",
            otcph,
            pktlen, hdrlen, ehlen, iphlen, tcphlen,
            (int)MBLKL(mp), (int)MBLKL(datamp)));

        hckflags = 0;
        nxge_hcksum_retrieve(mp,
            &start_offset, &stuff_offset, &value, NULL, &hckflags);

        dst = oiph->ip_dst.s_addr;
        src = oiph->ip_src.s_addr;

        cksum = (dst >> 16) + (dst & 0xFFFF) +
            (src >> 16) + (src & 0xFFFF);
        l4cksum = cksum + IP_TCP_CSUM_COMP;

        sum = l4_len + l4cksum;
        sum = (sum & 0xFFFF) + (sum >> 16);

        NXGE_DEBUG_MSG((NULL, TX_CTL,
            "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x "
            "hckflags 0x%x start_offset %d stuff_offset %d "
            "value (original) 0x%x th_sum 0x%x "
            "pktlen %d l4_len %d (0x%x) "
            "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s",
            dst, src,
            (sum & 0xffff), (~sum & 0xffff),
            hckflags, start_offset, stuff_offset,
            value, otcph->th_sum,
            pktlen,
            l4_len,
            l4_len,
            ntohs(oiph->ip_len) - (int)MBLKL(mp),
            (int)MBLKL(datamp),
            nxge_dump_packet((char *)evh, 12)));

        /*
         * Start to process.
         */
        available = pktlen - hdrlen;
        segnum = (available - 1) / mss + 1;

        NXGE_DEBUG_MSG((NULL, TX_CTL,
            "==> nxge_do_softlso: pktlen %d "
            "MBLKL(mp): %d, MBLKL(datamp): %d "
            "available %d mss %d segnum %d",
            pktlen, (int)MBLKL(mp), (int)MBLKL(datamp),
            available,
            mss,
            segnum));

        VERIFY(segnum >= 2);

        /*
         * Try to pre-allocate all header messages
         */
        mp_chain = NULL;
        for (i = 0; i < segnum; i++) {
                if ((nmp = allocb(hdrlen, 0)) == NULL) {
                        /* Clean up the mp_chain */
                        while (mp_chain != NULL) {
                                nmp = mp_chain;
                                mp_chain = mp_chain->b_next;
                                freemsg(nmp);
                        }
                        NXGE_DEBUG_MSG((NULL, TX_CTL,
                            "<== nxge_do_softlso: "
                            "Could not allocate enough messages for headers!"));
                        freemsg(mp);
                        return (NULL);
                }
                nmp->b_next = mp_chain;
                mp_chain = nmp;

                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "==> nxge_do_softlso: "
                    "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p",
                    mp, nmp, mp_chain, mp_chain->b_next));
        }

        NXGE_DEBUG_MSG((NULL, TX_CTL,
            "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p",
            mp, nmp, mp_chain));

        /*
         * Associate payload with new packets
         */
        cmp = mp_chain;
        left = available;
        while (cmp != NULL) {
                nmp = dupb(datamp);
                if (nmp == NULL) {
                        do_cleanup = B_TRUE;
                        NXGE_DEBUG_MSG((NULL, TX_CTL,
                            "==>nxge_do_softlso: "
                            "Can not dupb(datamp), have to do clean up"));
                        goto cleanup_allocated_msgs;
                }

                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "==> nxge_do_softlso: (loop) before mp $%p cmp $%p "
                    "dupb nmp $%p len %d left %d msd %d ",
                    mp, cmp, nmp, len, left, mss));

                cmp->b_cont = nmp;
                nmp->b_rptr = rptr;
                len = (left < mss) ? left : mss;
                left -= len;

                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "==> nxge_do_softlso: (loop) after mp $%p cmp $%p "
                    "dupb nmp $%p len %d left %d mss %d ",
                    mp, cmp, nmp, len, left, mss));
                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "nxge_do_softlso: before available: %d, "
                    "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
                    available, left, len, segnum, (int)MBLKL(nmp)));

                len -= MBLKL(nmp);
                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "nxge_do_softlso: after available: %d, "
                    "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
                    available, left, len, segnum, (int)MBLKL(nmp)));

                while (len > 0) {
                        mblk_t *mmp = NULL;

                        NXGE_DEBUG_MSG((NULL, TX_CTL,
                            "nxge_do_softlso: (4) len > 0 available: %d, "
                            "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
                            available, left, len, segnum, (int)MBLKL(nmp)));

                        if (datamp->b_cont != NULL) {
                                datamp = datamp->b_cont;
                                rptr = datamp->b_rptr;
                                mmp = dupb(datamp);
                                if (mmp == NULL) {
                                        do_cleanup = B_TRUE;
                                        NXGE_DEBUG_MSG((NULL, TX_CTL,
                                            "==> nxge_do_softlso: "
                                            "Can not dupb(datamp) (1), :"
                                            "have to do clean up"));
                                        NXGE_DEBUG_MSG((NULL, TX_CTL,
                                            "==> nxge_do_softlso: "
                                            "available: %d, left: %d, "
                                            "len: %d, MBLKL(nmp): %d",
                                            available, left, len,
                                            (int)MBLKL(nmp)));
                                        goto cleanup_allocated_msgs;
                                }
                        } else {
                                NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
                                    "==> nxge_do_softlso: "
                                    "(1)available: %d, left: %d, "
                                    "len: %d, MBLKL(nmp): %d",
                                    available, left, len,
                                    (int)MBLKL(nmp)));
                                cmn_err(CE_PANIC,
                                    "==> nxge_do_softlso: "
                                    "Pointers must have been corrupted!\n"
                                    "datamp: $%p, nmp: $%p, rptr: $%p",
                                    (void *)datamp,
                                    (void *)nmp,
                                    (void *)rptr);
                        }
                        nmp->b_cont = mmp;
                        nmp = mmp;
                        len -= MBLKL(nmp);
                }
                if (len < 0) {
                        nmp->b_wptr += len;
                        rptr = nmp->b_wptr;
                        NXGE_DEBUG_MSG((NULL, TX_CTL,
                            "(5) len < 0 (less than 0)"
                            "available: %d, left: %d, len: %d, MBLKL(nmp): %d",
                            available, left, len, (int)MBLKL(nmp)));

                } else if (len == 0) {
                        if (datamp->b_cont != NULL) {
                                NXGE_DEBUG_MSG((NULL, TX_CTL,
                                    "(5) len == 0"
                                    "available: %d, left: %d, len: %d, "
                                    "MBLKL(nmp): %d",
                                    available, left, len, (int)MBLKL(nmp)));
                                datamp = datamp->b_cont;
                                rptr = datamp->b_rptr;
                        } else {
                                NXGE_DEBUG_MSG((NULL, TX_CTL,
                                    "(6)available b_cont == NULL : %d, "
                                    "left: %d, len: %d, MBLKL(nmp): %d",
                                    available, left, len, (int)MBLKL(nmp)));

                                VERIFY(cmp->b_next == NULL);
                                VERIFY(left == 0);
                                break; /* Done! */
                        }
                }
                cmp = cmp->b_next;

                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "(7) do_softlso: "
                    "next mp in mp_chain available len != 0 : %d, "
                    "left: %d, len: %d, MBLKL(nmp): %d",
                    available, left, len, (int)MBLKL(nmp)));
        }

        /*
         * From now, start to fill up all headers for the first message
         * Hardware checksum flags need to be updated separately for FULLCKSUM
         * and PARTIALCKSUM cases. For full checksum, copy the original flags
         * into every new packet is enough. But for HCK_PARTIALCKSUM, all
         * required fields need to be updated properly.
         */
        nmp = mp_chain;
        bcopy(mp->b_rptr, nmp->b_rptr, hdrlen);
        nmp->b_wptr = nmp->b_rptr + hdrlen;
        niph = (struct ip *)(nmp->b_rptr + ehlen);
        niph->ip_len = htons(mss + iphlen + tcphlen);
        ip_id = ntohs(niph->ip_id);
        ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
#ifdef __sparc
        bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4);
        tcp_seq = ntohl(tcp_seq_tmp);
#else
        tcp_seq = ntohl(ntcph->th_seq);
#endif

        ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST);

        DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
        DB_CKSUMSTART(nmp) = start_offset;
        DB_CKSUMSTUFF(nmp) = stuff_offset;

        /* calculate IP checksum and TCP pseudo header checksum */
        niph->ip_sum = 0;
        niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);

        l4_len = mss + tcphlen;
        sum = htons(l4_len) + l4cksum;
        sum = (sum & 0xFFFF) + (sum >> 16);
        ntcph->th_sum = (sum & 0xffff);

        NXGE_DEBUG_MSG((NULL, TX_CTL,
            "==> nxge_do_softlso: first mp $%p (mp_chain $%p) "
            "mss %d pktlen %d l4_len %d (0x%x) "
            "MBLKL(mp): %d, MBLKL(datamp): %d "
            "ip_sum 0x%x "
            "th_sum 0x%x sum 0x%x ) "
            "dump first ip->tcp %s",
            nmp, mp_chain,
            mss,
            pktlen,
            l4_len,
            l4_len,
            (int)MBLKL(mp), (int)MBLKL(datamp),
            niph->ip_sum,
            ntcph->th_sum,
            sum,
            nxge_dump_packet((char *)niph, 52)));

        cmp = nmp;
        while ((nmp = nmp->b_next)->b_next != NULL) {
                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "==>nxge_do_softlso: middle l4_len %d ", l4_len));
                bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen);
                nmp->b_wptr = nmp->b_rptr + hdrlen;
                niph = (struct ip *)(nmp->b_rptr + ehlen);
                niph->ip_id = htons(++ip_id);
                niph->ip_len = htons(mss + iphlen + tcphlen);
                ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
                tcp_seq += mss;

                ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG);

#ifdef __sparc
                tcp_seq_tmp = htonl(tcp_seq);
                bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4);
#else
                ntcph->th_seq = htonl(tcp_seq);
#endif
                DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
                DB_CKSUMSTART(nmp) = start_offset;
                DB_CKSUMSTUFF(nmp) = stuff_offset;

                /* calculate IP checksum and TCP pseudo header checksum */
                niph->ip_sum = 0;
                niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
                ntcph->th_sum = (sum & 0xffff);

                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "==> nxge_do_softlso: middle ip_sum 0x%x "
                    "th_sum 0x%x "
                    " mp $%p (mp_chain $%p) pktlen %d "
                    "MBLKL(mp): %d, MBLKL(datamp): %d ",
                    niph->ip_sum,
                    ntcph->th_sum,
                    nmp, mp_chain,
                    pktlen, (int)MBLKL(mp), (int)MBLKL(datamp)));
        }

        /* Last segment */
        /*
         * Set FIN and/or PSH flags if present only in the last packet.
         * The ip_len could be different from prior packets.
         */
        bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen);
        nmp->b_wptr = nmp->b_rptr + hdrlen;
        niph = (struct ip *)(nmp->b_rptr + ehlen);
        niph->ip_id = htons(++ip_id);
        niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen);
        ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
        tcp_seq += mss;
#ifdef __sparc
        tcp_seq_tmp = htonl(tcp_seq);
        bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4);
#else
        ntcph->th_seq = htonl(tcp_seq);
#endif
        ntcph->th_flags = (otcph->th_flags & ~TH_URG);

        DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
        DB_CKSUMSTART(nmp) = start_offset;
        DB_CKSUMSTUFF(nmp) = stuff_offset;

        /* calculate IP checksum and TCP pseudo header checksum */
        niph->ip_sum = 0;
        niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);

        l4_len = ntohs(niph->ip_len) - iphlen;
        sum = htons(l4_len) + l4cksum;
        sum = (sum & 0xFFFF) + (sum >> 16);
        ntcph->th_sum = (sum & 0xffff);

        NXGE_DEBUG_MSG((NULL, TX_CTL,
            "==> nxge_do_softlso: last next "
            "niph->ip_sum 0x%x "
            "ntcph->th_sum 0x%x sum 0x%x "
            "dump last ip->tcp %s "
            "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) "
            "l4_len %d (0x%x) "
            "MBLKL(mp): %d, MBLKL(datamp): %d ",
            niph->ip_sum,
            ntcph->th_sum, sum,
            nxge_dump_packet((char *)niph, 52),
            cmp, nmp, mp_chain,
            pktlen, pktlen,
            l4_len,
            l4_len,
            (int)MBLKL(mp), (int)MBLKL(datamp)));

cleanup_allocated_msgs:
        if (do_cleanup) {
                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "==> nxge_do_softlso: "
                    "Failed allocating messages, "
                    "have to clean up and fail!"));
                while (mp_chain != NULL) {
                        nmp = mp_chain;
                        mp_chain = mp_chain->b_next;
                        freemsg(nmp);
                }
        }
        /*
         * We're done here, so just free the original message and return the
         * new message chain, that could be NULL if failed, back to the caller.
         */
        freemsg(mp);

        NXGE_DEBUG_MSG((NULL, TX_CTL,
            "<== nxge_do_softlso:mp_chain $%p", mp_chain));
        return (mp_chain);
}

/*
 * Will be called before NIC driver do further operation on the message.
 * The input message may include LSO information, if so, go to softlso logic
 * to eliminate the oversized LSO packet for the incapable underlying h/w.
 * The return could be the same non-LSO message or a message chain for LSO case.
 *
 * The driver needs to call this function per packet and process the whole chain
 * if applied.
 */
static mblk_t *
nxge_lso_eliminate(mblk_t *mp)
{
        uint32_t lsoflags;
        uint32_t mss;

        NXGE_DEBUG_MSG((NULL, TX_CTL,
            "==>nxge_lso_eliminate:"));
        nxge_lso_info_get(mp, &mss, &lsoflags);

        if (lsoflags & HW_LSO) {
                mblk_t *nmp;

                NXGE_DEBUG_MSG((NULL, TX_CTL,
                    "==>nxge_lso_eliminate:"
                    "HW_LSO:mss %d mp $%p",
                    mss, mp));
                if ((nmp = nxge_do_softlso(mp, mss)) != NULL) {
                        NXGE_DEBUG_MSG((NULL, TX_CTL,
                            "<== nxge_lso_eliminate: "
                            "LSO: nmp not NULL nmp $%p mss %d mp $%p",
                            nmp, mss, mp));
                        return (nmp);
                } else {
                        NXGE_DEBUG_MSG((NULL, TX_CTL,
                            "<== nxge_lso_eliminate_ "
                            "LSO: failed nmp NULL nmp $%p mss %d mp $%p",
                            nmp, mss, mp));
                        return (NULL);
                }
        }

        NXGE_DEBUG_MSG((NULL, TX_CTL,
            "<== nxge_lso_eliminate"));
        return (mp);
}

static uint32_t
nxge_csgen(uint16_t *adr, int len)
{
        int             i, odd;
        uint32_t        sum = 0;
        uint32_t        c = 0;

        odd = len % 2;
        for (i = 0; i < (len / 2); i++) {
                sum += (adr[i] & 0xffff);
        }
        if (odd) {
                sum += adr[len / 2] & 0xff00;
        }
        while ((c = ((sum & 0xffff0000) >> 16)) != 0) {
                sum &= 0xffff;
                sum += c;
        }
        return (~sum & 0xffff);
}