/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <hxge_impl.h>

extern uint32_t hxge_reclaim_pending;
extern uint32_t hxge_bcopy_thresh;
extern uint32_t hxge_dvma_thresh;
extern uint32_t hxge_dma_stream_thresh;
extern uint32_t hxge_tx_minfree;
extern uint32_t hxge_tx_intr_thres;
extern uint32_t hxge_tx_max_gathers;
extern uint32_t hxge_tx_tiny_pack;
extern uint32_t hxge_tx_use_bcopy;

static int hxge_start(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp);

void
hxge_tx_ring_task(void *arg)
{
        p_tx_ring_t     ring = (p_tx_ring_t)arg;

        MUTEX_ENTER(&ring->lock);
        (void) hxge_txdma_reclaim(ring->hxgep, ring, 0);
        MUTEX_EXIT(&ring->lock);

        mac_tx_ring_update(ring->hxgep->mach, ring->ring_handle);
}

static void
hxge_tx_ring_dispatch(p_tx_ring_t ring)
{
        /*
         * Kick the ring task to reclaim some buffers.
         */
        (void) ddi_taskq_dispatch(ring->taskq,
            hxge_tx_ring_task, (void *)ring, DDI_SLEEP);
}

mblk_t *
hxge_tx_ring_send(void *arg, mblk_t *mp)
{
        p_hxge_ring_handle_t    rhp = (p_hxge_ring_handle_t)arg;
        p_hxge_t                hxgep;
        p_tx_ring_t             tx_ring_p;
        int                     status;

        ASSERT(rhp != NULL);
        ASSERT((rhp->index >= 0) && (rhp->index < HXGE_MAX_TDCS));

        hxgep = rhp->hxgep;
        tx_ring_p = hxgep->tx_rings->rings[rhp->index];
        ASSERT(hxgep == tx_ring_p->hxgep);

        status = hxge_start(hxgep, tx_ring_p, mp);
        if (status != 0) {
                hxge_tx_ring_dispatch(tx_ring_p);
                return (mp);
        }

        return ((mblk_t *)NULL);
}

static int
hxge_start(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp)
{
        int                     dma_status, status = 0;
        p_tx_desc_t             tx_desc_ring_vp;
        hpi_handle_t            hpi_desc_handle;
        hxge_os_dma_handle_t    tx_desc_dma_handle;
        p_tx_desc_t             tx_desc_p;
        p_tx_msg_t              tx_msg_ring;
        p_tx_msg_t              tx_msg_p;
        tx_desc_t               tx_desc, *tmp_desc_p;
        tx_desc_t               sop_tx_desc, *sop_tx_desc_p;
        p_tx_pkt_header_t       hdrp;
        p_tx_pkt_hdr_all_t      pkthdrp;
        uint8_t                 npads = 0;
        uint64_t                dma_ioaddr;
        uint32_t                dma_flags;
        int                     last_bidx;
        uint8_t                 *b_rptr;
        caddr_t                 kaddr;
        uint32_t                nmblks;
        uint32_t                ngathers;
        uint32_t                clen;
        int                     len;
        uint32_t                pkt_len, pack_len, min_len;
        uint32_t                bcopy_thresh;
        int                     i, cur_index, sop_index;
        uint16_t                tail_index;
        boolean_t               tail_wrap = B_FALSE;
        hxge_dma_common_t       desc_area;
        hxge_os_dma_handle_t    dma_handle;
        ddi_dma_cookie_t        dma_cookie;
        hpi_handle_t            hpi_handle;
        p_mblk_t                nmp;
        p_mblk_t                t_mp;
        uint32_t                ncookies;
        boolean_t               good_packet;
        boolean_t               mark_mode = B_FALSE;
        p_hxge_stats_t          statsp;
        p_hxge_tx_ring_stats_t  tdc_stats;
        t_uscalar_t             start_offset = 0;
        t_uscalar_t             stuff_offset = 0;
        t_uscalar_t             end_offset = 0;
        t_uscalar_t             value = 0;
        t_uscalar_t             cksum_flags = 0;
        boolean_t               cksum_on = B_FALSE;
        uint32_t                boff = 0;
        uint64_t                tot_xfer_len = 0, tmp_len = 0;
        boolean_t               header_set = B_FALSE;
        tdc_tdr_kick_t          kick;
        uint32_t                offset;
#ifdef HXGE_DEBUG
        p_tx_desc_t             tx_desc_ring_pp;
        p_tx_desc_t             tx_desc_pp;
        tx_desc_t               *save_desc_p;
        int                     dump_len;
        int                     sad_len;
        uint64_t                sad;
        int                     xfer_len;
        uint32_t                msgsize;
#endif

        HXGE_DEBUG_MSG((hxgep, TX_CTL,
            "==> hxge_start: tx dma channel %d", tx_ring_p->tdc));
        HXGE_DEBUG_MSG((hxgep, TX_CTL,
            "==> hxge_start: Starting tdc %d desc pending %d",
            tx_ring_p->tdc, tx_ring_p->descs_pending));

        statsp = hxgep->statsp;

        if (hxgep->statsp->port_stats.lb_mode == hxge_lb_normal) {
                if (!statsp->mac_stats.link_up) {
                        freemsg(mp);
                        HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: "
                            "link not up or LB mode"));
                        goto hxge_start_fail1;
                }
        }

        mac_hcksum_get(mp, &start_offset, &stuff_offset, &end_offset, &value,
            &cksum_flags);
        if (!HXGE_IS_VLAN_PACKET(mp->b_rptr)) {
                start_offset += sizeof (ether_header_t);
                stuff_offset += sizeof (ether_header_t);
        } else {
                start_offset += sizeof (struct ether_vlan_header);
                stuff_offset += sizeof (struct ether_vlan_header);
        }

        if (cksum_flags & HCK_PARTIALCKSUM) {
                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                    "==> hxge_start: mp $%p len %d "
                    "cksum_flags 0x%x (partial checksum) ",
                    mp, MBLKL(mp), cksum_flags));
                cksum_on = B_TRUE;
        }

        MUTEX_ENTER(&tx_ring_p->lock);
        ngathers = 0;
        sop_index = tx_ring_p->wr_index;
#ifdef  HXGE_DEBUG
        if (tx_ring_p->descs_pending) {
                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                    "==> hxge_start: desc pending %d ",
                    tx_ring_p->descs_pending));
        }

        dump_len = (int)(MBLKL(mp));
        dump_len = (dump_len > 128) ? 128: dump_len;

        HXGE_DEBUG_MSG((hxgep, TX_CTL,
            "==> hxge_start: tdc %d: dumping ...: b_rptr $%p "
            "(Before header reserve: ORIGINAL LEN %d)",
            tx_ring_p->tdc, mp->b_rptr, dump_len));

        HXGE_DEBUG_MSG((hxgep, TX_CTL,
            "==> hxge_start: dump packets (IP ORIGINAL b_rptr $%p): %s",
            mp->b_rptr, hxge_dump_packet((char *)mp->b_rptr, dump_len)));
#endif

        tdc_stats = tx_ring_p->tdc_stats;
        mark_mode = (tx_ring_p->descs_pending &&
            ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) <
            hxge_tx_minfree));

        HXGE_DEBUG_MSG((hxgep, TX_CTL,
            "TX Descriptor ring is channel %d mark mode %d",
            tx_ring_p->tdc, mark_mode));

        if (!hxge_txdma_reclaim(hxgep, tx_ring_p, hxge_tx_minfree)) {
                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                    "TX Descriptor ring is full: channel %d", tx_ring_p->tdc));
                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                    "TX Descriptor ring is full: channel %d", tx_ring_p->tdc));
                (void) atomic_cas_32((uint32_t *)&tx_ring_p->queueing, 0, 1);
                tdc_stats->tx_no_desc++;
                MUTEX_EXIT(&tx_ring_p->lock);
                status = 1;
                goto hxge_start_fail1;
        }

        nmp = mp;
        i = sop_index = tx_ring_p->wr_index;
        nmblks = 0;
        ngathers = 0;
        pkt_len = 0;
        pack_len = 0;
        clen = 0;
        last_bidx = -1;
        good_packet = B_TRUE;

        desc_area = tx_ring_p->tdc_desc;
        hpi_handle = desc_area.hpi_handle;
        hpi_desc_handle.regh = (hxge_os_acc_handle_t)
            DMA_COMMON_ACC_HANDLE(desc_area);
        hpi_desc_handle.hxgep = hxgep;
        tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
#ifdef  HXGE_DEBUG
        tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area);
#endif
        tx_desc_dma_handle = (hxge_os_dma_handle_t)DMA_COMMON_HANDLE(desc_area);
        tx_msg_ring = tx_ring_p->tx_msg_ring;

        HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: wr_index %d i %d",
            sop_index, i));

#ifdef  HXGE_DEBUG
        msgsize = msgdsize(nmp);
        HXGE_DEBUG_MSG((hxgep, TX_CTL,
            "==> hxge_start(1): wr_index %d i %d msgdsize %d",
            sop_index, i, msgsize));
#endif
        /*
         * The first 16 bytes of the premapped buffer are reserved
         * for header. No padding will be used.
         */
        pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE;
        if (hxge_tx_use_bcopy) {
                bcopy_thresh = (hxge_bcopy_thresh - TX_PKT_HEADER_SIZE);
        } else {
                bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE);
        }
        while (nmp) {
                good_packet = B_TRUE;
                b_rptr = nmp->b_rptr;
                len = MBLKL(nmp);
                if (len <= 0) {
                        nmp = nmp->b_cont;
                        continue;
                }
                nmblks++;

                HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(1): nmblks %d "
                    "len %d pkt_len %d pack_len %d",
                    nmblks, len, pkt_len, pack_len));
                /*
                 * Hardware limits the transfer length to 4K.
                 * If len is more than 4K, we need to break
                 * nmp into two chunks: Make first chunk smaller
                 * than 4K. The second chunk will be broken into
                 * less than 4K (if needed) during the next pass.
                 */
                if (len > (TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE)) {
                        if ((t_mp = dupb(nmp)) != NULL) {
                                nmp->b_wptr = nmp->b_rptr +
                                    (TX_MAX_TRANSFER_LENGTH -
                                    TX_PKT_HEADER_SIZE);
                                t_mp->b_rptr = nmp->b_wptr;
                                t_mp->b_cont = nmp->b_cont;
                                nmp->b_cont = t_mp;
                                len = MBLKL(nmp);
                        } else {
                                good_packet = B_FALSE;
                                goto hxge_start_fail2;
                        }
                }
                tx_desc.value = 0;
                tx_desc_p = &tx_desc_ring_vp[i];
#ifdef  HXGE_DEBUG
                tx_desc_pp = &tx_desc_ring_pp[i];
#endif
                tx_msg_p = &tx_msg_ring[i];
                hpi_desc_handle.regp = (uint64_t)tx_desc_p;
                if (!header_set &&
                    ((!hxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) ||
                    (len >= bcopy_thresh))) {
                        header_set = B_TRUE;
                        bcopy_thresh += TX_PKT_HEADER_SIZE;
                        boff = 0;
                        pack_len = 0;
                        kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
                        hdrp = (p_tx_pkt_header_t)kaddr;
                        clen = pkt_len;
                        dma_handle = tx_msg_p->buf_dma_handle;
                        dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
                        offset = tx_msg_p->offset_index * hxge_bcopy_thresh;
                        (void) ddi_dma_sync(dma_handle,
                            offset, hxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV);

                        tx_msg_p->flags.dma_type = USE_BCOPY;
                        goto hxge_start_control_header_only;
                }

                pkt_len += len;
                pack_len += len;

                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                    "==> hxge_start(3): desc entry %d DESC IOADDR $%p "
                    "desc_vp $%p tx_desc_p $%p desc_pp $%p tx_desc_pp $%p "
                    "len %d pkt_len %d pack_len %d",
                    i,
                    DMA_COMMON_IOADDR(desc_area),
                    tx_desc_ring_vp, tx_desc_p,
                    tx_desc_ring_pp, tx_desc_pp,
                    len, pkt_len, pack_len));

                if (len < bcopy_thresh) {
                        HXGE_DEBUG_MSG((hxgep, TX_CTL,
                            "==> hxge_start(4): USE BCOPY: "));
                        if (hxge_tx_tiny_pack) {
                                uint32_t blst = TXDMA_DESC_NEXT_INDEX(i, -1,
                                    tx_ring_p->tx_wrap_mask);
                                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                                    "==> hxge_start(5): pack"));
                                if ((pack_len <= bcopy_thresh) &&
                                    (last_bidx == blst)) {
                                        HXGE_DEBUG_MSG((hxgep, TX_CTL,
                                            "==> hxge_start: pack(6) "
                                            "(pkt_len %d pack_len %d)",
                                            pkt_len, pack_len));
                                        i = blst;
                                        tx_desc_p = &tx_desc_ring_vp[i];
#ifdef  HXGE_DEBUG
                                        tx_desc_pp = &tx_desc_ring_pp[i];
#endif
                                        tx_msg_p = &tx_msg_ring[i];
                                        boff = pack_len - len;
                                        ngathers--;
                                } else if (pack_len > bcopy_thresh &&
                                    header_set) {
                                        pack_len = len;
                                        boff = 0;
                                        bcopy_thresh = hxge_bcopy_thresh;
                                        HXGE_DEBUG_MSG((hxgep, TX_CTL,
                                            "==> hxge_start(7): > max NEW "
                                            "bcopy thresh %d "
                                            "pkt_len %d pack_len %d(next)",
                                            bcopy_thresh, pkt_len, pack_len));
                                }
                                last_bidx = i;
                        }
                        kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
                        if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) {
                                hdrp = (p_tx_pkt_header_t)kaddr;
                                header_set = B_TRUE;
                                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                                    "==> hxge_start(7_x2): "
                                    "pkt_len %d pack_len %d (new hdrp $%p)",
                                    pkt_len, pack_len, hdrp));
                        }
                        tx_msg_p->flags.dma_type = USE_BCOPY;
                        kaddr += boff;
                        HXGE_DEBUG_MSG((hxgep, TX_CTL,
                            "==> hxge_start(8): USE BCOPY: before bcopy "
                            "DESC IOADDR $%p entry %d bcopy packets %d "
                            "bcopy kaddr $%p bcopy ioaddr (SAD) $%p "
                            "bcopy clen %d bcopy boff %d",
                            DMA_COMMON_IOADDR(desc_area), i,
                            tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr,
                            clen, boff));
                        HXGE_DEBUG_MSG((hxgep, TX_CTL,
                            "==> hxge_start: 1USE BCOPY: "));
                        HXGE_DEBUG_MSG((hxgep, TX_CTL,
                            "==> hxge_start: 2USE BCOPY: "));
                        HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: "
                            "last USE BCOPY: copy from b_rptr $%p "
                            "to KADDR $%p (len %d offset %d",
                            b_rptr, kaddr, len, boff));
                        bcopy(b_rptr, kaddr, len);
#ifdef  HXGE_DEBUG
                        dump_len = (len > 128) ? 128: len;
                        HXGE_DEBUG_MSG((hxgep, TX_CTL,
                            "==> hxge_start: dump packets "
                            "(After BCOPY len %d)"
                            "(b_rptr $%p): %s", len, nmp->b_rptr,
                            hxge_dump_packet((char *)nmp->b_rptr,
                            dump_len)));
#endif
                        dma_handle = tx_msg_p->buf_dma_handle;
                        dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
                        offset = tx_msg_p->offset_index * hxge_bcopy_thresh;
                        (void) ddi_dma_sync(dma_handle,
                            offset, hxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV);
                        clen = len + boff;
                        tdc_stats->tx_hdr_pkts++;
                        HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(9): "
                            "USE BCOPY: DESC IOADDR $%p entry %d "
                            "bcopy packets %d bcopy kaddr $%p "
                            "bcopy ioaddr (SAD) $%p bcopy clen %d "
                            "bcopy boff %d",
                            DMA_COMMON_IOADDR(desc_area), i,
                            tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr,
                            clen, boff));
                } else {
                        HXGE_DEBUG_MSG((hxgep, TX_CTL,
                            "==> hxge_start(12): USE DVMA: len %d", len));
                        tx_msg_p->flags.dma_type = USE_DMA;
                        dma_flags = DDI_DMA_WRITE;
                        if (len < hxge_dma_stream_thresh) {
                                dma_flags |= DDI_DMA_CONSISTENT;
                        } else {
                                dma_flags |= DDI_DMA_STREAMING;
                        }

                        dma_handle = tx_msg_p->dma_handle;
                        dma_status = ddi_dma_addr_bind_handle(dma_handle, NULL,
                            (caddr_t)b_rptr, len, dma_flags,
                            DDI_DMA_DONTWAIT, NULL,
                            &dma_cookie, &ncookies);
                        if (dma_status == DDI_DMA_MAPPED) {
                                dma_ioaddr = dma_cookie.dmac_laddress;
                                len = (int)dma_cookie.dmac_size;
                                clen = (uint32_t)dma_cookie.dmac_size;
                                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                                    "==> hxge_start(12_1): "
                                    "USE DVMA: len %d clen %d ngathers %d",
                                    len, clen, ngathers));
                                hpi_desc_handle.regp = (uint64_t)tx_desc_p;
                                while (ncookies > 1) {
                                        ngathers++;
                                        /*
                                         * this is the fix for multiple
                                         * cookies, which are basically
                                         * a descriptor entry, we don't set
                                         * SOP bit as well as related fields
                                         */

                                        (void) hpi_txdma_desc_gather_set(
                                            hpi_desc_handle, &tx_desc,
                                            (ngathers -1), mark_mode,
                                            ngathers, dma_ioaddr, clen);
                                        tx_msg_p->tx_msg_size = clen;
                                        HXGE_DEBUG_MSG((hxgep, TX_CTL,
                                            "==> hxge_start:  DMA "
                                            "ncookie %d ngathers %d "
                                            "dma_ioaddr $%p len %d"
                                            "desc $%p descp $%p (%d)",
                                            ncookies, ngathers,
                                            dma_ioaddr, clen,
                                            *tx_desc_p, tx_desc_p, i));

                                        ddi_dma_nextcookie(dma_handle,
                                            &dma_cookie);
                                        dma_ioaddr = dma_cookie.dmac_laddress;

                                        len = (int)dma_cookie.dmac_size;
                                        clen = (uint32_t)dma_cookie.dmac_size;
                                        HXGE_DEBUG_MSG((hxgep, TX_CTL,
                                            "==> hxge_start(12_2): "
                                            "USE DVMA: len %d clen %d ",
                                            len, clen));

                                        i = TXDMA_DESC_NEXT_INDEX(i, 1,
                                            tx_ring_p->tx_wrap_mask);
                                        tx_desc_p = &tx_desc_ring_vp[i];

                                        hpi_desc_handle.regp =
                                            (uint64_t)tx_desc_p;
                                        tx_msg_p = &tx_msg_ring[i];
                                        tx_msg_p->flags.dma_type = USE_NONE;
                                        tx_desc.value = 0;
                                        ncookies--;
                                }
                                tdc_stats->tx_ddi_pkts++;
                                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                                    "==> hxge_start: DMA: ddi packets %d",
                                    tdc_stats->tx_ddi_pkts));
                        } else {
                                HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
                                    "dma mapping failed for %d "
                                    "bytes addr $%p flags %x (%d)",
                                    len, b_rptr, status, status));
                                good_packet = B_FALSE;
                                tdc_stats->tx_dma_bind_fail++;
                                tx_msg_p->flags.dma_type = USE_NONE;
                                status = 1;
                                goto hxge_start_fail2;
                        }
                } /* ddi dvma */

                nmp = nmp->b_cont;
hxge_start_control_header_only:
                hpi_desc_handle.regp = (uint64_t)tx_desc_p;
                ngathers++;

                if (ngathers == 1) {
#ifdef  HXGE_DEBUG
                        save_desc_p = &sop_tx_desc;
#endif
                        sop_tx_desc_p = &sop_tx_desc;
                        sop_tx_desc_p->value = 0;
                        sop_tx_desc_p->bits.tr_len = clen;
                        sop_tx_desc_p->bits.sad = dma_ioaddr >> 32;
                        sop_tx_desc_p->bits.sad_l = dma_ioaddr & 0xffffffff;
                } else {
#ifdef  HXGE_DEBUG
                        save_desc_p = &tx_desc;
#endif
                        tmp_desc_p = &tx_desc;
                        tmp_desc_p->value = 0;
                        tmp_desc_p->bits.tr_len = clen;
                        tmp_desc_p->bits.sad = dma_ioaddr >> 32;
                        tmp_desc_p->bits.sad_l = dma_ioaddr & 0xffffffff;

                        tx_desc_p->value = tmp_desc_p->value;
                }

                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                    "==> hxge_start(13): Desc_entry %d ngathers %d "
                    "desc_vp $%p tx_desc_p $%p "
                    "len %d clen %d pkt_len %d pack_len %d nmblks %d "
                    "dma_ioaddr (SAD) $%p mark %d",
                    i, ngathers, tx_desc_ring_vp, tx_desc_p,
                    len, clen, pkt_len, pack_len, nmblks,
                    dma_ioaddr, mark_mode));

#ifdef HXGE_DEBUG
                hpi_desc_handle.hxgep = hxgep;
                hpi_desc_handle.function.function = 0;
                hpi_desc_handle.function.instance = hxgep->instance;
                sad = save_desc_p->bits.sad;
                sad = (sad << 32) | save_desc_p->bits.sad_l;
                xfer_len = save_desc_p->bits.tr_len;

                HXGE_DEBUG_MSG((hxgep, TX_CTL, "\n\t: value 0x%llx\n"
                    "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t"
                    "mark %d sop %d\n",
                    save_desc_p->value, sad, save_desc_p->bits.tr_len,
                    xfer_len, save_desc_p->bits.num_ptr,
                    save_desc_p->bits.mark, save_desc_p->bits.sop));

                hpi_txdma_dump_desc_one(hpi_desc_handle, NULL, i);
#endif

                tx_msg_p->tx_msg_size = clen;
                i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask);
                if (ngathers > hxge_tx_max_gathers) {
                        good_packet = B_FALSE;
                        mac_hcksum_get(mp, &start_offset, &stuff_offset,
                            &end_offset, &value, &cksum_flags);

                        HXGE_DEBUG_MSG((NULL, TX_CTL,
                            "==> hxge_start(14): pull msg - "
                            "len %d pkt_len %d ngathers %d",
                            len, pkt_len, ngathers));
                        goto hxge_start_fail2;
                }
        } /* while (nmp) */

        tx_msg_p->tx_message = mp;
        tx_desc_p = &tx_desc_ring_vp[sop_index];
        hpi_desc_handle.regp = (uint64_t)tx_desc_p;

        pkthdrp = (p_tx_pkt_hdr_all_t)hdrp;
        pkthdrp->reserved = 0;
        hdrp->value = 0;
        (void) hxge_fill_tx_hdr(mp, B_FALSE, cksum_on,
            (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp);

        /*
         * Hardware header should not be counted as part of the frame
         * when determining the frame size
         */
        if ((pkt_len - TX_PKT_HEADER_SIZE) > (STD_FRAME_SIZE - ETHERFCSL)) {
                tdc_stats->tx_jumbo_pkts++;
        }

        min_len = (hxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2));
        if (pkt_len < min_len) {
                /* Assume we use bcopy to premapped buffers */
                kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
                HXGE_DEBUG_MSG((NULL, TX_CTL,
                    "==> hxge_start(14-1): < (msg_min + 16)"
                    "len %d pkt_len %d min_len %d bzero %d ngathers %d",
                    len, pkt_len, min_len, (min_len - pkt_len), ngathers));
                bzero((kaddr + pkt_len), (min_len - pkt_len));
                pkt_len = tx_msg_p->tx_msg_size = min_len;

                sop_tx_desc_p->bits.tr_len = min_len;

                HXGE_MEM_PIO_WRITE64(hpi_desc_handle, sop_tx_desc_p->value);
                tx_desc_p->value = sop_tx_desc_p->value;

                HXGE_DEBUG_MSG((NULL, TX_CTL,
                    "==> hxge_start(14-2): < msg_min - "
                    "len %d pkt_len %d min_len %d ngathers %d",
                    len, pkt_len, min_len, ngathers));
        }

        HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: cksum_flags 0x%x ",
            cksum_flags));
        if (cksum_flags & HCK_PARTIALCKSUM) {
                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                    "==> hxge_start: cksum_flags 0x%x (partial checksum) ",
                    cksum_flags));
                cksum_on = B_TRUE;
                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                    "==> hxge_start: from IP cksum_flags 0x%x "
                    "(partial checksum) "
                    "start_offset %d stuff_offset %d",
                    cksum_flags, start_offset, stuff_offset));
                tmp_len = (uint64_t)(start_offset >> 1);
                hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT);
                tmp_len = (uint64_t)(stuff_offset >> 1);
                hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT);

                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                    "==> hxge_start: from IP cksum_flags 0x%x "
                    "(partial checksum) "
                    "after SHIFT start_offset %d stuff_offset %d",
                    cksum_flags, start_offset, stuff_offset));
        }

        /*
         * pkt_len already includes 16 + paddings!!
         * Update the control header length
         */

        /*
         * Note that Hydra is different from Neptune where
         * tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE);
         */
        tot_xfer_len = pkt_len;
        tmp_len = hdrp->value |
            (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);

        HXGE_DEBUG_MSG((hxgep, TX_CTL,
            "==> hxge_start(15_x1): setting SOP "
            "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len "
            "0x%llx hdrp->value 0x%llx",
            tot_xfer_len, tot_xfer_len, pkt_len, tmp_len, hdrp->value));
#if defined(_BIG_ENDIAN)
        hdrp->value = ddi_swap64(tmp_len);
#else
        hdrp->value = tmp_len;
#endif
        HXGE_DEBUG_MSG((hxgep,
            TX_CTL, "==> hxge_start(15_x2): setting SOP "
            "after SWAP: tot_xfer_len 0x%llx pkt_len %d "
            "tmp_len 0x%llx hdrp->value 0x%llx",
            tot_xfer_len, pkt_len, tmp_len, hdrp->value));

        HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(15): setting SOP "
            "wr_index %d tot_xfer_len (%d) pkt_len %d npads %d",
            sop_index, tot_xfer_len, pkt_len, npads));

        sop_tx_desc_p->bits.sop = 1;
        sop_tx_desc_p->bits.mark = mark_mode;
        sop_tx_desc_p->bits.num_ptr = ngathers;

        if (mark_mode)
                tdc_stats->tx_marks++;

        HXGE_MEM_PIO_WRITE64(hpi_desc_handle, sop_tx_desc_p->value);
        HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(16): set SOP done"));

#ifdef HXGE_DEBUG
        hpi_desc_handle.hxgep = hxgep;
        hpi_desc_handle.function.function = 0;
        hpi_desc_handle.function.instance = hxgep->instance;

        HXGE_DEBUG_MSG((hxgep, TX_CTL, "\n\t: value 0x%llx\n"
            "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n",
            save_desc_p->value, sad, save_desc_p->bits.tr_len,
            xfer_len, save_desc_p->bits.num_ptr, save_desc_p->bits.mark,
            save_desc_p->bits.sop));
        (void) hpi_txdma_dump_desc_one(hpi_desc_handle, NULL, sop_index);

        dump_len = (pkt_len > 128) ? 128: pkt_len;
        HXGE_DEBUG_MSG((hxgep, TX_CTL,
            "==> hxge_start: dump packets(17) (after sop set, len "
            " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n"
            "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len,
            (char *)hdrp, hxge_dump_packet((char *)hdrp, dump_len)));
        HXGE_DEBUG_MSG((hxgep, TX_CTL,
            "==> hxge_start(18): TX desc sync: sop_index %d", sop_index));
#endif

        if ((ngathers == 1) || tx_ring_p->wr_index < i) {
                (void) ddi_dma_sync(tx_desc_dma_handle,
                    sop_index * sizeof (tx_desc_t),
                    ngathers * sizeof (tx_desc_t), DDI_DMA_SYNC_FORDEV);

                HXGE_DEBUG_MSG((hxgep, TX_CTL, "hxge_start(19): sync 1 "
                    "cs_off = 0x%02X cs_s_off = 0x%02X "
                    "pkt_len %d ngathers %d sop_index %d\n",
                    stuff_offset, start_offset,
                    pkt_len, ngathers, sop_index));
        } else { /* more than one descriptor and wrap around */
                uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index;
                (void) ddi_dma_sync(tx_desc_dma_handle,
                    sop_index * sizeof (tx_desc_t),
                    nsdescs * sizeof (tx_desc_t), DDI_DMA_SYNC_FORDEV);
                HXGE_DEBUG_MSG((hxgep, TX_CTL, "hxge_start(20): sync 1 "
                    "cs_off = 0x%02X cs_s_off = 0x%02X "
                    "pkt_len %d ngathers %d sop_index %d\n",
                    stuff_offset, start_offset, pkt_len, ngathers, sop_index));

                (void) ddi_dma_sync(tx_desc_dma_handle, 0,
                    (ngathers - nsdescs) * sizeof (tx_desc_t),
                    DDI_DMA_SYNC_FORDEV);
                HXGE_DEBUG_MSG((hxgep, TX_CTL, "hxge_start(21): sync 2 "
                    "cs_off = 0x%02X cs_s_off = 0x%02X "
                    "pkt_len %d ngathers %d sop_index %d\n",
                    stuff_offset, start_offset,
                    pkt_len, ngathers, sop_index));
        }

        tail_index = tx_ring_p->wr_index;
        tail_wrap = tx_ring_p->wr_index_wrap;

        tx_ring_p->wr_index = i;
        if (tx_ring_p->wr_index <= tail_index) {
                tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ?
                    B_FALSE : B_TRUE);
        }

        tx_ring_p->descs_pending += ngathers;
        HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: TX kick: "
            "channel %d wr_index %d wrap %d ngathers %d desc_pend %d",
            tx_ring_p->tdc, tx_ring_p->wr_index, tx_ring_p->wr_index_wrap,
            ngathers, tx_ring_p->descs_pending));
        HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: TX KICKING: "));

        kick.value = 0;
        kick.bits.wrap = tx_ring_p->wr_index_wrap;
        kick.bits.tail = (uint16_t)tx_ring_p->wr_index;

        /* Kick start the Transmit kick register */
        TXDMA_REG_WRITE64(HXGE_DEV_HPI_HANDLE(hxgep),
            TDC_TDR_KICK, (uint8_t)tx_ring_p->tdc, kick.value);
        tdc_stats->tx_starts++;
        MUTEX_EXIT(&tx_ring_p->lock);
        HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_start"));
        return (status);

hxge_start_fail2:
        if (good_packet == B_FALSE) {
                cur_index = sop_index;
                HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: clean up"));
                for (i = 0; i < ngathers; i++) {
                        tx_desc_p = &tx_desc_ring_vp[cur_index];
                        hpi_handle.regp = (uint64_t)tx_desc_p;
                        tx_msg_p = &tx_msg_ring[cur_index];
                        (void) hpi_txdma_desc_set_zero(hpi_handle, 1);
                        if (tx_msg_p->flags.dma_type == USE_DVMA) {
                                HXGE_DEBUG_MSG((hxgep, TX_CTL,
                                    "tx_desc_p = %X index = %d",
                                    tx_desc_p, tx_ring_p->rd_index));
                                (void) dvma_unload(tx_msg_p->dvma_handle,
                                    0, -1);
                                tx_msg_p->dvma_handle = NULL;
                                if (tx_ring_p->dvma_wr_index ==
                                    tx_ring_p->dvma_wrap_mask)
                                        tx_ring_p->dvma_wr_index = 0;
                                else
                                        tx_ring_p->dvma_wr_index++;
                                tx_ring_p->dvma_pending--;
                        } else if (tx_msg_p->flags.dma_type == USE_DMA) {
                                if (ddi_dma_unbind_handle(
                                    tx_msg_p->dma_handle)) {
                                        cmn_err(CE_WARN, "hxge_start: "
                                            "ddi_dma_unbind_handle failed");
                                }
                        }
                        tx_msg_p->flags.dma_type = USE_NONE;
                        cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1,
                            tx_ring_p->tx_wrap_mask);

                }
        }

        MUTEX_EXIT(&tx_ring_p->lock);

hxge_start_fail1:
        /* Add FMA to check the access handle hxge_hregh */
        HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_start"));
        return (status);
}