/*-
 * SPDX-License-Identifier: BSD-3-Clause
 *
 * Copyright (c) 2024 Google LLC
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 *    list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its contributors
 *    may be used to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "opt_inet6.h"

#include "gve.h"
#include "gve_dqo.h"

static void
gve_unmap_packet(struct gve_tx_ring *tx,
    struct gve_tx_pending_pkt_dqo *pending_pkt)
{
        bus_dmamap_sync(tx->dqo.buf_dmatag, pending_pkt->dmamap,
            BUS_DMASYNC_POSTWRITE);
        bus_dmamap_unload(tx->dqo.buf_dmatag, pending_pkt->dmamap);
}

static void
gve_clear_qpl_pending_pkt(struct gve_tx_pending_pkt_dqo *pending_pkt)
{
        pending_pkt->qpl_buf_head = -1;
        pending_pkt->num_qpl_bufs = 0;
}

static void
gve_free_tx_mbufs_dqo(struct gve_tx_ring *tx)
{
        struct gve_tx_pending_pkt_dqo *pending_pkt;
        int i;

        for (i = 0; i < tx->dqo.num_pending_pkts; i++) {
                pending_pkt = &tx->dqo.pending_pkts[i];
                if (!pending_pkt->mbuf)
                        continue;

                if (gve_is_qpl(tx->com.priv))
                        gve_clear_qpl_pending_pkt(pending_pkt);
                else
                        gve_unmap_packet(tx, pending_pkt);

                m_freem(pending_pkt->mbuf);
                pending_pkt->mbuf = NULL;
        }
}

void
gve_tx_free_ring_dqo(struct gve_priv *priv, int i)
{
        struct gve_tx_ring *tx = &priv->tx[i];
        struct gve_ring_com *com = &tx->com;
        int j;

        if (tx->dqo.desc_ring != NULL) {
                gve_dma_free_coherent(&tx->desc_ring_mem);
                tx->dqo.desc_ring = NULL;
        }

        if (tx->dqo.compl_ring != NULL) {
                gve_dma_free_coherent(&tx->dqo.compl_ring_mem);
                tx->dqo.compl_ring = NULL;
        }

        if (tx->dqo.pending_pkts != NULL) {
                gve_free_tx_mbufs_dqo(tx);

                if (!gve_is_qpl(priv) && tx->dqo.buf_dmatag) {
                        for (j = 0; j < tx->dqo.num_pending_pkts; j++)
                                if (tx->dqo.pending_pkts[j].state !=
                                    GVE_PACKET_STATE_UNALLOCATED)
                                        bus_dmamap_destroy(tx->dqo.buf_dmatag,
                                            tx->dqo.pending_pkts[j].dmamap);
                }

                free(tx->dqo.pending_pkts, M_GVE);
                tx->dqo.pending_pkts = NULL;
        }

        if (!gve_is_qpl(priv) && tx->dqo.buf_dmatag)
                bus_dma_tag_destroy(tx->dqo.buf_dmatag);

        if (gve_is_qpl(priv) && tx->dqo.qpl_bufs != NULL) {
                free(tx->dqo.qpl_bufs, M_GVE);
                tx->dqo.qpl_bufs = NULL;
        }

        if (com->qpl != NULL) {
                gve_free_qpl(priv, com->qpl);
                com->qpl = NULL;
        }
}

static int
gve_tx_alloc_rda_fields_dqo(struct gve_tx_ring *tx)
{
        struct gve_priv *priv = tx->com.priv;
        int err;
        int j;

        /*
         * DMA tag for mapping Tx mbufs
         * The maxsize, nsegments, and maxsegsize params should match
         * the if_sethwtso* arguments in gve_setup_ifnet in gve_main.c.
         */
        err = bus_dma_tag_create(
            bus_get_dma_tag(priv->dev), /* parent */
            1, 0,                       /* alignment, bounds */
            BUS_SPACE_MAXADDR,          /* lowaddr */
            BUS_SPACE_MAXADDR,          /* highaddr */
            NULL, NULL,                 /* filter, filterarg */
            GVE_TSO_MAXSIZE_DQO,        /* maxsize */
            GVE_TX_MAX_DATA_DESCS_DQO,  /* nsegments */
            GVE_TX_MAX_BUF_SIZE_DQO,    /* maxsegsize */
            BUS_DMA_ALLOCNOW,           /* flags */
            NULL,                       /* lockfunc */
            NULL,                       /* lockarg */
            &tx->dqo.buf_dmatag);
        if (err != 0) {
                device_printf(priv->dev, "%s: bus_dma_tag_create failed: %d\n",
                    __func__, err);
                return (err);
        }

        for (j = 0; j < tx->dqo.num_pending_pkts; j++) {
                err = bus_dmamap_create(tx->dqo.buf_dmatag, 0,
                    &tx->dqo.pending_pkts[j].dmamap);
                if (err != 0) {
                        device_printf(priv->dev,
                            "err in creating pending pkt dmamap %d: %d",
                            j, err);
                        return (err);
                }
                tx->dqo.pending_pkts[j].state = GVE_PACKET_STATE_FREE;
        }

        return (0);
}

int
gve_tx_alloc_ring_dqo(struct gve_priv *priv, int i)
{
        struct gve_tx_ring *tx = &priv->tx[i];
        uint16_t num_pending_pkts;
        int err;

        /* Descriptor ring */
        err = gve_dma_alloc_coherent(priv,
            sizeof(union gve_tx_desc_dqo) * priv->tx_desc_cnt,
            CACHE_LINE_SIZE, &tx->desc_ring_mem);
        if (err != 0) {
                device_printf(priv->dev,
                    "Failed to alloc desc ring for tx ring %d", i);
                goto abort;
        }
        tx->dqo.desc_ring = tx->desc_ring_mem.cpu_addr;

        /* Completion ring */
        err = gve_dma_alloc_coherent(priv,
            sizeof(struct gve_tx_compl_desc_dqo) * priv->tx_desc_cnt,
            CACHE_LINE_SIZE, &tx->dqo.compl_ring_mem);
        if (err != 0) {
                device_printf(priv->dev,
                    "Failed to alloc compl ring for tx ring %d", i);
                goto abort;
        }
        tx->dqo.compl_ring = tx->dqo.compl_ring_mem.cpu_addr;

        /*
         * pending_pkts array
         *
         * The max number of pending packets determines the maximum number of
         * descriptors which maybe written to the completion queue.
         *
         * We must set the number small enough to make sure we never overrun the
         * completion queue.
         */
        num_pending_pkts = priv->tx_desc_cnt;
        /*
         * Reserve space for descriptor completions, which will be reported at
         * most every GVE_TX_MIN_RE_INTERVAL packets.
         */
        num_pending_pkts -= num_pending_pkts / GVE_TX_MIN_RE_INTERVAL;

        tx->dqo.num_pending_pkts = num_pending_pkts;
        tx->dqo.pending_pkts = malloc(
            sizeof(struct gve_tx_pending_pkt_dqo) * num_pending_pkts,
            M_GVE, M_WAITOK | M_ZERO);

        if (gve_is_qpl(priv)) {
                int qpl_buf_cnt;

                tx->com.qpl = gve_alloc_qpl(priv, i, GVE_TX_NUM_QPL_PAGES_DQO,
                    /*single_kva*/false);
                if (tx->com.qpl == NULL) {
                        device_printf(priv->dev,
                            "Failed to alloc QPL for tx ring %d", i);
                        err = ENOMEM;
                        goto abort;
                }

                qpl_buf_cnt = GVE_TX_BUFS_PER_PAGE_DQO *
                    tx->com.qpl->num_pages;

                tx->dqo.qpl_bufs = malloc(
                    sizeof(*tx->dqo.qpl_bufs) * qpl_buf_cnt,
                    M_GVE, M_WAITOK | M_ZERO);
        } else
                gve_tx_alloc_rda_fields_dqo(tx);
        return (0);

abort:
        gve_tx_free_ring_dqo(priv, i);
        return (err);
}

static void
gve_extract_tx_metadata_dqo(const struct mbuf *mbuf,
    struct gve_tx_metadata_dqo *metadata)
{
        uint32_t hash = mbuf->m_pkthdr.flowid;
        uint16_t path_hash;

        metadata->version = GVE_TX_METADATA_VERSION_DQO;
        if (hash) {
                path_hash = hash ^ (hash >> 16);

                path_hash &= (1 << 15) - 1;
                if (__predict_false(path_hash == 0))
                        path_hash = ~path_hash;

                metadata->path_hash = path_hash;
        }
}

static void
gve_tx_fill_pkt_desc_dqo(struct gve_tx_ring *tx,
    uint32_t *desc_idx, uint32_t len, uint64_t addr,
    int16_t compl_tag, bool eop, bool csum_enabled)
{
        while (len > 0) {
                struct gve_tx_pkt_desc_dqo *desc =
                    &tx->dqo.desc_ring[*desc_idx].pkt;
                uint32_t cur_len = MIN(len, GVE_TX_MAX_BUF_SIZE_DQO);
                bool cur_eop = eop && cur_len == len;

                *desc = (struct gve_tx_pkt_desc_dqo){
                        .buf_addr = htole64(addr),
                        .dtype = GVE_TX_PKT_DESC_DTYPE_DQO,
                        .end_of_packet = cur_eop,
                        .checksum_offload_enable = csum_enabled,
                        .compl_tag = htole16(compl_tag),
                        .buf_size = cur_len,
                };

                addr += cur_len;
                len -= cur_len;
                *desc_idx = (*desc_idx + 1) & tx->dqo.desc_mask;
        }
}

static void
gve_tx_fill_tso_ctx_desc(struct gve_tx_tso_context_desc_dqo *desc,
    const struct mbuf *mbuf, const struct gve_tx_metadata_dqo *metadata,
    int header_len)
{
        *desc = (struct gve_tx_tso_context_desc_dqo){
                .header_len = header_len,
                .cmd_dtype = {
                        .dtype = GVE_TX_TSO_CTX_DESC_DTYPE_DQO,
                        .tso = 1,
                },
                .flex0 = metadata->bytes[0],
                .flex5 = metadata->bytes[5],
                .flex6 = metadata->bytes[6],
                .flex7 = metadata->bytes[7],
                .flex8 = metadata->bytes[8],
                .flex9 = metadata->bytes[9],
                .flex10 = metadata->bytes[10],
                .flex11 = metadata->bytes[11],
        };
        desc->tso_total_len = mbuf->m_pkthdr.len - header_len;
        desc->mss = mbuf->m_pkthdr.tso_segsz;
}

static void
gve_tx_fill_general_ctx_desc(struct gve_tx_general_context_desc_dqo *desc,
    const struct gve_tx_metadata_dqo *metadata)
{
        *desc = (struct gve_tx_general_context_desc_dqo){
                .flex0 = metadata->bytes[0],
                .flex1 = metadata->bytes[1],
                .flex2 = metadata->bytes[2],
                .flex3 = metadata->bytes[3],
                .flex4 = metadata->bytes[4],
                .flex5 = metadata->bytes[5],
                .flex6 = metadata->bytes[6],
                .flex7 = metadata->bytes[7],
                .flex8 = metadata->bytes[8],
                .flex9 = metadata->bytes[9],
                .flex10 = metadata->bytes[10],
                .flex11 = metadata->bytes[11],
                .cmd_dtype = {.dtype = GVE_TX_GENERAL_CTX_DESC_DTYPE_DQO},
        };
}

#define PULLUP_HDR(m, len)                              \
do {                                                    \
        if (__predict_false((m)->m_len < (len))) {      \
                (m) = m_pullup((m), (len));             \
                if ((m) == NULL)                        \
                        return (EINVAL);                \
        }                                               \
} while (0)

static int
gve_prep_tso(struct mbuf *mbuf, int *header_len)
{
        uint8_t l3_off, l4_off = 0;
        struct ether_header *eh;
        struct tcphdr *th;
        u_short csum;

        PULLUP_HDR(mbuf, sizeof(*eh));
        eh = mtod(mbuf, struct ether_header *);
        KASSERT(eh->ether_type != ETHERTYPE_VLAN,
            ("VLAN-tagged packets not supported"));
        l3_off = ETHER_HDR_LEN;

#ifdef INET6
        if (ntohs(eh->ether_type) == ETHERTYPE_IPV6) {
                struct ip6_hdr *ip6;

                PULLUP_HDR(mbuf, l3_off + sizeof(*ip6));
                ip6 = (struct ip6_hdr *)(mtodo(mbuf, l3_off));
                l4_off = l3_off + sizeof(struct ip6_hdr);
                csum = in6_cksum_pseudo(ip6, /*len=*/0, IPPROTO_TCP,
                    /*csum=*/0);
        } else
#endif
        if (ntohs(eh->ether_type) == ETHERTYPE_IP) {
                struct ip *ip;

                PULLUP_HDR(mbuf, l3_off + sizeof(*ip));
                ip = (struct ip *)(mtodo(mbuf, l3_off));
                l4_off = l3_off + (ip->ip_hl << 2);
                csum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
                    htons(IPPROTO_TCP));
        }

        PULLUP_HDR(mbuf, l4_off + sizeof(struct tcphdr *));
        th = (struct tcphdr *)(mtodo(mbuf, l4_off));
        *header_len = l4_off + (th->th_off << 2);

        /*
         * Hardware requires the th->th_sum to not include the TCP payload,
         * hence we recompute the csum with it excluded.
         */
        th->th_sum = csum;

        return (0);
}

static int
gve_tx_fill_ctx_descs(struct gve_tx_ring *tx, struct mbuf *mbuf,
    bool is_tso, uint32_t *desc_idx)
{
        struct gve_tx_general_context_desc_dqo *gen_desc;
        struct gve_tx_tso_context_desc_dqo *tso_desc;
        struct gve_tx_metadata_dqo metadata;
        int header_len;
        int err;

        metadata = (struct gve_tx_metadata_dqo){0};
        gve_extract_tx_metadata_dqo(mbuf, &metadata);

        if (is_tso) {
                err = gve_prep_tso(mbuf, &header_len);
                if (__predict_false(err)) {
                        counter_enter();
                        counter_u64_add_protected(
                            tx->stats.tx_delayed_pkt_tsoerr, 1);
                        counter_exit();
                        return (err);
                }

                tso_desc = &tx->dqo.desc_ring[*desc_idx].tso_ctx;
                gve_tx_fill_tso_ctx_desc(tso_desc, mbuf, &metadata, header_len);

                *desc_idx = (*desc_idx + 1) & tx->dqo.desc_mask;
                counter_enter();
                counter_u64_add_protected(tx->stats.tso_packet_cnt, 1);
                counter_exit();
        }

        gen_desc = &tx->dqo.desc_ring[*desc_idx].general_ctx;
        gve_tx_fill_general_ctx_desc(gen_desc, &metadata);
        *desc_idx = (*desc_idx + 1) & tx->dqo.desc_mask;
        return (0);
}

static int
gve_map_mbuf_dqo(struct gve_tx_ring *tx,
    struct mbuf **mbuf, bus_dmamap_t dmamap,
    bus_dma_segment_t *segs, int *nsegs, int attempt)
{
        struct mbuf *m_new = NULL;
        int err;

        err = bus_dmamap_load_mbuf_sg(tx->dqo.buf_dmatag, dmamap,
            *mbuf, segs, nsegs, BUS_DMA_NOWAIT);

        switch (err) {
        case __predict_true(0):
                break;
        case EFBIG:
                if (__predict_false(attempt > 0))
                        goto abort;

                counter_enter();
                counter_u64_add_protected(
                    tx->stats.tx_mbuf_collapse, 1);
                counter_exit();

                /* Try m_collapse before m_defrag */
                m_new = m_collapse(*mbuf, M_NOWAIT,
                    GVE_TX_MAX_DATA_DESCS_DQO);
                if (m_new == NULL) {
                        counter_enter();
                        counter_u64_add_protected(
                            tx->stats.tx_mbuf_defrag, 1);
                        counter_exit();
                        m_new = m_defrag(*mbuf, M_NOWAIT);
                }

                if (__predict_false(m_new == NULL)) {
                        counter_enter();
                        counter_u64_add_protected(
                            tx->stats.tx_mbuf_defrag_err, 1);
                        counter_exit();

                        m_freem(*mbuf);
                        *mbuf = NULL;
                        err = ENOMEM;
                        goto abort;
                } else {
                        *mbuf = m_new;
                        return (gve_map_mbuf_dqo(tx, mbuf, dmamap,
                            segs, nsegs, ++attempt));
                }
        case ENOMEM:
                counter_enter();
                counter_u64_add_protected(
                    tx->stats.tx_mbuf_dmamap_enomem_err, 1);
                counter_exit();
                goto abort;
        default:
                goto abort;
        }

        return (0);

abort:
        counter_enter();
        counter_u64_add_protected(tx->stats.tx_mbuf_dmamap_err, 1);
        counter_exit();
        return (err);
}

static uint32_t
num_avail_desc_ring_slots(const struct gve_tx_ring *tx)
{
        uint32_t num_used = (tx->dqo.desc_tail - tx->dqo.desc_head) &
            tx->dqo.desc_mask;

        return (tx->dqo.desc_mask - num_used);
}

static struct gve_tx_pending_pkt_dqo *
gve_alloc_pending_packet(struct gve_tx_ring *tx)
{
        int32_t index = tx->dqo.free_pending_pkts_csm;
        struct gve_tx_pending_pkt_dqo *pending_pkt;

        /*
         * No pending packets available in the consumer list,
         * try to steal the producer list.
         */
        if (__predict_false(index == -1)) {
                tx->dqo.free_pending_pkts_csm = atomic_swap_32(
                    &tx->dqo.free_pending_pkts_prd, -1);

                index = tx->dqo.free_pending_pkts_csm;
                if (__predict_false(index == -1))
                        return (NULL);
        }

        pending_pkt = &tx->dqo.pending_pkts[index];

        /* Remove pending_pkt from the consumer list */
        tx->dqo.free_pending_pkts_csm = pending_pkt->next;
        pending_pkt->state = GVE_PACKET_STATE_PENDING_DATA_COMPL;

        gve_set_timestamp(&pending_pkt->enqueue_time_sec);

        return (pending_pkt);
}

static void
gve_free_pending_packet(struct gve_tx_ring *tx,
    struct gve_tx_pending_pkt_dqo *pending_pkt)
{
        int index = pending_pkt - tx->dqo.pending_pkts;
        int32_t old_head;

        pending_pkt->state = GVE_PACKET_STATE_FREE;

        gve_invalidate_timestamp(&pending_pkt->enqueue_time_sec);

        /* Add pending_pkt to the producer list */
        while (true) {
                old_head = atomic_load_acq_32(&tx->dqo.free_pending_pkts_prd);

                pending_pkt->next = old_head;
                if (atomic_cmpset_32(&tx->dqo.free_pending_pkts_prd,
                    old_head, index))
                        break;
        }
}

/*
 * Has the side-effect of retrieving the value of the last desc index
 * processed by the NIC. hw_tx_head is written to by the completions-processing
 * taskqueue upon receiving descriptor-completions.
 */
static bool
gve_tx_has_desc_room_dqo(struct gve_tx_ring *tx, int needed_descs)
{
        if (needed_descs <= num_avail_desc_ring_slots(tx))
                return (true);

        tx->dqo.desc_head = atomic_load_acq_32(&tx->dqo.hw_tx_head);
        if (needed_descs > num_avail_desc_ring_slots(tx)) {
                counter_enter();
                counter_u64_add_protected(
                    tx->stats.tx_delayed_pkt_nospace_descring, 1);
                counter_exit();
                return (false);
        }

        return (0);
}

static void
gve_tx_request_desc_compl(struct gve_tx_ring *tx, uint32_t desc_idx)
{
        uint32_t last_report_event_interval;
        uint32_t last_desc_idx;

        last_desc_idx = (desc_idx - 1) & tx->dqo.desc_mask;
        last_report_event_interval =
            (last_desc_idx - tx->dqo.last_re_idx) & tx->dqo.desc_mask;

        if (__predict_false(last_report_event_interval >=
            GVE_TX_MIN_RE_INTERVAL)) {
                tx->dqo.desc_ring[last_desc_idx].pkt.report_event = true;
                tx->dqo.last_re_idx = last_desc_idx;
        }
}

static bool
gve_tx_have_enough_qpl_bufs(struct gve_tx_ring *tx, int num_bufs)
{
        uint32_t available = tx->dqo.qpl_bufs_produced_cached -
            tx->dqo.qpl_bufs_consumed;

        if (__predict_true(available >= num_bufs))
                return (true);

        tx->dqo.qpl_bufs_produced_cached = atomic_load_acq_32(
            &tx->dqo.qpl_bufs_produced);
        available = tx->dqo.qpl_bufs_produced_cached -
            tx->dqo.qpl_bufs_consumed;

        if (__predict_true(available >= num_bufs))
                return (true);
        return (false);
}

static int32_t
gve_tx_alloc_qpl_buf(struct gve_tx_ring *tx)
{
        int32_t buf = tx->dqo.free_qpl_bufs_csm;

        if (__predict_false(buf == -1)) {
                tx->dqo.free_qpl_bufs_csm = atomic_swap_32(
                    &tx->dqo.free_qpl_bufs_prd, -1);
                buf = tx->dqo.free_qpl_bufs_csm;
                if (__predict_false(buf == -1))
                        return (-1);
        }

        tx->dqo.free_qpl_bufs_csm = tx->dqo.qpl_bufs[buf];
        tx->dqo.qpl_bufs_consumed++;
        return (buf);
}

/*
 * Tx buffer i corresponds to
 * qpl_page_id = i / GVE_TX_BUFS_PER_PAGE_DQO
 * qpl_page_offset = (i % GVE_TX_BUFS_PER_PAGE_DQO) * GVE_TX_BUF_SIZE_DQO
 */
static void
gve_tx_buf_get_addr_dqo(struct gve_tx_ring *tx,
    int32_t index, void **va, bus_addr_t *dma_addr)
{
        int page_id = index >> (PAGE_SHIFT - GVE_TX_BUF_SHIFT_DQO);
        int offset = (index & (GVE_TX_BUFS_PER_PAGE_DQO - 1)) <<
            GVE_TX_BUF_SHIFT_DQO;

        *va = (char *)tx->com.qpl->dmas[page_id].cpu_addr + offset;
        *dma_addr = tx->com.qpl->dmas[page_id].bus_addr + offset;
}

static struct gve_dma_handle *
gve_get_page_dma_handle(struct gve_tx_ring *tx, int32_t index)
{
        int page_id = index >> (PAGE_SHIFT - GVE_TX_BUF_SHIFT_DQO);

        return (&tx->com.qpl->dmas[page_id]);
}

static void
gve_tx_copy_mbuf_and_write_pkt_descs(struct gve_tx_ring *tx,
    struct mbuf *mbuf, struct gve_tx_pending_pkt_dqo *pkt,
    bool csum_enabled, int16_t completion_tag,
    uint32_t *desc_idx)
{
        int32_t pkt_len = mbuf->m_pkthdr.len;
        struct gve_dma_handle *dma;
        uint32_t copy_offset = 0;
        int32_t prev_buf = -1;
        uint32_t copy_len;
        bus_addr_t addr;
        int32_t buf;
        void *va;

        MPASS(pkt->num_qpl_bufs == 0);
        MPASS(pkt->qpl_buf_head == -1);

        while (copy_offset < pkt_len) {
                buf = gve_tx_alloc_qpl_buf(tx);
                /* We already checked for availability */
                MPASS(buf != -1);

                gve_tx_buf_get_addr_dqo(tx, buf, &va, &addr);
                copy_len = MIN(GVE_TX_BUF_SIZE_DQO, pkt_len - copy_offset);
                m_copydata(mbuf, copy_offset, copy_len, va);
                copy_offset += copy_len;

                dma = gve_get_page_dma_handle(tx, buf);
                bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE);

                gve_tx_fill_pkt_desc_dqo(tx, desc_idx,
                    copy_len, addr, completion_tag,
                    /*eop=*/copy_offset == pkt_len,
                    csum_enabled);

                /* Link all the qpl bufs for a packet */
                if (prev_buf == -1)
                        pkt->qpl_buf_head = buf;
                else
                        tx->dqo.qpl_bufs[prev_buf] = buf;

                prev_buf = buf;
                pkt->num_qpl_bufs++;
        }

        tx->dqo.qpl_bufs[buf] = -1;
}

int
gve_xmit_dqo_qpl(struct gve_tx_ring *tx, struct mbuf *mbuf)
{
        uint32_t desc_idx = tx->dqo.desc_tail;
        struct gve_tx_pending_pkt_dqo *pkt;
        int total_descs_needed;
        int16_t completion_tag;
        bool has_csum_flag;
        int csum_flags;
        bool is_tso;
        int nsegs;
        int err;

        csum_flags = mbuf->m_pkthdr.csum_flags;
        has_csum_flag = csum_flags & (CSUM_TCP | CSUM_UDP |
            CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_TSO);
        is_tso = csum_flags & CSUM_TSO;

        nsegs = howmany(mbuf->m_pkthdr.len, GVE_TX_BUF_SIZE_DQO);
        /* Check if we have enough room in the desc ring */
        total_descs_needed = 1 +     /* general_ctx_desc */
            nsegs +                  /* pkt_desc */
            (is_tso ? 1 : 0);        /* tso_ctx_desc */
        if (__predict_false(!gve_tx_has_desc_room_dqo(tx, total_descs_needed)))
                return (ENOBUFS);

        if (!gve_tx_have_enough_qpl_bufs(tx, nsegs)) {
                counter_enter();
                counter_u64_add_protected(
                    tx->stats.tx_delayed_pkt_nospace_qpl_bufs, 1);
                counter_exit();
                return (ENOBUFS);
        }

        pkt = gve_alloc_pending_packet(tx);
        if (pkt == NULL) {
                counter_enter();
                counter_u64_add_protected(
                    tx->stats.tx_delayed_pkt_nospace_compring, 1);
                counter_exit();
                return (ENOBUFS);
        }
        completion_tag = pkt - tx->dqo.pending_pkts;
        pkt->mbuf = mbuf;

        err = gve_tx_fill_ctx_descs(tx, mbuf, is_tso, &desc_idx);
        if (err)
                goto abort;

        gve_tx_copy_mbuf_and_write_pkt_descs(tx, mbuf, pkt,
            has_csum_flag, completion_tag, &desc_idx);

        /* Remember the index of the last desc written */
        tx->dqo.desc_tail = desc_idx;

        /*
         * Request a descriptor completion on the last descriptor of the
         * packet if we are allowed to by the HW enforced interval.
         */
        gve_tx_request_desc_compl(tx, desc_idx);

        tx->req += total_descs_needed; /* tx->req is just a sysctl counter */
        return (0);

abort:
        pkt->mbuf = NULL;
        gve_free_pending_packet(tx, pkt);
        return (err);
}

int
gve_xmit_dqo(struct gve_tx_ring *tx, struct mbuf **mbuf_ptr)
{
        bus_dma_segment_t segs[GVE_TX_MAX_DATA_DESCS_DQO];
        uint32_t desc_idx = tx->dqo.desc_tail;
        struct gve_tx_pending_pkt_dqo *pkt;
        struct mbuf *mbuf = *mbuf_ptr;
        int total_descs_needed;
        int16_t completion_tag;
        bool has_csum_flag;
        int csum_flags;
        bool is_tso;
        int nsegs;
        int err;
        int i;

        csum_flags = mbuf->m_pkthdr.csum_flags;
        has_csum_flag = csum_flags & (CSUM_TCP | CSUM_UDP |
            CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_TSO);
        is_tso = csum_flags & CSUM_TSO;

        /*
         * This mbuf might end up needing more than 1 pkt desc.
         * The actual number, `nsegs` is known only after the
         * expensive gve_map_mbuf_dqo call. This check beneath
         * exists to fail early when the desc ring is really full.
         */
        total_descs_needed = 1 +     /* general_ctx_desc */
            1 +                      /* pkt_desc */
            (is_tso ? 1 : 0);        /* tso_ctx_desc */
        if (__predict_false(!gve_tx_has_desc_room_dqo(tx, total_descs_needed)))
                return (ENOBUFS);

        pkt = gve_alloc_pending_packet(tx);
        if (pkt == NULL) {
                counter_enter();
                counter_u64_add_protected(
                    tx->stats.tx_delayed_pkt_nospace_compring, 1);
                counter_exit();
                return (ENOBUFS);
        }
        completion_tag = pkt - tx->dqo.pending_pkts;

        err = gve_map_mbuf_dqo(tx, mbuf_ptr, pkt->dmamap,
            segs, &nsegs, /*attempt=*/0);
        if (err)
                goto abort;
        mbuf = *mbuf_ptr;  /* gve_map_mbuf_dqo might replace the mbuf chain */
        pkt->mbuf = mbuf;

        total_descs_needed = 1 + /* general_ctx_desc */
            nsegs +              /* pkt_desc */
            (is_tso ? 1 : 0);    /* tso_ctx_desc */
        if (__predict_false(
            !gve_tx_has_desc_room_dqo(tx, total_descs_needed))) {
                err = ENOBUFS;
                goto abort_with_dma;
        }

        err = gve_tx_fill_ctx_descs(tx, mbuf, is_tso, &desc_idx);
        if (err)
                goto abort_with_dma;

        bus_dmamap_sync(tx->dqo.buf_dmatag, pkt->dmamap, BUS_DMASYNC_PREWRITE);
        for (i = 0; i < nsegs; i++) {
                gve_tx_fill_pkt_desc_dqo(tx, &desc_idx,
                    segs[i].ds_len, segs[i].ds_addr,
                    completion_tag, /*eop=*/i == (nsegs - 1),
                    has_csum_flag);
        }

        /* Remember the index of the last desc written */
        tx->dqo.desc_tail = desc_idx;

        /*
         * Request a descriptor completion on the last descriptor of the
         * packet if we are allowed to by the HW enforced interval.
         */
        gve_tx_request_desc_compl(tx, desc_idx);

        tx->req += total_descs_needed; /* tx->req is just a sysctl counter */
        return (0);

abort_with_dma:
        gve_unmap_packet(tx, pkt);
abort:
        pkt->mbuf = NULL;
        gve_free_pending_packet(tx, pkt);
        return (err);
}

static void
gve_reap_qpl_bufs_dqo(struct gve_tx_ring *tx,
    struct gve_tx_pending_pkt_dqo *pkt)
{
        int32_t buf = pkt->qpl_buf_head;
        struct gve_dma_handle *dma;
        int32_t qpl_buf_tail;
        int32_t old_head;
        int i;

        for (i = 0; i < pkt->num_qpl_bufs; i++) {
                dma = gve_get_page_dma_handle(tx, buf);
                bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_POSTWRITE);
                qpl_buf_tail = buf;
                buf = tx->dqo.qpl_bufs[buf];
        }
        MPASS(buf == -1);
        buf = qpl_buf_tail;

        while (true) {
                old_head = atomic_load_32(&tx->dqo.free_qpl_bufs_prd);
                tx->dqo.qpl_bufs[buf] = old_head;

                /*
                 * The "rel" ensures that the update to dqo.free_qpl_bufs_prd
                 * is visible only after the linked list from this pkt is
                 * attached above to old_head.
                 */
                if (atomic_cmpset_rel_32(&tx->dqo.free_qpl_bufs_prd,
                    old_head, pkt->qpl_buf_head))
                        break;
        }
        /*
         * The "rel" ensures that the update to dqo.qpl_bufs_produced is
         * visible only adter the update to dqo.free_qpl_bufs_prd above.
         */
        atomic_add_rel_32(&tx->dqo.qpl_bufs_produced, pkt->num_qpl_bufs);

        gve_clear_qpl_pending_pkt(pkt);
}

static uint64_t
gve_handle_packet_completion(struct gve_priv *priv,
    struct gve_tx_ring *tx, uint16_t compl_tag)
{
        struct gve_tx_pending_pkt_dqo *pending_pkt;
        int32_t pkt_len;

        if (__predict_false(compl_tag >= tx->dqo.num_pending_pkts)) {
                device_printf(priv->dev, "Invalid TX completion tag: %d\n",
                    compl_tag);
                return (0);
        }

        pending_pkt = &tx->dqo.pending_pkts[compl_tag];

        /* Packet is allocated but not pending data completion. */
        if (__predict_false(pending_pkt->state !=
            GVE_PACKET_STATE_PENDING_DATA_COMPL)) {
                device_printf(priv->dev,
                    "No pending data completion: %d\n", compl_tag);
                return (0);
        }

        pkt_len = pending_pkt->mbuf->m_pkthdr.len;

        if (gve_is_qpl(priv))
                gve_reap_qpl_bufs_dqo(tx, pending_pkt);
        else
                gve_unmap_packet(tx, pending_pkt);

        m_freem(pending_pkt->mbuf);
        pending_pkt->mbuf = NULL;
        gve_free_pending_packet(tx, pending_pkt);
        return (pkt_len);
}

int
gve_check_tx_timeout_dqo(struct gve_priv *priv, struct gve_tx_ring *tx)
{
        struct gve_tx_pending_pkt_dqo *pending_pkt;
        int num_timeouts;
        uint16_t pkt_idx;

        num_timeouts = 0;
        for (pkt_idx = 0; pkt_idx < tx->dqo.num_pending_pkts; pkt_idx++) {
                pending_pkt = &tx->dqo.pending_pkts[pkt_idx];

                if (!gve_timestamp_valid(&pending_pkt->enqueue_time_sec))
                        continue;

                if (__predict_false(
                    gve_seconds_since(&pending_pkt->enqueue_time_sec) >
                    GVE_TX_TIMEOUT_PKT_SEC))
                        num_timeouts += 1;
        }

        return (num_timeouts);
}

int
gve_tx_intr_dqo(void *arg)
{
        struct gve_tx_ring *tx = arg;
        struct gve_priv *priv = tx->com.priv;
        struct gve_ring_com *com = &tx->com;

        if (__predict_false((if_getdrvflags(priv->ifp) & IFF_DRV_RUNNING) == 0))
                return (FILTER_STRAY);

        /* Interrupts are automatically masked */
        taskqueue_enqueue(com->cleanup_tq, &com->cleanup_task);
        return (FILTER_HANDLED);
}

static void
gve_tx_clear_desc_ring_dqo(struct gve_tx_ring *tx)
{
        struct gve_ring_com *com = &tx->com;
        int i;

        for (i = 0; i < com->priv->tx_desc_cnt; i++)
                tx->dqo.desc_ring[i] = (union gve_tx_desc_dqo){};

        bus_dmamap_sync(tx->desc_ring_mem.tag, tx->desc_ring_mem.map,
            BUS_DMASYNC_PREWRITE);
}

static void
gve_tx_clear_compl_ring_dqo(struct gve_tx_ring *tx)
{
        struct gve_ring_com *com = &tx->com;
        int entries;
        int i;

        entries = com->priv->tx_desc_cnt;
        for (i = 0; i < entries; i++)
                tx->dqo.compl_ring[i] = (struct gve_tx_compl_desc_dqo){};

        bus_dmamap_sync(tx->dqo.compl_ring_mem.tag, tx->dqo.compl_ring_mem.map,
            BUS_DMASYNC_PREWRITE);
}

void
gve_clear_tx_ring_dqo(struct gve_priv *priv, int i)
{
        struct gve_tx_ring *tx = &priv->tx[i];
        int j;

        tx->dqo.desc_head = 0;
        tx->dqo.desc_tail = 0;
        tx->dqo.desc_mask = priv->tx_desc_cnt - 1;
        tx->dqo.last_re_idx = 0;

        tx->dqo.compl_head = 0;
        tx->dqo.compl_mask = priv->tx_desc_cnt - 1;
        atomic_store_32(&tx->dqo.hw_tx_head, 0);
        tx->dqo.cur_gen_bit = 0;

        gve_free_tx_mbufs_dqo(tx);

        for (j = 0; j < tx->dqo.num_pending_pkts; j++) {
                if (gve_is_qpl(tx->com.priv))
                        gve_clear_qpl_pending_pkt(&tx->dqo.pending_pkts[j]);
                gve_invalidate_timestamp(
                    &tx->dqo.pending_pkts[j].enqueue_time_sec);
                tx->dqo.pending_pkts[j].next =
                    (j == tx->dqo.num_pending_pkts - 1) ? -1 : j + 1;
                tx->dqo.pending_pkts[j].state = GVE_PACKET_STATE_FREE;
        }
        tx->dqo.free_pending_pkts_csm = 0;
        atomic_store_rel_32(&tx->dqo.free_pending_pkts_prd, -1);

        if (gve_is_qpl(priv)) {
                int qpl_buf_cnt = GVE_TX_BUFS_PER_PAGE_DQO *
                    tx->com.qpl->num_pages;

                for (j = 0; j < qpl_buf_cnt - 1; j++)
                        tx->dqo.qpl_bufs[j] = j + 1;
                tx->dqo.qpl_bufs[j] = -1;

                tx->dqo.free_qpl_bufs_csm = 0;
                atomic_store_32(&tx->dqo.free_qpl_bufs_prd, -1);
                atomic_store_32(&tx->dqo.qpl_bufs_produced, qpl_buf_cnt);
                tx->dqo.qpl_bufs_produced_cached = qpl_buf_cnt;
                tx->dqo.qpl_bufs_consumed = 0;
        }

        gve_tx_clear_desc_ring_dqo(tx);
        gve_tx_clear_compl_ring_dqo(tx);
}

static uint8_t
gve_tx_get_gen_bit(uint8_t *desc)
{
        uint8_t byte;

        /*
         * Prevent generation bit from being read after the rest of the
         * descriptor.
         */
        byte = atomic_load_acq_8(desc + GVE_TX_DESC_DQO_GEN_BYTE_OFFSET);
        return ((byte & GVE_TX_DESC_DQO_GEN_BIT_MASK) != 0);
}

static bool
gve_tx_cleanup_dqo(struct gve_priv *priv, struct gve_tx_ring *tx, int budget)
{
        struct gve_tx_compl_desc_dqo *compl_desc;
        uint64_t bytes_done = 0;
        uint64_t pkts_done = 0;
        uint16_t compl_tag;
        int work_done = 0;
        uint16_t tx_head;
        uint16_t type;

        while (work_done < budget) {
                bus_dmamap_sync(tx->dqo.compl_ring_mem.tag,
                    tx->dqo.compl_ring_mem.map,
                    BUS_DMASYNC_POSTREAD);

                compl_desc = &tx->dqo.compl_ring[tx->dqo.compl_head];
                if (gve_tx_get_gen_bit((uint8_t *)compl_desc) ==
                    tx->dqo.cur_gen_bit)
                        break;

                type = compl_desc->type;
                if (type == GVE_COMPL_TYPE_DQO_DESC) {
                        /* This is the last descriptor fetched by HW plus one */
                        tx_head = le16toh(compl_desc->tx_head);
                        atomic_store_rel_32(&tx->dqo.hw_tx_head, tx_head);
                } else if (type == GVE_COMPL_TYPE_DQO_PKT) {
                        compl_tag = le16toh(compl_desc->completion_tag);
                        bytes_done += gve_handle_packet_completion(priv,
                            tx, compl_tag);
                        pkts_done++;
                }

                tx->dqo.compl_head = (tx->dqo.compl_head + 1) &
                    tx->dqo.compl_mask;
                /* Flip the generation bit when we wrap around */
                tx->dqo.cur_gen_bit ^= tx->dqo.compl_head == 0;
                work_done++;
        }

        /*
         * Waking the xmit taskqueue has to occur after room has been made in
         * the queue.
         */
        atomic_thread_fence_seq_cst();
        if (atomic_load_bool(&tx->stopped) && work_done) {
                atomic_store_bool(&tx->stopped, false);
                taskqueue_enqueue(tx->xmit_tq, &tx->xmit_task);
        }

        tx->done += work_done; /* tx->done is just a sysctl counter */
        counter_enter();
        counter_u64_add_protected(tx->stats.tbytes, bytes_done);
        counter_u64_add_protected(tx->stats.tpackets, pkts_done);
        counter_exit();

        return (work_done == budget);
}

void
gve_tx_cleanup_tq_dqo(void *arg, int pending)
{
        struct gve_tx_ring *tx = arg;
        struct gve_priv *priv = tx->com.priv;

        if (__predict_false((if_getdrvflags(priv->ifp) & IFF_DRV_RUNNING) == 0))
                return;

        if (gve_tx_cleanup_dqo(priv, tx, /*budget=*/1024)) {
                taskqueue_enqueue(tx->com.cleanup_tq, &tx->com.cleanup_task);
                return;
        }

        gve_db_bar_dqo_write_4(priv, tx->com.irq_db_offset,
            GVE_ITR_NO_UPDATE_DQO | GVE_ITR_ENABLE_BIT_DQO);
}