/*-
 * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
 * Copyright (c) 2022 NVIDIA corporation & affiliates.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "opt_kern_tls.h"
#include "opt_rss.h"
#include "opt_ratelimit.h"

#include <dev/mlx5/mlx5_en/en.h>
#include <machine/atomic.h>
#include <dev/mlx5/mlx5_accel/ipsec.h>

static inline bool
mlx5e_do_send_cqe_inline(struct mlx5e_sq *sq)
{
        sq->cev_counter++;
        /* interleave the CQEs */
        if (sq->cev_counter >= sq->cev_factor) {
                sq->cev_counter = 0;
                return (true);
        }
        return (false);
}

bool
mlx5e_do_send_cqe(struct mlx5e_sq *sq)
{

        return (mlx5e_do_send_cqe_inline(sq));
}

void
mlx5e_send_nop(struct mlx5e_sq *sq, u32 ds_cnt)
{
        u16 pi = sq->pc & sq->wq.sz_m1;
        struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);

        memset(&wqe->ctrl, 0, sizeof(wqe->ctrl));

        wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);
        wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
        if (mlx5e_do_send_cqe_inline(sq))
                wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
        else
                wqe->ctrl.fm_ce_se = 0;

        /* Copy data for doorbell */
        memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));

        sq->mbuf[pi].mbuf = NULL;
        sq->mbuf[pi].num_bytes = 0;
        sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
        sq->pc += sq->mbuf[pi].num_wqebbs;
}

static uint32_t mlx5e_hash_value;

static void
mlx5e_hash_init(void *arg)
{
        mlx5e_hash_value = m_ether_tcpip_hash_init();
}

/* Make kernel call mlx5e_hash_init after the random stack finished initializing */
SYSINIT(mlx5e_hash_init, SI_SUB_RANDOM, SI_ORDER_ANY, &mlx5e_hash_init, NULL);

static struct mlx5e_sq *
mlx5e_select_queue_by_send_tag(if_t ifp, struct mbuf *mb)
{
        struct m_snd_tag *mb_tag;
        struct mlx5e_sq *sq;

        mb_tag = mb->m_pkthdr.snd_tag;

#ifdef KERN_TLS
top:
#endif
        /* get pointer to sendqueue */
        switch (mb_tag->sw->type) {
#ifdef RATELIMIT
        case IF_SND_TAG_TYPE_RATE_LIMIT:
                sq = container_of(mb_tag,
                    struct mlx5e_rl_channel, tag)->sq;
                break;
#ifdef KERN_TLS
        case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
                mb_tag = container_of(mb_tag, struct mlx5e_tls_tag, tag)->rl_tag;
                goto top;
#endif
#endif
        case IF_SND_TAG_TYPE_UNLIMITED:
                sq = &container_of(mb_tag,
                    struct mlx5e_channel, tag)->sq[0];
                KASSERT((mb_tag->refcount > 0),
                    ("mlx5e_select_queue: Channel refs are zero for unlimited tag"));
                break;
#ifdef KERN_TLS
        case IF_SND_TAG_TYPE_TLS:
                mb_tag = container_of(mb_tag, struct mlx5e_tls_tag, tag)->rl_tag;
                goto top;
#endif
        default:
                sq = NULL;
                break;
        }

        /* check if valid */
        if (sq != NULL && READ_ONCE(sq->running) != 0)
                return (sq);

        return (NULL);
}

static struct mlx5e_sq *
mlx5e_select_queue(if_t ifp, struct mbuf *mb)
{
        struct mlx5e_priv *priv = if_getsoftc(ifp);
        struct mlx5e_sq *sq;
        u32 ch;
        u32 tc;

        /* obtain VLAN information if present */
        if (mb->m_flags & M_VLANTAG) {
                tc = (mb->m_pkthdr.ether_vtag >> 13);
                if (tc >= priv->num_tc)
                        tc = priv->default_vlan_prio;
        } else {
                tc = priv->default_vlan_prio;
        }

        ch = priv->params.num_channels;

        /* check if flowid is set */
        if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) {
#ifdef RSS
                u32 temp;

                if (rss_hash2bucket(mb->m_pkthdr.flowid,
                    M_HASHTYPE_GET(mb), &temp) == 0)
                        ch = temp % ch;
                else
#endif
                        ch = (mb->m_pkthdr.flowid % 128) % ch;
        } else {
                ch = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 |
                    MBUF_HASHFLAG_L4, mb, mlx5e_hash_value) % ch;
        }

        /* check if send queue is running */
        sq = &priv->channel[ch].sq[tc];
        if (likely(READ_ONCE(sq->running) != 0))
                return (sq);
        return (NULL);
}

static inline u16
mlx5e_get_l2_header_size(struct mlx5e_sq *sq, struct mbuf *mb)
{
        struct ether_vlan_header *eh;
        uint16_t eth_type;
        int min_inline;

        eh = mtod(mb, struct ether_vlan_header *);
        if (unlikely(mb->m_len < ETHER_HDR_LEN)) {
                goto max_inline;
        } else if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
                if (unlikely(mb->m_len < (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)))
                        goto max_inline;
                eth_type = ntohs(eh->evl_proto);
                min_inline = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
        } else {
                eth_type = ntohs(eh->evl_encap_proto);
                min_inline = ETHER_HDR_LEN;
        }

        switch (eth_type) {
        case ETHERTYPE_IP:
        case ETHERTYPE_IPV6:
                /*
                 * Make sure the TOS(IPv4) or traffic class(IPv6)
                 * field gets inlined. Else the SQ may stall.
                 */
                min_inline += 4;
                break;
        default:
                goto max_inline;
        }

        /*
         * m_copydata() will be used on the remaining header which
         * does not need to reside within the first m_len bytes of
         * data:
         */
        if (mb->m_pkthdr.len < min_inline)
                goto max_inline;
        return (min_inline);

max_inline:
        return (MIN(mb->m_pkthdr.len, sq->max_inline));
}

/*
 * This function parse IPv4 and IPv6 packets looking for TCP and UDP
 * headers.
 *
 * Upon return the pointer at which the "ppth" argument points, is set
 * to the location of the TCP header. NULL is used if no TCP header is
 * present.
 *
 * The return value indicates the number of bytes from the beginning
 * of the packet until the first byte after the TCP or UDP header. If
 * this function returns zero, the parsing failed.
 */
int
mlx5e_get_full_header_size(const struct mbuf *mb, const struct tcphdr **ppth)
{
        const struct ether_vlan_header *eh;
        const struct tcphdr *th;
        const struct ip *ip;
        int ip_hlen, tcp_hlen;
        const struct ip6_hdr *ip6;
        uint16_t eth_type;
        int eth_hdr_len;

        eh = mtod(mb, const struct ether_vlan_header *);
        if (unlikely(mb->m_len < ETHER_HDR_LEN))
                goto failure;
        if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
                if (unlikely(mb->m_len < ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN))
                        goto failure;
                eth_type = ntohs(eh->evl_proto);
                eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
        } else {
                eth_type = ntohs(eh->evl_encap_proto);
                eth_hdr_len = ETHER_HDR_LEN;
        }

        switch (eth_type) {
        case ETHERTYPE_IP:
                ip = (const struct ip *)(mb->m_data + eth_hdr_len);
                if (unlikely(mb->m_len < eth_hdr_len + sizeof(*ip)))
                        goto failure;
                switch (ip->ip_p) {
                case IPPROTO_TCP:
                        ip_hlen = ip->ip_hl << 2;
                        eth_hdr_len += ip_hlen;
                        goto tcp_packet;
                case IPPROTO_UDP:
                        ip_hlen = ip->ip_hl << 2;
                        eth_hdr_len += ip_hlen + sizeof(struct udphdr);
                        th = NULL;
                        goto udp_packet;
                default:
                        goto failure;
                }
                break;
        case ETHERTYPE_IPV6:
                ip6 = (const struct ip6_hdr *)(mb->m_data + eth_hdr_len);
                if (unlikely(mb->m_len < eth_hdr_len + sizeof(*ip6)))
                        goto failure;
                switch (ip6->ip6_nxt) {
                case IPPROTO_TCP:
                        eth_hdr_len += sizeof(*ip6);
                        goto tcp_packet;
                case IPPROTO_UDP:
                        eth_hdr_len += sizeof(*ip6) + sizeof(struct udphdr);
                        th = NULL;
                        goto udp_packet;
                default:
                        goto failure;
                }
                break;
        default:
                goto failure;
        }
tcp_packet:
        if (unlikely(mb->m_len < eth_hdr_len + sizeof(*th))) {
                const struct mbuf *m_th = mb->m_next;
                if (unlikely(mb->m_len != eth_hdr_len ||
                    m_th == NULL || m_th->m_len < sizeof(*th)))
                        goto failure;
                th = (const struct tcphdr *)(m_th->m_data);
        } else {
                th = (const struct tcphdr *)(mb->m_data + eth_hdr_len);
        }
        tcp_hlen = th->th_off << 2;
        eth_hdr_len += tcp_hlen;
udp_packet:
        /*
         * m_copydata() will be used on the remaining header which
         * does not need to reside within the first m_len bytes of
         * data:
         */
        if (unlikely(mb->m_pkthdr.len < eth_hdr_len))
                goto failure;
        if (ppth != NULL)
                *ppth = th;
        return (eth_hdr_len);
failure:
        if (ppth != NULL)
                *ppth = NULL;
        return (0);
}

/*
 * Locate a pointer inside a mbuf chain. Returns NULL upon failure.
 */
static inline void *
mlx5e_parse_mbuf_chain(const struct mbuf **mb, int *poffset, int eth_hdr_len,
    int min_len)
{
        if (unlikely(mb[0]->m_len == eth_hdr_len)) {
                poffset[0] = eth_hdr_len;
                if (unlikely((mb[0] = mb[0]->m_next) == NULL))
                        return (NULL);
        }
        if (unlikely(mb[0]->m_len < eth_hdr_len - poffset[0] + min_len))
                return (NULL);
        return (mb[0]->m_data + eth_hdr_len - poffset[0]);
}

/*
 * This function parse IPv4 and IPv6 packets looking for UDP, VXLAN
 * and TCP headers.
 *
 * The return value indicates the number of bytes from the beginning
 * of the packet until the first byte after the TCP header. If this
 * function returns zero, the parsing failed.
 */
static int
mlx5e_get_vxlan_header_size(const struct mbuf *mb, struct mlx5e_tx_wqe *wqe,
    uint8_t cs_mask, uint8_t opcode)
{
        const struct ether_vlan_header *eh;
        struct ip *ip4;
        struct ip6_hdr *ip6;
        struct tcphdr *th;
        struct udphdr *udp;
        bool has_outer_vlan_tag;
        uint16_t eth_type;
        uint8_t ip_type;
        int pkt_hdr_len;
        int eth_hdr_len;
        int tcp_hlen;
        int ip_hlen;
        int offset;

        pkt_hdr_len = mb->m_pkthdr.len;
        has_outer_vlan_tag = (mb->m_flags & M_VLANTAG) != 0;
        offset = 0;

        eh = mtod(mb, const struct ether_vlan_header *);
        if (unlikely(mb->m_len < ETHER_HDR_LEN))
                return (0);

        if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
                if (unlikely(mb->m_len < ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN))
                        return (0);
                eth_type = eh->evl_proto;
                eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
        } else {
                eth_type = eh->evl_encap_proto;
                eth_hdr_len = ETHER_HDR_LEN;
        }

        switch (eth_type) {
        case htons(ETHERTYPE_IP):
                ip4 = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len,
                    sizeof(*ip4));
                if (unlikely(ip4 == NULL))
                        return (0);
                ip_type = ip4->ip_p;
                if (unlikely(ip_type != IPPROTO_UDP))
                        return (0);
                wqe->eth.swp_outer_l3_offset = eth_hdr_len / 2;
                wqe->eth.cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM;
                ip_hlen = ip4->ip_hl << 2;
                eth_hdr_len += ip_hlen;
                udp = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len,
                    sizeof(*udp));
                if (unlikely(udp == NULL))
                        return (0);
                wqe->eth.swp_outer_l4_offset = eth_hdr_len / 2;
                wqe->eth.swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_TYPE;
                eth_hdr_len += sizeof(*udp);
                break;
        case htons(ETHERTYPE_IPV6):
                ip6 = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len,
                    sizeof(*ip6));
                if (unlikely(ip6 == NULL))
                        return (0);
                ip_type = ip6->ip6_nxt;
                if (unlikely(ip_type != IPPROTO_UDP))
                        return (0);
                wqe->eth.swp_outer_l3_offset = eth_hdr_len / 2;
                wqe->eth.cs_flags = MLX5_ETH_WQE_L4_CSUM;
                eth_hdr_len += sizeof(*ip6);
                udp = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len,
                    sizeof(*udp));
                if (unlikely(udp == NULL))
                        return (0);
                wqe->eth.swp_outer_l4_offset = eth_hdr_len / 2;
                wqe->eth.swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_TYPE |
                    MLX5_ETH_WQE_SWP_OUTER_L3_TYPE;
                eth_hdr_len += sizeof(*udp);
                break;
        default:
                return (0);
        }

        /*
         * If the hardware is not computing inner IP checksum, then
         * skip inlining the inner outer UDP and VXLAN header:
         */
        if (unlikely((cs_mask & MLX5_ETH_WQE_L3_INNER_CSUM) == 0))
                goto done;
        if (unlikely(mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len,
            8) == NULL))
                return (0);
        eth_hdr_len += 8;

        /* Check for ethernet header again. */
        eh = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len, ETHER_HDR_LEN);
        if (unlikely(eh == NULL))
                return (0);
        if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
                if (unlikely(mb->m_len < eth_hdr_len - offset + ETHER_HDR_LEN +
                    ETHER_VLAN_ENCAP_LEN))
                        return (0);
                eth_type = eh->evl_proto;
                eth_hdr_len += ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
        } else {
                eth_type = eh->evl_encap_proto;
                eth_hdr_len += ETHER_HDR_LEN;
        }

        /* Check for IP header again. */
        switch (eth_type) {
        case htons(ETHERTYPE_IP):
                ip4 = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len,
                    sizeof(*ip4));
                if (unlikely(ip4 == NULL))
                        return (0);
                wqe->eth.swp_inner_l3_offset = eth_hdr_len / 2;
                wqe->eth.cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM;
                ip_type = ip4->ip_p;
                ip_hlen = ip4->ip_hl << 2;
                eth_hdr_len += ip_hlen;
                break;
        case htons(ETHERTYPE_IPV6):
                ip6 = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len,
                    sizeof(*ip6));
                if (unlikely(ip6 == NULL))
                        return (0);
                wqe->eth.swp_inner_l3_offset = eth_hdr_len / 2;
                wqe->eth.swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_TYPE;
                ip_type = ip6->ip6_nxt;
                eth_hdr_len += sizeof(*ip6);
                break;
        default:
                return (0);
        }

        /*
         * If the hardware is not computing inner UDP/TCP checksum,
         * then skip inlining the inner UDP/TCP header:
         */
        if (unlikely((cs_mask & MLX5_ETH_WQE_L4_INNER_CSUM) == 0))
                goto done;

        switch (ip_type) {
        case IPPROTO_UDP:
                udp = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len,
                    sizeof(*udp));
                if (unlikely(udp == NULL))
                        return (0);
                wqe->eth.swp_inner_l4_offset = (eth_hdr_len / 2);
                wqe->eth.cs_flags |= MLX5_ETH_WQE_L4_INNER_CSUM;
                wqe->eth.swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_TYPE;
                eth_hdr_len += sizeof(*udp);
                break;
        case IPPROTO_TCP:
                th = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len,
                    sizeof(*th));
                if (unlikely(th == NULL))
                        return (0);
                wqe->eth.swp_inner_l4_offset = eth_hdr_len / 2;
                wqe->eth.cs_flags |= MLX5_ETH_WQE_L4_INNER_CSUM;
                tcp_hlen = th->th_off << 2;
                eth_hdr_len += tcp_hlen;
                break;
        default:
                return (0);
        }
done:
        if (unlikely(pkt_hdr_len < eth_hdr_len))
                return (0);

        /* Account for software inserted VLAN tag, if any. */
        if (unlikely(has_outer_vlan_tag)) {
                wqe->eth.swp_outer_l3_offset += ETHER_VLAN_ENCAP_LEN / 2;
                wqe->eth.swp_outer_l4_offset += ETHER_VLAN_ENCAP_LEN / 2;
                wqe->eth.swp_inner_l3_offset += ETHER_VLAN_ENCAP_LEN / 2;
                wqe->eth.swp_inner_l4_offset += ETHER_VLAN_ENCAP_LEN / 2;
        }

        /*
         * When inner checksums are set, outer L4 checksum flag must
         * be disabled.
         */
        if (wqe->eth.cs_flags & (MLX5_ETH_WQE_L3_INNER_CSUM |
            MLX5_ETH_WQE_L4_INNER_CSUM))
                wqe->eth.cs_flags &= ~MLX5_ETH_WQE_L4_CSUM;

        return (eth_hdr_len);
}

struct mlx5_wqe_dump_seg {
        struct mlx5_wqe_ctrl_seg ctrl;
        struct mlx5_wqe_data_seg data;
} __aligned(MLX5_SEND_WQE_BB);

CTASSERT(DIV_ROUND_UP(2, MLX5_SEND_WQEBB_NUM_DS) == 1);

int
mlx5e_sq_dump_xmit(struct mlx5e_sq *sq, struct mlx5e_xmit_args *parg, struct mbuf **mbp)
{
        bus_dma_segment_t segs[MLX5E_MAX_TX_MBUF_FRAGS];
        struct mlx5_wqe_dump_seg *wqe;
        struct mlx5_wqe_dump_seg *wqe_last;
        int nsegs;
        int xsegs;
        u32 off;
        u32 msb;
        int err;
        int x;
        struct mbuf *mb;
        const u32 ds_cnt = 2;
        u16 pi;
        const u8 opcode = MLX5_OPCODE_DUMP;

        /* get pointer to mbuf */
        mb = *mbp;

        /* get producer index */
        pi = sq->pc & sq->wq.sz_m1;

        sq->mbuf[pi].num_bytes = mb->m_pkthdr.len;
        sq->mbuf[pi].num_wqebbs = 0;

        /* check number of segments in mbuf */
        err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map,
            mb, segs, &nsegs, BUS_DMA_NOWAIT);
        if (err == EFBIG) {
                /* update statistics */
                sq->stats.defragged++;
                /* too many mbuf fragments */
                mb = m_defrag(*mbp, M_NOWAIT);
                if (mb == NULL) {
                        mb = *mbp;
                        goto tx_drop;
                }
                /* try again */
                err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map,
                    mb, segs, &nsegs, BUS_DMA_NOWAIT);
        }

        if (err != 0)
                goto tx_drop;

        /* make sure all mbuf data, if any, is visible to the bus */
        bus_dmamap_sync(sq->dma_tag, sq->mbuf[pi].dma_map,
            BUS_DMASYNC_PREWRITE);

        /* compute number of real DUMP segments */
        msb = sq->priv->params_ethtool.hw_mtu_msb;
        for (x = xsegs = 0; x != nsegs; x++)
                xsegs += howmany((u32)segs[x].ds_len, msb);

        /* check if there are no segments */
        if (unlikely(xsegs == 0)) {
                bus_dmamap_unload(sq->dma_tag, sq->mbuf[pi].dma_map);
                m_freem(mb);
                *mbp = NULL;    /* safety clear */
                return (0);
        }

        /* return ENOBUFS if the queue is full */
        if (unlikely(!mlx5e_sq_has_room_for(sq, xsegs))) {
                sq->stats.enobuf++;
                bus_dmamap_unload(sq->dma_tag, sq->mbuf[pi].dma_map);
                m_freem(mb);
                *mbp = NULL;    /* safety clear */
                return (ENOBUFS);
        }

        wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
        wqe_last = mlx5_wq_cyc_get_wqe(&sq->wq, sq->wq.sz_m1);

        for (x = 0; x != nsegs; x++) {
                for (off = 0; off < segs[x].ds_len; off += msb) {
                        u32 len = segs[x].ds_len - off;

                        /* limit length */
                        if (likely(len > msb))
                                len = msb;

                        memset(&wqe->ctrl, 0, sizeof(wqe->ctrl));

                        /* fill control segment */
                        wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
                        wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
                        wqe->ctrl.imm = cpu_to_be32(parg->tisn << 8);

                        /* fill data segment */
                        wqe->data.addr = cpu_to_be64((uint64_t)segs[x].ds_addr + off);
                        wqe->data.lkey = sq->mkey_be;
                        wqe->data.byte_count = cpu_to_be32(len);

                        /* advance to next building block */
                        if (unlikely(wqe == wqe_last))
                                wqe = mlx5_wq_cyc_get_wqe(&sq->wq, 0);
                        else
                                wqe++;

                        sq->mbuf[pi].num_wqebbs++;
                        sq->pc++;
                }
        }

        wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
        wqe_last = mlx5_wq_cyc_get_wqe(&sq->wq, (sq->pc - 1) & sq->wq.sz_m1);

        /* put in place data fence */
        wqe->ctrl.fm_ce_se |= MLX5_FENCE_MODE_INITIATOR_SMALL;

        /* check if we should generate a completion event */
        if (mlx5e_do_send_cqe_inline(sq))
                wqe_last->ctrl.fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE;

        /* copy data for doorbell */
        memcpy(sq->doorbell.d32, wqe_last, sizeof(sq->doorbell.d32));

        /* store pointer to mbuf */
        sq->mbuf[pi].mbuf = mb;
        sq->mbuf[pi].mst = m_snd_tag_ref(parg->mst);

        /* count all traffic going out */
        sq->stats.packets++;
        sq->stats.bytes += sq->mbuf[pi].num_bytes;

        *mbp = NULL;    /* safety clear */
        return (0);

tx_drop:
        sq->stats.dropped++;
        *mbp = NULL;
        m_freem(mb);
        return err;
}

int
mlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp)
{
        bus_dma_segment_t segs[MLX5E_MAX_TX_MBUF_FRAGS];
        struct mlx5e_xmit_args args = {};
        struct mlx5_wqe_data_seg *dseg;
        struct mlx5e_tx_wqe *wqe;
        if_t ifp;
        int nsegs;
        int err;
        int x;
        struct mbuf *mb;
        u16 ds_cnt;
        u16 pi;
        u8 opcode;

#ifdef KERN_TLS
top:
#endif
        /* Return ENOBUFS if the queue is full */
        if (unlikely(!mlx5e_sq_has_room_for(sq, 2 * MLX5_SEND_WQE_MAX_WQEBBS))) {
                sq->stats.enobuf++;
                return (ENOBUFS);
        }

        /* Align SQ edge with NOPs to avoid WQE wrap around */
        pi = ((~sq->pc) & sq->wq.sz_m1);
        if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) {
                /* Send one multi NOP message instead of many */
                mlx5e_send_nop(sq, (pi + 1) * MLX5_SEND_WQEBB_NUM_DS);
                pi = ((~sq->pc) & sq->wq.sz_m1);
                if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) {
                        sq->stats.enobuf++;
                        return (ENOMEM);
                }
        }

#ifdef KERN_TLS
        /* Special handling for TLS packets, if any */
        switch (mlx5e_sq_tls_xmit(sq, &args, mbp)) {
        case MLX5E_TLS_LOOP:
                goto top;
        case MLX5E_TLS_FAILURE:
                mb = *mbp;
                err = ENOMEM;
                goto tx_drop;
        case MLX5E_TLS_DEFERRED:
                return (0);
        case MLX5E_TLS_CONTINUE:
        default:
                break;
        }
#endif

        /* Setup local variables */
        pi = sq->pc & sq->wq.sz_m1;
        wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
        ifp = sq->ifp;

        memset(wqe, 0, sizeof(*wqe));

        /* get pointer to mbuf */
        mb = *mbp;

        mlx5e_accel_ipsec_handle_tx(mb, wqe);

        /* Send a copy of the frame to the BPF listener, if any */
        if (ifp != NULL)
                ETHER_BPF_MTAP(ifp, mb);

        if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) {
                wqe->eth.cs_flags |= MLX5_ETH_WQE_L3_CSUM;
        }
        if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) {
                wqe->eth.cs_flags |= MLX5_ETH_WQE_L4_CSUM;
        }
        if (wqe->eth.cs_flags == 0) {
                sq->stats.csum_offload_none++;
        }
        if (mb->m_pkthdr.csum_flags & CSUM_TSO) {
                u32 payload_len;
                u32 mss = mb->m_pkthdr.tso_segsz;
                u32 num_pkts;

                wqe->eth.mss = cpu_to_be16(mss);
                opcode = MLX5_OPCODE_LSO;
                if (args.ihs == 0)
                        args.ihs = mlx5e_get_full_header_size(mb, NULL);
                if (unlikely(args.ihs == 0)) {
                        err = EINVAL;
                        goto tx_drop;
                }
                payload_len = mb->m_pkthdr.len - args.ihs;
                if (payload_len == 0)
                        num_pkts = 1;
                else
                        num_pkts = DIV_ROUND_UP(payload_len, mss);
                sq->mbuf[pi].num_bytes = payload_len + (num_pkts * args.ihs);


                sq->stats.tso_packets++;
                sq->stats.tso_bytes += payload_len;
        } else if (mb->m_pkthdr.csum_flags & CSUM_ENCAP_VXLAN) {
                /* check for inner TCP TSO first */
                if (mb->m_pkthdr.csum_flags & (CSUM_INNER_IP_TSO |
                    CSUM_INNER_IP6_TSO)) {
                        u32 payload_len;
                        u32 mss = mb->m_pkthdr.tso_segsz;
                        u32 num_pkts;

                        wqe->eth.mss = cpu_to_be16(mss);
                        opcode = MLX5_OPCODE_LSO;

                        if (likely(args.ihs == 0)) {
                                args.ihs = mlx5e_get_vxlan_header_size(mb, wqe,
                                       MLX5_ETH_WQE_L3_INNER_CSUM |
                                       MLX5_ETH_WQE_L4_INNER_CSUM |
                                       MLX5_ETH_WQE_L4_CSUM |
                                       MLX5_ETH_WQE_L3_CSUM,
                                       opcode);
                                if (unlikely(args.ihs == 0)) {
                                        err = EINVAL;
                                        goto tx_drop;
                                }
                        }

                        payload_len = mb->m_pkthdr.len - args.ihs;
                        if (payload_len == 0)
                                num_pkts = 1;
                        else
                                num_pkts = DIV_ROUND_UP(payload_len, mss);
                        sq->mbuf[pi].num_bytes = payload_len +
                            num_pkts * args.ihs;

                        sq->stats.tso_packets++;
                        sq->stats.tso_bytes += payload_len;
                } else {
                        opcode = MLX5_OPCODE_SEND;

                        if (likely(args.ihs == 0)) {
                                uint8_t cs_mask;

                                if (mb->m_pkthdr.csum_flags &
                                    (CSUM_INNER_IP_TCP | CSUM_INNER_IP_UDP |
                                     CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_UDP)) {
                                        cs_mask =
                                            MLX5_ETH_WQE_L3_INNER_CSUM |
                                            MLX5_ETH_WQE_L4_INNER_CSUM |
                                            MLX5_ETH_WQE_L4_CSUM |
                                            MLX5_ETH_WQE_L3_CSUM;
                                } else if (mb->m_pkthdr.csum_flags & CSUM_INNER_IP) {
                                        cs_mask =
                                            MLX5_ETH_WQE_L3_INNER_CSUM |
                                            MLX5_ETH_WQE_L4_CSUM |
                                            MLX5_ETH_WQE_L3_CSUM;
                                } else {
                                        cs_mask =
                                            MLX5_ETH_WQE_L4_CSUM |
                                            MLX5_ETH_WQE_L3_CSUM;
                                }
                                args.ihs = mlx5e_get_vxlan_header_size(mb, wqe,
                                    cs_mask, opcode);
                                if (unlikely(args.ihs == 0)) {
                                        err = EINVAL;
                                        goto tx_drop;
                                }
                        }

                        sq->mbuf[pi].num_bytes = max_t (unsigned int,
                            mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN);
                }
        } else {
                opcode = MLX5_OPCODE_SEND;

                if (args.ihs == 0) {
                        switch (sq->min_inline_mode) {
                        case MLX5_INLINE_MODE_IP:
                        case MLX5_INLINE_MODE_TCP_UDP:
                                args.ihs = mlx5e_get_full_header_size(mb, NULL);
                                if (unlikely(args.ihs == 0))
                                        args.ihs = mlx5e_get_l2_header_size(sq, mb);
                                break;
                        case MLX5_INLINE_MODE_L2:
                                args.ihs = mlx5e_get_l2_header_size(sq, mb);
                                break;
                        case MLX5_INLINE_MODE_NONE:
                                /* FALLTHROUGH */
                        default:
                                if ((mb->m_flags & M_VLANTAG) != 0 &&
                                    (sq->min_insert_caps & MLX5E_INSERT_VLAN) != 0) {
                                        /* inlining VLAN data is not required */
                                        wqe->eth.vlan_cmd = htons(0x8000); /* bit 0 CVLAN */
                                        wqe->eth.vlan_hdr = htons(mb->m_pkthdr.ether_vtag);
                                        args.ihs = 0;
                                } else if ((mb->m_flags & M_VLANTAG) == 0 &&
                                    (sq->min_insert_caps & MLX5E_INSERT_NON_VLAN) != 0) {
                                        /* inlining non-VLAN data is not required */
                                        args.ihs = 0;
                                } else {
                                        /* we are forced to inlining L2 header, if any */
                                        args.ihs = mlx5e_get_l2_header_size(sq, mb);
                                }
                                break;
                        }
                }
                sq->mbuf[pi].num_bytes = max_t (unsigned int,
                    mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN);
        }

        if (likely(args.ihs == 0)) {
                /* nothing to inline */
        } else if ((mb->m_flags & M_VLANTAG) != 0) {
                struct ether_vlan_header *eh = (struct ether_vlan_header *)
                    wqe->eth.inline_hdr_start;

                /* Range checks */
                if (unlikely(args.ihs > (sq->max_inline - ETHER_VLAN_ENCAP_LEN))) {
                        if (mb->m_pkthdr.csum_flags & (CSUM_TSO | CSUM_ENCAP_VXLAN)) {
                                err = EINVAL;
                                goto tx_drop;
                        }
                        args.ihs = (sq->max_inline - ETHER_VLAN_ENCAP_LEN);
                } else if (unlikely(args.ihs < ETHER_HDR_LEN)) {
                        err = EINVAL;
                        goto tx_drop;
                }
                m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh);
                m_adj(mb, ETHER_HDR_LEN);
                /* Insert 4 bytes VLAN tag into data stream */
                eh->evl_proto = eh->evl_encap_proto;
                eh->evl_encap_proto = htons(ETHERTYPE_VLAN);
                eh->evl_tag = htons(mb->m_pkthdr.ether_vtag);
                /* Copy rest of header data, if any */
                m_copydata(mb, 0, args.ihs - ETHER_HDR_LEN, (caddr_t)(eh + 1));
                m_adj(mb, args.ihs - ETHER_HDR_LEN);
                /* Extend header by 4 bytes */
                args.ihs += ETHER_VLAN_ENCAP_LEN;
                wqe->eth.inline_hdr_sz = cpu_to_be16(args.ihs);
        } else {
                /* check if inline header size is too big */
                if (unlikely(args.ihs > sq->max_inline)) {
                        if (unlikely(mb->m_pkthdr.csum_flags & (CSUM_TSO |
                            CSUM_ENCAP_VXLAN))) {
                                err = EINVAL;
                                goto tx_drop;
                        }
                        args.ihs = sq->max_inline;
                }
                m_copydata(mb, 0, args.ihs, wqe->eth.inline_hdr_start);
                m_adj(mb, args.ihs);
                wqe->eth.inline_hdr_sz = cpu_to_be16(args.ihs);
        }

        ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
        if (args.ihs > sizeof(wqe->eth.inline_hdr_start)) {
                ds_cnt += DIV_ROUND_UP(args.ihs - sizeof(wqe->eth.inline_hdr_start),
                    MLX5_SEND_WQE_DS);
        }
        dseg = ((struct mlx5_wqe_data_seg *)&wqe->ctrl) + ds_cnt;

        err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map,
            mb, segs, &nsegs, BUS_DMA_NOWAIT);
        if (err == EFBIG) {
                /* Update statistics */
                sq->stats.defragged++;
                /* Too many mbuf fragments */
                mb = m_defrag(*mbp, M_NOWAIT);
                if (mb == NULL) {
                        mb = *mbp;
                        goto tx_drop;
                }
                /* Try again */
                err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map,
                    mb, segs, &nsegs, BUS_DMA_NOWAIT);
        }
        /* Catch errors */
        if (err != 0)
                goto tx_drop;

        /* Make sure all mbuf data, if any, is visible to the bus */
        if (nsegs != 0) {
                bus_dmamap_sync(sq->dma_tag, sq->mbuf[pi].dma_map,
                    BUS_DMASYNC_PREWRITE);
        } else {
                /* All data was inlined, free the mbuf. */
                bus_dmamap_unload(sq->dma_tag, sq->mbuf[pi].dma_map);
                m_freem(mb);
                mb = NULL;
        }

        for (x = 0; x != nsegs; x++) {
                if (segs[x].ds_len == 0)
                        continue;
                dseg->addr = cpu_to_be64((uint64_t)segs[x].ds_addr);
                dseg->lkey = sq->mkey_be;
                dseg->byte_count = cpu_to_be32((uint32_t)segs[x].ds_len);
                dseg++;
        }

        ds_cnt = (dseg - ((struct mlx5_wqe_data_seg *)&wqe->ctrl));

        wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
        wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
        wqe->ctrl.imm = cpu_to_be32(args.tisn << 8);

        if (mlx5e_do_send_cqe_inline(sq))
                wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
        else
                wqe->ctrl.fm_ce_se = 0;

        /* Copy data for doorbell */
        memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));

        /* Store pointer to mbuf */
        sq->mbuf[pi].mbuf = mb;
        sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
        if (unlikely(args.mst != NULL))
                sq->mbuf[pi].mst = m_snd_tag_ref(args.mst);
        else
                MPASS(sq->mbuf[pi].mst == NULL);

        sq->pc += sq->mbuf[pi].num_wqebbs;

        /* Count all traffic going out */
        sq->stats.packets++;
        sq->stats.bytes += sq->mbuf[pi].num_bytes;

        *mbp = NULL;    /* safety clear */
        return (0);

tx_drop:
        sq->stats.dropped++;
        *mbp = NULL;
        m_freem(mb);
        return err;
}

static void
mlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget)
{
        u16 sqcc;

        /*
         * sq->cc must be updated only after mlx5_cqwq_update_db_record(),
         * otherwise a cq overrun may occur
         */
        sqcc = sq->cc;

        while (budget > 0) {
                struct mlx5_cqe64 *cqe;
                struct m_snd_tag *mst;
                struct mbuf *mb;
                bool match;
                u16 sqcc_this;
                u16 delta;
                u16 x;
                u16 ci;

                cqe = mlx5e_get_cqe(&sq->cq);
                if (!cqe)
                        break;

                mlx5_cqwq_pop(&sq->cq.wq);

                /* check if the completion event indicates an error */
                if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
                        mlx5e_dump_err_cqe(&sq->cq, sq->sqn, (const void *)cqe);
                        sq->stats.cqe_err++;
                }

                /* setup local variables */
                sqcc_this = be16toh(cqe->wqe_counter);
                match = false;

                /* update budget according to the event factor */
                budget -= sq->cev_factor;

                for (x = 0;; x++) {
                        if (unlikely(match != false)) {
                                break;
                        } else if (unlikely(x == sq->cev_factor)) {
                                /* WQE counter match not found */
                                sq->stats.cqe_err++;
                                break;
                        }
                        ci = sqcc & sq->wq.sz_m1;
                        delta = sqcc_this - sqcc;
                        match = (delta < sq->mbuf[ci].num_wqebbs);
                        mb = sq->mbuf[ci].mbuf;
                        sq->mbuf[ci].mbuf = NULL;
                        mst = sq->mbuf[ci].mst;
                        sq->mbuf[ci].mst = NULL;

                        if (unlikely(mb == NULL)) {
                                if (unlikely(sq->mbuf[ci].num_bytes == 0))
                                        sq->stats.nop++;
                        } else {
                                bus_dmamap_sync(sq->dma_tag, sq->mbuf[ci].dma_map,
                                    BUS_DMASYNC_POSTWRITE);
                                bus_dmamap_unload(sq->dma_tag, sq->mbuf[ci].dma_map);

                                /* Free transmitted mbuf */
                                m_freem(mb);
                        }

                        if (unlikely(mst != NULL))
                                m_snd_tag_rele(mst);

                        sqcc += sq->mbuf[ci].num_wqebbs;
                }
        }

        mlx5_cqwq_update_db_record(&sq->cq.wq);

        /* Ensure cq space is freed before enabling more cqes */
        atomic_thread_fence_rel();

        sq->cc = sqcc;
}

static int
mlx5e_xmit_locked(if_t ifp, struct mlx5e_sq *sq, struct mbuf *mb)
{
        int err = 0;

        if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 ||
            READ_ONCE(sq->running) == 0)) {
                m_freem(mb);
                return (ENETDOWN);
        }

        /* Do transmit */
        if (mlx5e_sq_xmit(sq, &mb) != 0) {
                /* NOTE: m_freem() is NULL safe */
                m_freem(mb);
                err = ENOBUFS;
        }

        /* Write the doorbell record, if any. */
        mlx5e_tx_notify_hw(sq, false);

        /*
         * Check if we need to start the event timer which flushes the
         * transmit ring on timeout:
         */
        if (unlikely(sq->cev_next_state == MLX5E_CEV_STATE_INITIAL &&
            sq->cev_factor != 1)) {
                /* start the timer */
                mlx5e_sq_cev_timeout(sq);
        } else {
                /* don't send NOPs yet */
                sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS;
        }
        return (err);
}

int
mlx5e_xmit(if_t ifp, struct mbuf *mb)
{
        struct mlx5e_sq *sq;
        int ret;

        if (mb->m_pkthdr.csum_flags & CSUM_SND_TAG) {
                MPASS(mb->m_pkthdr.snd_tag->ifp == ifp);
                sq = mlx5e_select_queue_by_send_tag(ifp, mb);
                if (unlikely(sq == NULL)) {
                        goto select_queue;
                }
        } else {
select_queue:
                sq = mlx5e_select_queue(ifp, mb);
                if (unlikely(sq == NULL)) {
                        /* Free mbuf */
                        m_freem(mb);

                        /* Invalid send queue */
                        return (ENXIO);
                }
        }

        mtx_lock(&sq->lock);
        ret = mlx5e_xmit_locked(ifp, sq, mb);
        mtx_unlock(&sq->lock);

        return (ret);
}

void
mlx5e_tx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused)
{
        struct mlx5e_sq *sq = container_of(mcq, struct mlx5e_sq, cq.mcq);

        mtx_lock(&sq->comp_lock);
        mlx5e_poll_tx_cq(sq, MLX5E_BUDGET_MAX);
        mlx5e_cq_arm(&sq->cq, MLX5_GET_DOORBELL_LOCK(&sq->priv->doorbell_lock));
        mtx_unlock(&sq->comp_lock);
}