/*
 * aQuantia Corporation Network Driver
 * Copyright (C) 2014-2018 aQuantia Corporation. All rights reserved
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *   (1) Redistributions of source code must retain the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer.
 *
 *   (2) Redistributions in binary form must reproduce the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer in the documentation and/or other materials provided
 *   with the distribution.
 *
 *   (3)The name of the author may not be used to endorse or promote
 *   products derived from this software without specific prior
 *   written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

#include <sys/param.h>
#include <sys/bitstring.h>
#include <sys/kernel.h>
#include <sys/socket.h>
#include <machine/param.h>
#include <net/ethernet.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/iflib.h>
#include <netinet/in.h>

#include "aq_common.h"

#include "aq_ring.h"
#include "aq_dbg.h"
#include "aq_device.h"
#include "aq_hw.h"
#include "aq_hw_llh.h"

/* iflib txrx interface prototypes */
static int aq_isc_txd_encap(void *arg, if_pkt_info_t pi);
static void aq_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
static int aq_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear);
static void aq_ring_rx_refill(void* arg, if_rxd_update_t iru);
static void aq_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx);
static int aq_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget);
static int aq_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);

struct if_txrx aq_txrx = {
        .ift_txd_encap = aq_isc_txd_encap,
        .ift_txd_flush = aq_isc_txd_flush,
        .ift_txd_credits_update = aq_isc_txd_credits_update,
        .ift_rxd_available = aq_isc_rxd_available,
        .ift_rxd_pkt_get = aq_isc_rxd_pkt_get,
        .ift_rxd_refill = aq_ring_rx_refill,
        .ift_rxd_flush = aq_isc_rxd_flush,
        .ift_legacy_intr = NULL
};


static inline uint32_t
aq_next(uint32_t i, uint32_t lim)
{
        return (i == lim) ? 0 : i + 1;
}

int
aq_ring_rx_init(struct aq_hw *hw, struct aq_ring *ring)
{
        int err;
        uint32_t dma_desc_addr_lsw = (uint32_t)ring->rx_descs_phys & 0xffffffff;
        uint32_t dma_desc_addr_msw = (uint32_t)(ring->rx_descs_phys >> 32);

        AQ_DBG_ENTERA("[%d]", ring->index);

        rdm_rx_desc_en_set(hw, false, ring->index);

        rdm_rx_desc_head_splitting_set(hw, 0U, ring->index);

        reg_rx_dma_desc_base_addresslswset(hw, dma_desc_addr_lsw, ring->index);

        reg_rx_dma_desc_base_addressmswset(hw, dma_desc_addr_msw, ring->index);

        rdm_rx_desc_len_set(hw, ring->rx_size / 8U, ring->index);

        rdm_rx_desc_data_buff_size_set(hw, ring->rx_buf_size / 1024U,
            ring->index);

        rdm_rx_desc_head_buff_size_set(hw, 0U, ring->index);
        rdm_rx_desc_head_splitting_set(hw, 0U, ring->index);
        rpo_rx_desc_vlan_stripping_set(hw,
            (if_getcapenable(iflib_get_ifp(ring->dev->ctx)) &
            IFCAP_VLAN_HWTAGGING) != 0 ? 1U : 0U, ring->index);

        /* Rx ring set mode */

        /* Mapping interrupt vector */
        itr_irq_map_rx_set(hw, ring->msix, ring->index);
        itr_irq_map_en_rx_set(hw, true, ring->index);

        rdm_cpu_id_set(hw, 0, ring->index);
        rdm_rx_desc_dca_en_set(hw, 0U, ring->index);
        rdm_rx_head_dca_en_set(hw, 0U, ring->index);
        rdm_rx_pld_dca_en_set(hw, 0U, ring->index);

        err = aq_hw_err_from_flags(hw);
        AQ_DBG_EXIT(err);
        return (err);
}

int
aq_ring_tx_init(struct aq_hw *hw, struct aq_ring *ring)
{
        int err;
        uint32_t dma_desc_addr_lsw = (uint32_t)ring->tx_descs_phys & 0xffffffff;
        uint32_t dma_desc_addr_msw = (uint32_t)(ring->tx_descs_phys >> 32);

        AQ_DBG_ENTERA("[%d]", ring->index);

        tdm_tx_desc_en_set(hw, 0U, ring->index);

        reg_tx_dma_desc_base_addresslswset(hw, dma_desc_addr_lsw, ring->index);

        reg_tx_dma_desc_base_addressmswset(hw, dma_desc_addr_msw, ring->index);

        tdm_tx_desc_len_set(hw, ring->tx_size / 8U, ring->index);

        aq_ring_tx_tail_update(hw, ring, 0U);

        /* Set Tx threshold */
        tdm_tx_desc_wr_wb_threshold_set(hw, 0U, ring->index);

        /* Mapping interrupt vector */
        itr_irq_map_tx_set(hw, ring->msix, ring->index);
        itr_irq_map_en_tx_set(hw, true, ring->index);

        tdm_cpu_id_set(hw, 0, ring->index);
        tdm_tx_desc_dca_en_set(hw, 0U, ring->index);

        err = aq_hw_err_from_flags(hw);
        AQ_DBG_EXIT(err);
        return (err);
}

int
aq_ring_tx_tail_update(struct aq_hw *hw, struct aq_ring *ring, uint32_t tail)
{
        AQ_DBG_ENTERA("[%d]", ring->index);
        reg_tx_dma_desc_tail_ptr_set(hw, tail, ring->index);
        AQ_DBG_EXIT(0);
        return (0);
}

int
aq_ring_tx_start(struct aq_hw *hw, struct aq_ring *ring)
{
        int err;

        AQ_DBG_ENTERA("[%d]", ring->index);
        tdm_tx_desc_en_set(hw, 1U, ring->index);
        err = aq_hw_err_from_flags(hw);
        AQ_DBG_EXIT(err);
        return (err);
}

int
aq_ring_rx_start(struct aq_hw *hw, struct aq_ring *ring)
{
        int err;

        AQ_DBG_ENTERA("[%d]", ring->index);
        rdm_rx_desc_en_set(hw, 1U, ring->index);
        err = aq_hw_err_from_flags(hw);
        AQ_DBG_EXIT(err);
        return (err);
}

int
aq_ring_tx_stop(struct aq_hw *hw, struct aq_ring *ring)
{
        int err;

        AQ_DBG_ENTERA("[%d]", ring->index);
        tdm_tx_desc_en_set(hw, 0U, ring->index);
        err = aq_hw_err_from_flags(hw);
        AQ_DBG_EXIT(err);
        return (err);
}

int
aq_ring_rx_stop(struct aq_hw *hw, struct aq_ring *ring)
{
        int err;

        AQ_DBG_ENTERA("[%d]", ring->index);
        rdm_rx_desc_en_set(hw, 0U, ring->index);
        /* Invalidate Descriptor Cache to prevent writing to the cached
         * descriptors and to the data pointer of those descriptors
         */
        rdm_rx_dma_desc_cache_init_tgl(hw);
        err = aq_hw_err_from_flags(hw);
        AQ_DBG_EXIT(err);
        return (err);
}

static void
aq_ring_rx_refill(void* arg, if_rxd_update_t iru)
{
        struct aq_dev *aq_dev = arg;
        volatile struct aq_rx_desc *rx_desc;
        struct aq_ring *ring;
        qidx_t i, pidx;

        AQ_DBG_ENTERA("ring=%d iru_pidx=%d iru_count=%d iru->iru_buf_size=%d",
            iru->iru_qsidx, iru->iru_pidx, iru->iru_count, iru->iru_buf_size);

        ring = aq_dev->rx_rings[iru->iru_qsidx];
        pidx = iru->iru_pidx;

        for (i = 0; i < iru->iru_count; i++) {
                rx_desc = (volatile struct aq_rx_desc *) &ring->rx_descs[pidx];
                rx_desc->read.buf_addr = htole64(iru->iru_paddrs[i]);
                rx_desc->read.hdr_addr = 0;

                pidx=aq_next(pidx, ring->rx_size - 1);
        }

        AQ_DBG_EXIT(0);
}

static void
aq_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx)
{
        struct aq_dev *aq_dev = arg;
        struct aq_ring *ring = aq_dev->rx_rings[rxqid];

        AQ_DBG_ENTERA("[%d] tail=%u", ring->index, pidx);
        reg_rx_dma_desc_tail_ptr_set(&aq_dev->hw, pidx, ring->index);
        AQ_DBG_EXIT(0);
}

static int
aq_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
{
        struct aq_dev *aq_dev = arg;
        struct aq_ring *ring = aq_dev->rx_rings[rxqid];
        volatile struct aq_rx_desc *rx_desc = (volatile struct aq_rx_desc *) ring->rx_descs;
        int cnt, i, iter;

        AQ_DBG_ENTERA("[%d] head=%u, budget %d", ring->index, idx, budget);

        for (iter = 0, cnt = 0, i = idx;
            iter < ring->rx_size && cnt <= budget;) {
                trace_aq_rx_descr(ring->index, i,
                    (volatile uint64_t*)&rx_desc[i]);
                if (!rx_desc[i].wb.dd)
                        break;

                if (rx_desc[i].wb.eop)
                        cnt++;
                iter++;
                i = aq_next(i, ring->rx_size - 1);
        }

        AQ_DBG_EXIT(cnt);
        return (cnt);
}

static void
aq_rx_set_cso_flags(volatile struct aq_rx_desc *rx_desc,  if_rxd_info_t ri)
{
        if ((rx_desc->wb.pkt_type & 0x3) == 0) { // IPv4
                if (rx_desc->wb.rx_cntl & BIT(0)) { // IPv4 csum checked
                        ri->iri_csum_flags |= CSUM_IP_CHECKED;
                        if (!(rx_desc->wb.rx_stat & BIT(1)))
                                ri->iri_csum_flags |= CSUM_IP_VALID;
                }
        }
        if (rx_desc->wb.rx_cntl & BIT(1)) { // TCP/UDP csum checked
                ri->iri_csum_flags |= CSUM_L4_CALC;
                if (!(rx_desc->wb.rx_stat & BIT(2)) && // L4 csum error
                        (rx_desc->wb.rx_stat & BIT(3))) {  // L4 csum valid
                        ri->iri_csum_flags |= CSUM_L4_VALID;
                        ri->iri_csum_data = htons(0xffff);
                }
        }
}

static uint8_t bsd_rss_type[16] = {
        [AQ_RX_RSS_TYPE_IPV4] = M_HASHTYPE_RSS_IPV4,
        [AQ_RX_RSS_TYPE_IPV6] = M_HASHTYPE_RSS_IPV6,
        [AQ_RX_RSS_TYPE_IPV4_TCP] = M_HASHTYPE_RSS_TCP_IPV4,
        [AQ_RX_RSS_TYPE_IPV6_TCP] = M_HASHTYPE_RSS_TCP_IPV6,
        [AQ_RX_RSS_TYPE_IPV4_UDP] = M_HASHTYPE_RSS_UDP_IPV4,
        [AQ_RX_RSS_TYPE_IPV6_UDP] = M_HASHTYPE_RSS_UDP_IPV6,
};



static int
aq_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
{
        struct aq_dev *aq_dev = arg;
        struct aq_ring *ring = aq_dev->rx_rings[ri->iri_qsidx];
        volatile struct aq_rx_desc *rx_desc;
        if_t ifp;
        int cidx, rc = 0, i;
        size_t len, total_len;
        bool is_error = false;

        AQ_DBG_ENTERA("[%d] start=%d", ring->index, ri->iri_cidx);
        cidx = ri->iri_cidx;
        ifp = iflib_get_ifp(aq_dev->ctx);
        i = 0;
        total_len = 0;

        do {
                if (i >= aq_dev->sctx->isc_rx_nsegments)
                        goto rx_err;

                rx_desc = (volatile struct aq_rx_desc *) &ring->rx_descs[cidx];

                trace_aq_rx_descr(ring->index, cidx,
                    (volatile uint64_t *)rx_desc);

                /* MAC error (rx_stat) or RX-DMA fault (rdm_err) -> drop. */
                if ((rx_desc->wb.rx_stat & BIT(0)) != 0 || rx_desc->wb.rdm_err)
                        is_error = true;

                if (!rx_desc->wb.eop) {
                        len = ring->rx_buf_size;
                } else if (is_error) {
                        total_len = le32toh(rx_desc->wb.pkt_len);
                        len = 0;
                } else {
                        total_len = le32toh(rx_desc->wb.pkt_len);
                        if (total_len < (size_t)i * ring->rx_buf_size)
                                goto rx_err;
                        len = total_len - (size_t)i * ring->rx_buf_size;
                        if (len > (size_t)ring->rx_buf_size)
                                goto rx_err;
                }
                ri->iri_frags[i].irf_flid = 0;
                ri->iri_frags[i].irf_idx = cidx;
                ri->iri_frags[i].irf_len = len;

                if ((if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) != 0 &&
                    (rx_desc->wb.pkt_type & 0x60) != 0) {
                        ri->iri_flags |= M_VLANTAG;
                        ri->iri_vtag = le32toh(rx_desc->wb.vlan);
                }

                i++;
                cidx = aq_next(cidx, ring->rx_size - 1);
        } while (!rx_desc->wb.eop);

        ri->iri_nfrags = i;
        ri->iri_len = total_len;

        /* Per-frame RX error: drop (zero-length frags), don't reset. */
        if (is_error) {
                counter_u64_add(ring->stats.rx_err, 1);
                for (i = 0; i < ri->iri_nfrags; i++)
                        ri->iri_frags[i].irf_len = 0;
                if (ri->iri_len == 0)
                        ri->iri_len = 1;        /* iflib asserts iri_len != 0 */
                goto exit;
        }

        if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) {
                aq_rx_set_cso_flags(rx_desc, ri);
        }
        ri->iri_rsstype = bsd_rss_type[rx_desc->wb.rss_type & 0xF];
        if (ri->iri_rsstype != M_HASHTYPE_NONE) {
                ri->iri_flowid = le32toh(rx_desc->wb.rss_hash);
        }

        counter_u64_add(ring->stats.rx_bytes, total_len);
        counter_u64_add(ring->stats.rx_pkts, 1);
        goto exit;

rx_err:
        counter_u64_add(ring->stats.rx_err, 1);
        rc = EBADMSG;
exit:
        AQ_DBG_EXIT(rc);
        return (rc);
}

/*****************************************************************************/
/*                                                                           */
/*****************************************************************************/

static void
aq_setup_offloads(struct aq_dev *aq_dev, if_pkt_info_t pi, volatile struct aq_tx_desc *txd,
    uint32_t tx_cmd)
{
        AQ_DBG_ENTER();
        txd->cmd |= tx_desc_cmd_fcs;
        txd->cmd |= ((pi->ipi_flags & IPI_TX_IPV4) &&
            (pi->ipi_csum_flags & (CSUM_IP | CSUM_IP_TSO))) ?
            tx_desc_cmd_ipv4 : 0;
        txd->cmd |= (pi->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP |
            CSUM_IP_UDP | CSUM_IP6_UDP)) ?  tx_desc_cmd_l4cs : 0;
        txd->cmd |= (pi->ipi_flags & IPI_TX_INTR) ? tx_desc_cmd_wb : 0;
        txd->cmd |= tx_cmd;
        AQ_DBG_EXIT(0);
}

static int
aq_ring_tso_setup(struct aq_dev *aq_dev, if_pkt_info_t pi, uint32_t *hdrlen,
    volatile union aq_txc_desc *txc)
{
        uint32_t tx_cmd = 0;

        AQ_DBG_ENTER();
        if (pi->ipi_csum_flags & CSUM_TSO) {
                AQ_DBG_PRINT("aq_tso_setup(): TSO enabled");
                tx_cmd |= tx_desc_cmd_lso | tx_desc_cmd_l4cs;

                if (pi->ipi_ipproto != IPPROTO_TCP) {
                        AQ_DBG_PRINT("aq_tso_setup not a tcp");
                        AQ_DBG_EXIT(0);
                        return (0);
                }

                txc->cmd = 0x4; /* TCP */

                if (pi->ipi_csum_flags & CSUM_IP6_TCP)
                    txc->cmd |= 0x2;

                txc->l2_len = pi->ipi_ehdrlen;
                txc->l3_len = pi->ipi_ip_hlen;
                txc->l4_len = pi->ipi_tcp_hlen;
                txc->mss_len = pi->ipi_tso_segsz;
                *hdrlen = txc->l2_len + txc->l3_len + txc->l4_len;
        }

        // Set VLAN tag
        if (pi->ipi_mflags & M_VLANTAG) {
                tx_cmd |= tx_desc_cmd_vlan;
                txc->vlan_tag = htole16(pi->ipi_vtag);
        }

        if (tx_cmd) {
                txc->type = tx_desc_type_ctx;
                txc->idx = 0;
        }

        AQ_DBG_EXIT(tx_cmd);
        return (tx_cmd);
}

static int
aq_isc_txd_encap(void *arg, if_pkt_info_t pi)
{
        struct aq_dev *aq_dev = arg;
        struct aq_ring *ring;
        volatile union aq_txc_desc *txc;
        volatile struct aq_tx_desc *txd = NULL;
        bus_dma_segment_t *segs;
        qidx_t pidx;
        uint32_t hdrlen=0, pay_len;
        uint8_t tx_cmd = 0;
        int i, desc_count = 0;

        AQ_DBG_ENTERA("[%d] start=%d", pi->ipi_qsidx, pi->ipi_pidx);
        ring = aq_dev->tx_rings[pi->ipi_qsidx];

        segs = pi->ipi_segs;
        pidx = pi->ipi_pidx;
        txc = (volatile union aq_txc_desc *)&ring->tx_descs[pidx];
        AQ_DBG_PRINT("txc at 0x%p, txd at 0x%p len %d", txc, txd, pi->ipi_len);

        pay_len = pi->ipi_len;

        txc->flags1 = 0U;
        txc->flags2 = 0U;

        tx_cmd = aq_ring_tso_setup(aq_dev, pi, &hdrlen, txc);
        AQ_DBG_PRINT("tx_cmd = 0x%x", tx_cmd);

        if (tx_cmd) {
                trace_aq_tx_context_descr(ring->index, pidx,
                    (volatile void*)txc);
                /* We've consumed the first desc, adjust counters */
                pidx = aq_next(pidx, ring->tx_size - 1);

                txd = &ring->tx_descs[pidx];
                txd->flags = 0U;
        } else {
                txd = (volatile struct aq_tx_desc *)txc;
        }
        AQ_DBG_PRINT("txc at 0x%p, txd at 0x%p", txc, txd);

        txd->ct_en = !!tx_cmd;

        txd->type = tx_desc_type_desc;

        aq_setup_offloads(aq_dev, pi, txd, tx_cmd);

        if (tx_cmd) {
                txd->ct_idx = 0;
        }

        pay_len -= hdrlen;

        txd->pay_len = pay_len;

        AQ_DBG_PRINT("num_frag[%d] pay_len[%d]", pi->ipi_nsegs, pay_len);
        for (i = 0; i < pi->ipi_nsegs; i++) {
                if (desc_count > 0) {
                        txd = &ring->tx_descs[pidx];
                        txd->flags = 0U;
                }

                txd->buf_addr = htole64(segs[i].ds_addr);

                txd->type = tx_desc_type_desc;
                txd->len = segs[i].ds_len;
                txd->pay_len = pay_len;
                if (i < pi->ipi_nsegs - 1)
                        trace_aq_tx_descr(ring->index, pidx,
                            (volatile void*)txd);

                pidx = aq_next(pidx, ring->tx_size - 1);

                desc_count++;
        }
        // Last descriptor requires EOP and WB
        txd->eop = 1U;

        AQ_DBG_DUMP_DESC(txd);
        trace_aq_tx_descr(ring->index, pidx, (volatile void*)txd);
        ring->tx_tail = pidx;

        counter_u64_add(ring->stats.tx_pkts, 1);
        counter_u64_add(ring->stats.tx_bytes, pay_len);

        pi->ipi_new_pidx = pidx;

        AQ_DBG_EXIT(0);
        return (0);
}

static void
aq_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
{
        struct aq_dev *aq_dev = arg;
        struct aq_ring *ring = aq_dev->tx_rings[txqid];
        AQ_DBG_ENTERA("[%d] tail=%d", ring->index, pidx);

        // Update the write pointer - submits packet for transmission
        aq_ring_tx_tail_update(&aq_dev->hw, ring, pidx);
        AQ_DBG_EXIT(0);
}


static inline unsigned int
aq_avail_desc(int a, int b, int size)
{
        return (((b >= a)) ? ((size) - b + a) : (a - b));
}

static int
aq_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear)
{
        struct aq_dev *aq_dev = arg;
        struct aq_ring *ring = aq_dev->tx_rings[txqid];
        uint32_t head;
        int avail;

        AQ_DBG_ENTERA("[%d] clear=%d", ring->index, clear);
        avail = 0;
        head = tdm_tx_desc_head_ptr_get(&aq_dev->hw, ring->index);
        AQ_DBG_PRINT("swhead %d hwhead %d", ring->tx_head, head);

        if (head >= ring->tx_size) {
                /* Malformed HW head; report no completions. */
                goto done;
        }

        if (ring->tx_head == head) {
                avail = 0;
                goto done;
        }

        avail = aq_avail_desc(head, ring->tx_head, ring->tx_size);
        if (clear)
                ring->tx_head = head;

done:
        AQ_DBG_EXIT(avail);
        return (avail);
}