// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
/* Copyright (C) 2015-2019 Netronome Systems, Inc. */

#include "nfp_app.h"
#include "nfp_net_dp.h"
#include "nfp_net_xsk.h"

/**
 * nfp_net_rx_alloc_one() - Allocate and map page frag for RX
 * @dp:         NFP Net data path struct
 * @dma_addr:   Pointer to storage for DMA address (output param)
 *
 * This function will allcate a new page frag, map it for DMA.
 *
 * Return: allocated page frag or NULL on failure.
 */
void *nfp_net_rx_alloc_one(struct nfp_net_dp *dp, dma_addr_t *dma_addr)
{
        void *frag;

        if (!dp->xdp_prog) {
                frag = netdev_alloc_frag(dp->fl_bufsz);
        } else {
                struct page *page;

                page = alloc_page(GFP_KERNEL);
                frag = page ? page_address(page) : NULL;
        }
        if (!frag) {
                nn_dp_warn(dp, "Failed to alloc receive page frag\n");
                return NULL;
        }

        *dma_addr = nfp_net_dma_map_rx(dp, frag);
        if (dma_mapping_error(dp->dev, *dma_addr)) {
                nfp_net_free_frag(frag, dp->xdp_prog);
                nn_dp_warn(dp, "Failed to map DMA RX buffer\n");
                return NULL;
        }

        return frag;
}

/**
 * nfp_net_tx_ring_init() - Fill in the boilerplate for a TX ring
 * @tx_ring:  TX ring structure
 * @dp:       NFP Net data path struct
 * @r_vec:    IRQ vector servicing this ring
 * @idx:      Ring index
 * @is_xdp:   Is this an XDP TX ring?
 */
static void
nfp_net_tx_ring_init(struct nfp_net_tx_ring *tx_ring, struct nfp_net_dp *dp,
                     struct nfp_net_r_vector *r_vec, unsigned int idx,
                     bool is_xdp)
{
        struct nfp_net *nn = r_vec->nfp_net;

        tx_ring->idx = idx;
        tx_ring->r_vec = r_vec;
        tx_ring->is_xdp = is_xdp;
        u64_stats_init(&tx_ring->r_vec->tx_sync);

        tx_ring->qcidx = tx_ring->idx * nn->stride_tx;
        tx_ring->txrwb = dp->txrwb ? &dp->txrwb[idx] : NULL;
        tx_ring->qcp_q = nn->tx_bar + NFP_QCP_QUEUE_OFF(tx_ring->qcidx);
}

/**
 * nfp_net_rx_ring_init() - Fill in the boilerplate for a RX ring
 * @rx_ring:  RX ring structure
 * @r_vec:    IRQ vector servicing this ring
 * @idx:      Ring index
 */
static void
nfp_net_rx_ring_init(struct nfp_net_rx_ring *rx_ring,
                     struct nfp_net_r_vector *r_vec, unsigned int idx)
{
        struct nfp_net *nn = r_vec->nfp_net;

        rx_ring->idx = idx;
        rx_ring->r_vec = r_vec;
        u64_stats_init(&rx_ring->r_vec->rx_sync);

        rx_ring->fl_qcidx = rx_ring->idx * nn->stride_rx;
        rx_ring->qcp_fl = nn->rx_bar + NFP_QCP_QUEUE_OFF(rx_ring->fl_qcidx);
}

/**
 * nfp_net_rx_ring_reset() - Reflect in SW state of freelist after disable
 * @rx_ring:    RX ring structure
 *
 * Assumes that the device is stopped, must be idempotent.
 */
void nfp_net_rx_ring_reset(struct nfp_net_rx_ring *rx_ring)
{
        unsigned int wr_idx, last_idx;

        /* wr_p == rd_p means ring was never fed FL bufs.  RX rings are always
         * kept at cnt - 1 FL bufs.
         */
        if (rx_ring->wr_p == 0 && rx_ring->rd_p == 0)
                return;

        /* Move the empty entry to the end of the list */
        wr_idx = D_IDX(rx_ring, rx_ring->wr_p);
        last_idx = rx_ring->cnt - 1;
        if (rx_ring->r_vec->xsk_pool) {
                rx_ring->xsk_rxbufs[wr_idx] = rx_ring->xsk_rxbufs[last_idx];
                memset(&rx_ring->xsk_rxbufs[last_idx], 0,
                       sizeof(*rx_ring->xsk_rxbufs));
        } else {
                rx_ring->rxbufs[wr_idx] = rx_ring->rxbufs[last_idx];
                memset(&rx_ring->rxbufs[last_idx], 0, sizeof(*rx_ring->rxbufs));
        }

        memset(rx_ring->rxds, 0, rx_ring->size);
        rx_ring->wr_p = 0;
        rx_ring->rd_p = 0;
}

/**
 * nfp_net_rx_ring_bufs_free() - Free any buffers currently on the RX ring
 * @dp:         NFP Net data path struct
 * @rx_ring:    RX ring to remove buffers from
 *
 * Assumes that the device is stopped and buffers are in [0, ring->cnt - 1)
 * entries.  After device is disabled nfp_net_rx_ring_reset() must be called
 * to restore required ring geometry.
 */
static void
nfp_net_rx_ring_bufs_free(struct nfp_net_dp *dp,
                          struct nfp_net_rx_ring *rx_ring)
{
        unsigned int i;

        if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx))
                return;

        for (i = 0; i < rx_ring->cnt - 1; i++) {
                /* NULL skb can only happen when initial filling of the ring
                 * fails to allocate enough buffers and calls here to free
                 * already allocated ones.
                 */
                if (!rx_ring->rxbufs[i].frag)
                        continue;

                nfp_net_dma_unmap_rx(dp, rx_ring->rxbufs[i].dma_addr);
                nfp_net_free_frag(rx_ring->rxbufs[i].frag, dp->xdp_prog);
                rx_ring->rxbufs[i].dma_addr = 0;
                rx_ring->rxbufs[i].frag = NULL;
        }
}

/**
 * nfp_net_rx_ring_bufs_alloc() - Fill RX ring with buffers (don't give to FW)
 * @dp:         NFP Net data path struct
 * @rx_ring:    RX ring to remove buffers from
 */
static int
nfp_net_rx_ring_bufs_alloc(struct nfp_net_dp *dp,
                           struct nfp_net_rx_ring *rx_ring)
{
        struct nfp_net_rx_buf *rxbufs;
        unsigned int i;

        if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx))
                return 0;

        rxbufs = rx_ring->rxbufs;

        for (i = 0; i < rx_ring->cnt - 1; i++) {
                rxbufs[i].frag = nfp_net_rx_alloc_one(dp, &rxbufs[i].dma_addr);
                if (!rxbufs[i].frag) {
                        nfp_net_rx_ring_bufs_free(dp, rx_ring);
                        return -ENOMEM;
                }
        }

        return 0;
}

int nfp_net_tx_rings_prepare(struct nfp_net *nn, struct nfp_net_dp *dp)
{
        unsigned int r;

        dp->tx_rings = kzalloc_objs(*dp->tx_rings, dp->num_tx_rings);
        if (!dp->tx_rings)
                return -ENOMEM;

        if (dp->ctrl & NFP_NET_CFG_CTRL_TXRWB) {
                dp->txrwb = dma_alloc_coherent(dp->dev,
                                               dp->num_tx_rings * sizeof(u64),
                                               &dp->txrwb_dma, GFP_KERNEL);
                if (!dp->txrwb)
                        goto err_free_rings;
        }

        for (r = 0; r < dp->num_tx_rings; r++) {
                int bias = 0;

                if (r >= dp->num_stack_tx_rings)
                        bias = dp->num_stack_tx_rings;

                nfp_net_tx_ring_init(&dp->tx_rings[r], dp,
                                     &nn->r_vecs[r - bias], r, bias);

                if (nfp_net_tx_ring_alloc(dp, &dp->tx_rings[r]))
                        goto err_free_prev;

                if (nfp_net_tx_ring_bufs_alloc(dp, &dp->tx_rings[r]))
                        goto err_free_ring;
        }

        return 0;

err_free_prev:
        while (r--) {
                nfp_net_tx_ring_bufs_free(dp, &dp->tx_rings[r]);
err_free_ring:
                nfp_net_tx_ring_free(dp, &dp->tx_rings[r]);
        }
        if (dp->txrwb)
                dma_free_coherent(dp->dev, dp->num_tx_rings * sizeof(u64),
                                  dp->txrwb, dp->txrwb_dma);
err_free_rings:
        kfree(dp->tx_rings);
        return -ENOMEM;
}

void nfp_net_tx_rings_free(struct nfp_net_dp *dp)
{
        unsigned int r;

        for (r = 0; r < dp->num_tx_rings; r++) {
                nfp_net_tx_ring_bufs_free(dp, &dp->tx_rings[r]);
                nfp_net_tx_ring_free(dp, &dp->tx_rings[r]);
        }

        if (dp->txrwb)
                dma_free_coherent(dp->dev, dp->num_tx_rings * sizeof(u64),
                                  dp->txrwb, dp->txrwb_dma);
        kfree(dp->tx_rings);
}

/**
 * nfp_net_rx_ring_free() - Free resources allocated to a RX ring
 * @rx_ring:  RX ring to free
 */
static void nfp_net_rx_ring_free(struct nfp_net_rx_ring *rx_ring)
{
        struct nfp_net_r_vector *r_vec = rx_ring->r_vec;
        struct nfp_net_dp *dp = &r_vec->nfp_net->dp;

        if (dp->netdev)
                xdp_rxq_info_unreg(&rx_ring->xdp_rxq);

        if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx))
                kvfree(rx_ring->xsk_rxbufs);
        else
                kvfree(rx_ring->rxbufs);

        if (rx_ring->rxds)
                dma_free_coherent(dp->dev, rx_ring->size,
                                  rx_ring->rxds, rx_ring->dma);

        rx_ring->cnt = 0;
        rx_ring->rxbufs = NULL;
        rx_ring->xsk_rxbufs = NULL;
        rx_ring->rxds = NULL;
        rx_ring->dma = 0;
        rx_ring->size = 0;
}

/**
 * nfp_net_rx_ring_alloc() - Allocate resource for a RX ring
 * @dp:       NFP Net data path struct
 * @rx_ring:  RX ring to allocate
 *
 * Return: 0 on success, negative errno otherwise.
 */
static int
nfp_net_rx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_rx_ring *rx_ring)
{
        enum xdp_mem_type mem_type;
        size_t rxbuf_sw_desc_sz;
        int err;

        if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx)) {
                mem_type = MEM_TYPE_XSK_BUFF_POOL;
                rxbuf_sw_desc_sz = sizeof(*rx_ring->xsk_rxbufs);
        } else {
                mem_type = MEM_TYPE_PAGE_ORDER0;
                rxbuf_sw_desc_sz = sizeof(*rx_ring->rxbufs);
        }

        if (dp->netdev) {
                err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, dp->netdev,
                                       rx_ring->idx, rx_ring->r_vec->napi.napi_id);
                if (err < 0)
                        return err;

                err = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, mem_type, NULL);
                if (err)
                        goto err_alloc;
        }

        rx_ring->cnt = dp->rxd_cnt;
        rx_ring->size = array_size(rx_ring->cnt, sizeof(*rx_ring->rxds));
        rx_ring->rxds = dma_alloc_coherent(dp->dev, rx_ring->size,
                                           &rx_ring->dma,
                                           GFP_KERNEL | __GFP_NOWARN);
        if (!rx_ring->rxds) {
                netdev_warn(dp->netdev, "failed to allocate RX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n",
                            rx_ring->cnt);
                goto err_alloc;
        }

        if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx)) {
                rx_ring->xsk_rxbufs = kvcalloc(rx_ring->cnt, rxbuf_sw_desc_sz,
                                               GFP_KERNEL);
                if (!rx_ring->xsk_rxbufs)
                        goto err_alloc;
        } else {
                rx_ring->rxbufs = kvcalloc(rx_ring->cnt, rxbuf_sw_desc_sz,
                                           GFP_KERNEL);
                if (!rx_ring->rxbufs)
                        goto err_alloc;
        }

        return 0;

err_alloc:
        nfp_net_rx_ring_free(rx_ring);
        return -ENOMEM;
}

int nfp_net_rx_rings_prepare(struct nfp_net *nn, struct nfp_net_dp *dp)
{
        unsigned int r;

        dp->rx_rings = kzalloc_objs(*dp->rx_rings, dp->num_rx_rings);
        if (!dp->rx_rings)
                return -ENOMEM;

        for (r = 0; r < dp->num_rx_rings; r++) {
                nfp_net_rx_ring_init(&dp->rx_rings[r], &nn->r_vecs[r], r);

                if (nfp_net_rx_ring_alloc(dp, &dp->rx_rings[r]))
                        goto err_free_prev;

                if (nfp_net_rx_ring_bufs_alloc(dp, &dp->rx_rings[r]))
                        goto err_free_ring;
        }

        return 0;

err_free_prev:
        while (r--) {
                nfp_net_rx_ring_bufs_free(dp, &dp->rx_rings[r]);
err_free_ring:
                nfp_net_rx_ring_free(&dp->rx_rings[r]);
        }
        kfree(dp->rx_rings);
        return -ENOMEM;
}

void nfp_net_rx_rings_free(struct nfp_net_dp *dp)
{
        unsigned int r;

        for (r = 0; r < dp->num_rx_rings; r++) {
                nfp_net_rx_ring_bufs_free(dp, &dp->rx_rings[r]);
                nfp_net_rx_ring_free(&dp->rx_rings[r]);
        }

        kfree(dp->rx_rings);
}

void
nfp_net_rx_ring_hw_cfg_write(struct nfp_net *nn,
                             struct nfp_net_rx_ring *rx_ring, unsigned int idx)
{
        /* Write the DMA address, size and MSI-X info to the device */
        nn_writeq(nn, NFP_NET_CFG_RXR_ADDR(idx), rx_ring->dma);
        nn_writeb(nn, NFP_NET_CFG_RXR_SZ(idx), ilog2(rx_ring->cnt));
        nn_writeb(nn, NFP_NET_CFG_RXR_VEC(idx), rx_ring->r_vec->irq_entry);
}

void
nfp_net_tx_ring_hw_cfg_write(struct nfp_net *nn,
                             struct nfp_net_tx_ring *tx_ring, unsigned int idx)
{
        nn_writeq(nn, NFP_NET_CFG_TXR_ADDR(idx), tx_ring->dma);
        if (tx_ring->txrwb) {
                *tx_ring->txrwb = 0;
                nn_writeq(nn, NFP_NET_CFG_TXR_WB_ADDR(idx),
                          nn->dp.txrwb_dma + idx * sizeof(u64));
        }
        nn_writeb(nn, NFP_NET_CFG_TXR_SZ(idx), ilog2(tx_ring->cnt));
        nn_writeb(nn, NFP_NET_CFG_TXR_VEC(idx), tx_ring->r_vec->irq_entry);
}

void nfp_net_vec_clear_ring_data(struct nfp_net *nn, unsigned int idx)
{
        nn_writeq(nn, NFP_NET_CFG_RXR_ADDR(idx), 0);
        nn_writeb(nn, NFP_NET_CFG_RXR_SZ(idx), 0);
        nn_writeb(nn, NFP_NET_CFG_RXR_VEC(idx), 0);

        nn_writeq(nn, NFP_NET_CFG_TXR_ADDR(idx), 0);
        nn_writeq(nn, NFP_NET_CFG_TXR_WB_ADDR(idx), 0);
        nn_writeb(nn, NFP_NET_CFG_TXR_SZ(idx), 0);
        nn_writeb(nn, NFP_NET_CFG_TXR_VEC(idx), 0);
}

netdev_tx_t nfp_net_tx(struct sk_buff *skb, struct net_device *netdev)
{
        struct nfp_net *nn = netdev_priv(netdev);

        return nn->dp.ops->xmit(skb, netdev);
}

bool __nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb)
{
        struct nfp_net_r_vector *r_vec = &nn->r_vecs[0];

        return nn->dp.ops->ctrl_tx_one(nn, r_vec, skb, false);
}

bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb)
{
        struct nfp_net_r_vector *r_vec = &nn->r_vecs[0];
        bool ret;

        spin_lock_bh(&r_vec->lock);
        ret = nn->dp.ops->ctrl_tx_one(nn, r_vec, skb, false);
        spin_unlock_bh(&r_vec->lock);

        return ret;
}

bool nfp_net_vlan_strip(struct sk_buff *skb, const struct nfp_net_rx_desc *rxd,
                        const struct nfp_meta_parsed *meta)
{
        u16 tpid = 0, tci = 0;

        if (rxd->rxd.flags & PCIE_DESC_RX_VLAN) {
                tpid = ETH_P_8021Q;
                tci = le16_to_cpu(rxd->rxd.vlan);
        } else if (meta->vlan.stripped) {
                if (meta->vlan.tpid == NFP_NET_VLAN_CTAG)
                        tpid = ETH_P_8021Q;
                else if (meta->vlan.tpid == NFP_NET_VLAN_STAG)
                        tpid = ETH_P_8021AD;
                else
                        return false;

                tci = meta->vlan.tci;
        }
        if (tpid)
                __vlan_hwaccel_put_tag(skb, htons(tpid), tci);

        return true;
}