// SPDX-License-Identifier: GPL-2.0-only
// Copyright 2024 Cisco Systems, Inc.  All rights reserved.

#include <linux/skbuff.h>
#include <linux/if_vlan.h>
#include <net/busy_poll.h>
#include "enic.h"
#include "enic_res.h"
#include "enic_rq.h"
#include "vnic_rq.h"
#include "cq_enet_desc.h"

#define ENIC_LARGE_PKT_THRESHOLD                1000

static void enic_intr_update_pkt_size(struct vnic_rx_bytes_counter *pkt_size,
                                      u32 pkt_len)
{
        if (pkt_len > ENIC_LARGE_PKT_THRESHOLD)
                pkt_size->large_pkt_bytes_cnt += pkt_len;
        else
                pkt_size->small_pkt_bytes_cnt += pkt_len;
}

static void enic_rq_cq_desc_dec(void *cq_desc, u8 cq_desc_size, u8 *type,
                                u8 *color, u16 *q_number, u16 *completed_index)
{
        /* type_color is the last field for all cq structs */
        u8 type_color;

        switch (cq_desc_size) {
        case VNIC_RQ_CQ_ENTRY_SIZE_16: {
                struct cq_enet_rq_desc *desc =
                        (struct cq_enet_rq_desc *)cq_desc;
                type_color = desc->type_color;

                /* Make sure color bit is read from desc *before* other fields
                 * are read from desc.  Hardware guarantees color bit is last
                 * bit (byte) written.  Adding the rmb() prevents the compiler
                 * and/or CPU from reordering the reads which would potentially
                 * result in reading stale values.
                 */
                rmb();

                *q_number = le16_to_cpu(desc->q_number_rss_type_flags) &
                            CQ_DESC_Q_NUM_MASK;
                *completed_index = le16_to_cpu(desc->completed_index_flags) &
                                   CQ_DESC_COMP_NDX_MASK;
                break;
        }
        case VNIC_RQ_CQ_ENTRY_SIZE_32: {
                struct cq_enet_rq_desc_32 *desc =
                        (struct cq_enet_rq_desc_32 *)cq_desc;
                type_color = desc->type_color;

                /* Make sure color bit is read from desc *before* other fields
                 * are read from desc.  Hardware guarantees color bit is last
                 * bit (byte) written.  Adding the rmb() prevents the compiler
                 * and/or CPU from reordering the reads which would potentially
                 * result in reading stale values.
                 */
                rmb();

                *q_number = le16_to_cpu(desc->q_number_rss_type_flags) &
                            CQ_DESC_Q_NUM_MASK;
                *completed_index = le16_to_cpu(desc->completed_index_flags) &
                                   CQ_DESC_COMP_NDX_MASK;
                *completed_index |= (desc->fetch_index_flags & CQ_DESC_32_FI_MASK) <<
                                CQ_DESC_COMP_NDX_BITS;
                break;
        }
        case VNIC_RQ_CQ_ENTRY_SIZE_64: {
                struct cq_enet_rq_desc_64 *desc =
                        (struct cq_enet_rq_desc_64 *)cq_desc;
                type_color = desc->type_color;

                /* Make sure color bit is read from desc *before* other fields
                 * are read from desc.  Hardware guarantees color bit is last
                 * bit (byte) written.  Adding the rmb() prevents the compiler
                 * and/or CPU from reordering the reads which would potentially
                 * result in reading stale values.
                 */
                rmb();

                *q_number = le16_to_cpu(desc->q_number_rss_type_flags) &
                            CQ_DESC_Q_NUM_MASK;
                *completed_index = le16_to_cpu(desc->completed_index_flags) &
                                   CQ_DESC_COMP_NDX_MASK;
                *completed_index |= (desc->fetch_index_flags & CQ_DESC_64_FI_MASK) <<
                                CQ_DESC_COMP_NDX_BITS;
                break;
        }
        }

        *color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK;
        *type = type_color & CQ_DESC_TYPE_MASK;
}

static void enic_rq_set_skb_flags(struct vnic_rq *vrq, u8 type, u32 rss_hash,
                                  u8 rss_type, u8 fcoe, u8 fcoe_fc_crc_ok,
                                  u8 vlan_stripped, u8 csum_not_calc,
                                  u8 tcp_udp_csum_ok, u8 ipv6, u8 ipv4_csum_ok,
                                  u16 vlan_tci, struct sk_buff *skb)
{
        struct enic *enic = vnic_dev_priv(vrq->vdev);
        struct net_device *netdev = enic->netdev;
        struct enic_rq_stats *rqstats =  &enic->rq[vrq->index].stats;
        bool outer_csum_ok = true, encap = false;

        if ((netdev->features & NETIF_F_RXHASH) && rss_hash && type == 3) {
                switch (rss_type) {
                case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv4:
                case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6:
                case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6_EX:
                        skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L4);
                        rqstats->l4_rss_hash++;
                        break;
                case CQ_ENET_RQ_DESC_RSS_TYPE_IPv4:
                case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6:
                case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6_EX:
                        skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L3);
                        rqstats->l3_rss_hash++;
                        break;
                }
        }
        if (enic->vxlan.vxlan_udp_port_number) {
                switch (enic->vxlan.patch_level) {
                case 0:
                        if (fcoe) {
                                encap = true;
                                outer_csum_ok = fcoe_fc_crc_ok;
                        }
                        break;
                case 2:
                        if (type == 7 && (rss_hash & BIT(0))) {
                                encap = true;
                                outer_csum_ok = (rss_hash & BIT(1)) &&
                                                (rss_hash & BIT(2));
                        }
                        break;
                }
        }

        /* Hardware does not provide whole packet checksum. It only
         * provides pseudo checksum. Since hw validates the packet
         * checksum but not provide us the checksum value. use
         * CHECSUM_UNNECESSARY.
         *
         * In case of encap pkt tcp_udp_csum_ok/tcp_udp_csum_ok is
         * inner csum_ok. outer_csum_ok is set by hw when outer udp
         * csum is correct or is zero.
         */
        if ((netdev->features & NETIF_F_RXCSUM) && !csum_not_calc &&
            tcp_udp_csum_ok && outer_csum_ok && (ipv4_csum_ok || ipv6)) {
                skb->ip_summed = CHECKSUM_UNNECESSARY;
                skb->csum_level = encap;
                if (encap)
                        rqstats->csum_unnecessary_encap++;
                else
                        rqstats->csum_unnecessary++;
        }

        if (vlan_stripped) {
                __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci);
                rqstats->vlan_stripped++;
        }
}

/*
 * cq_enet_rq_desc accesses section uses only the 1st 15 bytes of the cq which
 * is identical for all type (16,32 and 64 byte) of cqs.
 */
static void cq_enet_rq_desc_dec(struct cq_enet_rq_desc *desc, u8 *ingress_port,
                                u8 *fcoe, u8 *eop, u8 *sop, u8 *rss_type,
                                u8 *csum_not_calc, u32 *rss_hash,
                                u16 *bytes_written, u8 *packet_error,
                                u8 *vlan_stripped, u16 *vlan_tci,
                                u16 *checksum, u8 *fcoe_sof,
                                u8 *fcoe_fc_crc_ok, u8 *fcoe_enc_error,
                                u8 *fcoe_eof, u8 *tcp_udp_csum_ok, u8 *udp,
                                u8 *tcp, u8 *ipv4_csum_ok, u8 *ipv6, u8 *ipv4,
                                u8 *ipv4_fragment, u8 *fcs_ok)
{
        u16 completed_index_flags;
        u16 q_number_rss_type_flags;
        u16 bytes_written_flags;

        completed_index_flags = le16_to_cpu(desc->completed_index_flags);
        q_number_rss_type_flags =
                le16_to_cpu(desc->q_number_rss_type_flags);
        bytes_written_flags = le16_to_cpu(desc->bytes_written_flags);

        *ingress_port = (completed_index_flags &
                CQ_ENET_RQ_DESC_FLAGS_INGRESS_PORT) ? 1 : 0;
        *fcoe = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_FCOE) ?
                1 : 0;
        *eop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_EOP) ?
                1 : 0;
        *sop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_SOP) ?
                1 : 0;

        *rss_type = (u8)((q_number_rss_type_flags >> CQ_DESC_Q_NUM_BITS) &
                CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
        *csum_not_calc = (q_number_rss_type_flags &
                CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ? 1 : 0;

        *rss_hash = le32_to_cpu(desc->rss_hash);

        *bytes_written = bytes_written_flags &
                CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
        *packet_error = (bytes_written_flags &
                CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ? 1 : 0;
        *vlan_stripped = (bytes_written_flags &
                CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) ? 1 : 0;

        /*
         * Tag Control Information(16) = user_priority(3) + cfi(1) + vlan(12)
         */
        *vlan_tci = le16_to_cpu(desc->vlan);

        if (*fcoe) {
                *fcoe_sof = (u8)(le16_to_cpu(desc->checksum_fcoe) &
                        CQ_ENET_RQ_DESC_FCOE_SOF_MASK);
                *fcoe_fc_crc_ok = (desc->flags &
                        CQ_ENET_RQ_DESC_FCOE_FC_CRC_OK) ? 1 : 0;
                *fcoe_enc_error = (desc->flags &
                        CQ_ENET_RQ_DESC_FCOE_ENC_ERROR) ? 1 : 0;
                *fcoe_eof = (u8)((le16_to_cpu(desc->checksum_fcoe) >>
                        CQ_ENET_RQ_DESC_FCOE_EOF_SHIFT) &
                        CQ_ENET_RQ_DESC_FCOE_EOF_MASK);
                *checksum = 0;
        } else {
                *fcoe_sof = 0;
                *fcoe_fc_crc_ok = 0;
                *fcoe_enc_error = 0;
                *fcoe_eof = 0;
                *checksum = le16_to_cpu(desc->checksum_fcoe);
        }

        *tcp_udp_csum_ok =
                (desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ? 1 : 0;
        *udp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_UDP) ? 1 : 0;
        *tcp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP) ? 1 : 0;
        *ipv4_csum_ok =
                (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ? 1 : 0;
        *ipv6 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV6) ? 1 : 0;
        *ipv4 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4) ? 1 : 0;
        *ipv4_fragment =
                (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT) ? 1 : 0;
        *fcs_ok = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) ? 1 : 0;
}

static bool enic_rq_pkt_error(struct vnic_rq *vrq, u8 packet_error, u8 fcs_ok,
                              u16 bytes_written)
{
        struct enic *enic = vnic_dev_priv(vrq->vdev);
        struct enic_rq_stats *rqstats = &enic->rq[vrq->index].stats;

        if (packet_error) {
                if (!fcs_ok) {
                        if (bytes_written > 0)
                                rqstats->bad_fcs++;
                        else if (bytes_written == 0)
                                rqstats->pkt_truncated++;
                }
                return true;
        }
        return false;
}

int enic_rq_alloc_buf(struct vnic_rq *rq)
{
        struct enic *enic = vnic_dev_priv(rq->vdev);
        struct net_device *netdev = enic->netdev;
        struct enic_rq *erq = &enic->rq[rq->index];
        struct enic_rq_stats *rqstats = &erq->stats;
        unsigned int offset = 0;
        unsigned int len = netdev->mtu + VLAN_ETH_HLEN;
        unsigned int os_buf_index = 0;
        dma_addr_t dma_addr;
        struct vnic_rq_buf *buf = rq->to_use;
        struct page *page;
        unsigned int truesize = len;

        if (buf->os_buf) {
                enic_queue_rq_desc(rq, buf->os_buf, os_buf_index, buf->dma_addr,
                                   buf->len);

                return 0;
        }

        page = page_pool_dev_alloc(erq->pool, &offset, &truesize);
        if (unlikely(!page)) {
                rqstats->pp_alloc_fail++;
                return -ENOMEM;
        }
        buf->offset = offset;
        buf->truesize = truesize;
        dma_addr = page_pool_get_dma_addr(page) + offset;
        enic_queue_rq_desc(rq, (void *)page, os_buf_index, dma_addr, len);

        return 0;
}

void enic_free_rq_buf(struct vnic_rq *rq, struct vnic_rq_buf *buf)
{
        struct enic *enic = vnic_dev_priv(rq->vdev);
        struct enic_rq *erq = &enic->rq[rq->index];

        if (!buf->os_buf)
                return;

        page_pool_put_full_page(erq->pool, (struct page *)buf->os_buf, true);
        buf->os_buf = NULL;
}

static void enic_rq_indicate_buf(struct enic *enic, struct vnic_rq *rq,
                                 struct vnic_rq_buf *buf, void *cq_desc,
                                 u8 type, u16 q_number, u16 completed_index)
{
        struct sk_buff *skb;
        struct vnic_cq *cq = &enic->cq[enic_cq_rq(enic, rq->index)];
        struct enic_rq_stats *rqstats = &enic->rq[rq->index].stats;
        struct napi_struct *napi;

        u8 eop, sop, ingress_port, vlan_stripped;
        u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof;
        u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok;
        u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc;
        u8 packet_error;
        u16 bytes_written, vlan_tci, checksum;
        u32 rss_hash;

        rqstats->packets++;

        cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, &ingress_port,
                            &fcoe, &eop, &sop, &rss_type, &csum_not_calc,
                            &rss_hash, &bytes_written, &packet_error,
                            &vlan_stripped, &vlan_tci, &checksum, &fcoe_sof,
                            &fcoe_fc_crc_ok, &fcoe_enc_error, &fcoe_eof,
                            &tcp_udp_csum_ok, &udp, &tcp, &ipv4_csum_ok, &ipv6,
                            &ipv4, &ipv4_fragment, &fcs_ok);

        if (enic_rq_pkt_error(rq, packet_error, fcs_ok, bytes_written))
                return;

        if (eop && bytes_written > 0) {
                /* Good receive
                 */
                rqstats->bytes += bytes_written;
                napi = &enic->napi[rq->index];
                skb = napi_get_frags(napi);
                if (unlikely(!skb)) {
                        net_warn_ratelimited("%s: skb alloc error rq[%d], desc[%d]\n",
                                             enic->netdev->name, rq->index,
                                             completed_index);
                        rqstats->no_skb++;
                        return;
                }

                prefetch(skb->data - NET_IP_ALIGN);

                dma_sync_single_for_cpu(&enic->pdev->dev, buf->dma_addr,
                                        bytes_written, DMA_FROM_DEVICE);
                skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
                                (struct page *)buf->os_buf, buf->offset,
                                bytes_written, buf->truesize);
                skb_record_rx_queue(skb, q_number);
                enic_rq_set_skb_flags(rq, type, rss_hash, rss_type, fcoe,
                                      fcoe_fc_crc_ok, vlan_stripped,
                                      csum_not_calc, tcp_udp_csum_ok, ipv6,
                                      ipv4_csum_ok, vlan_tci, skb);
                skb_mark_for_recycle(skb);
                napi_gro_frags(napi);
                if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce)
                        enic_intr_update_pkt_size(&cq->pkt_size_counter,
                                                  bytes_written);
                buf->os_buf = NULL;
                buf->dma_addr = 0;
                buf = buf->next;
        } else {
                /* Buffer overflow
                 */
                rqstats->pkt_truncated++;
        }
}

static void enic_rq_service(struct enic *enic, void *cq_desc, u8 type,
                            u16 q_number, u16 completed_index)
{
        struct enic_rq_stats *rqstats = &enic->rq[q_number].stats;
        struct vnic_rq *vrq = &enic->rq[q_number].vrq;
        struct vnic_rq_buf *vrq_buf = vrq->to_clean;
        int skipped;

        while (1) {
                skipped = (vrq_buf->index != completed_index);
                if (!skipped)
                        enic_rq_indicate_buf(enic, vrq, vrq_buf, cq_desc, type,
                                             q_number, completed_index);
                else
                        rqstats->desc_skip++;

                vrq->ring.desc_avail++;
                vrq->to_clean = vrq_buf->next;
                vrq_buf = vrq_buf->next;
                if (!skipped)
                        break;
        }
}

unsigned int enic_rq_cq_service(struct enic *enic, unsigned int cq_index,
                                unsigned int work_to_do)
{
        struct vnic_cq *cq = &enic->cq[cq_index];
        void *cq_desc = vnic_cq_to_clean(cq);
        u16 q_number, completed_index;
        unsigned int work_done = 0;
        u8 type, color;

        enic_rq_cq_desc_dec(cq_desc, enic->ext_cq, &type, &color, &q_number,
                            &completed_index);

        while (color != cq->last_color) {
                enic_rq_service(enic, cq_desc, type, q_number, completed_index);
                vnic_cq_inc_to_clean(cq);

                if (++work_done >= work_to_do)
                        break;

                cq_desc = vnic_cq_to_clean(cq);
                enic_rq_cq_desc_dec(cq_desc, enic->ext_cq, &type, &color,
                                    &q_number, &completed_index);
        }

        return work_done;
}