// SPDX-License-Identifier: GPL-2.0
/*
 * SpacemiT K1 Ethernet driver
 *
 * Copyright (C) 2023-2025 SpacemiT (Hangzhou) Technology Co. Ltd
 * Copyright (C) 2025 Vivian Wang <wangruikang@iscas.ac.cn>
 */

#include <linux/bitfield.h>
#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/etherdevice.h>
#include <linux/ethtool.h>
#include <linux/if_vlan.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/iopoll.h>
#include <linux/kernel.h>
#include <linux/mfd/syscon.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_irq.h>
#include <linux/of_mdio.h>
#include <linux/of_net.h>
#include <linux/phy.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/pm.h>
#include <linux/regmap.h>
#include <linux/reset.h>
#include <linux/rtnetlink.h>
#include <linux/timer.h>
#include <linux/types.h>

#include "k1_emac.h"

#define DRIVER_NAME "k1_emac"

#define EMAC_DEFAULT_BUFSIZE            1536
#define EMAC_RX_BUF_2K                  2048
#define EMAC_RX_BUF_MAX                 FIELD_MAX(RX_DESC_1_BUFFER_SIZE_1_MASK)

/* Tuning parameters from SpacemiT */
#define EMAC_TX_FRAMES                  64
#define EMAC_TX_COAL_TIMEOUT            40000
#define EMAC_RX_FRAMES                  64
#define EMAC_RX_COAL_TIMEOUT            (600 * 312)

#define DEFAULT_TX_ALMOST_FULL          0x1f8
#define DEFAULT_TX_THRESHOLD            1518
#define DEFAULT_RX_THRESHOLD            12
#define DEFAULT_TX_RING_NUM             1024
#define DEFAULT_RX_RING_NUM             1024
#define DEFAULT_DMA_BURST               MREGBIT_BURST_16WORD
#define HASH_TABLE_SIZE                 64

struct desc_buf {
        u64 dma_addr;
        void *buff_addr;
        u16 dma_len;
        u8 map_as_page;
};

struct emac_tx_desc_buffer {
        struct sk_buff *skb;
        struct desc_buf buf[2];
};

struct emac_rx_desc_buffer {
        struct sk_buff *skb;
        u64 dma_addr;
        void *buff_addr;
        u16 dma_len;
        u8 map_as_page;
};

/**
 * struct emac_desc_ring - Software-side information for one descriptor ring
 * Same structure used for both RX and TX
 * @desc_addr: Virtual address to the descriptor ring memory
 * @desc_dma_addr: DMA address of the descriptor ring
 * @total_size: Size of ring in bytes
 * @total_cnt: Number of descriptors
 * @head: Next descriptor to associate a buffer with
 * @tail: Next descriptor to check status bit
 * @rx_desc_buf: Array of descriptors for RX
 * @tx_desc_buf: Array of descriptors for TX, with max of two buffers each
 */
struct emac_desc_ring {
        void *desc_addr;
        dma_addr_t desc_dma_addr;
        u32 total_size;
        u32 total_cnt;
        u32 head;
        u32 tail;
        union {
                struct emac_rx_desc_buffer *rx_desc_buf;
                struct emac_tx_desc_buffer *tx_desc_buf;
        };
};

struct emac_priv {
        void __iomem *iobase;
        u32 dma_buf_sz;
        struct emac_desc_ring tx_ring;
        struct emac_desc_ring rx_ring;

        struct net_device *ndev;
        struct napi_struct napi;
        struct platform_device *pdev;
        struct clk *bus_clk;
        struct clk *ref_clk;
        struct regmap *regmap_apmu;
        u32 regmap_apmu_offset;
        int irq;

        phy_interface_t phy_interface;

        union emac_hw_tx_stats tx_stats, tx_stats_off;
        union emac_hw_rx_stats rx_stats, rx_stats_off;

        u32 tx_count_frames;
        u32 tx_coal_frames;
        u32 tx_coal_timeout;
        struct work_struct tx_timeout_task;

        struct timer_list txtimer;
        struct timer_list stats_timer;

        u32 tx_delay;
        u32 rx_delay;

        /* Softirq-safe, hold while touching hardware statistics */
        spinlock_t stats_lock;
};

static void emac_wr(struct emac_priv *priv, u32 reg, u32 val)
{
        writel(val, priv->iobase + reg);
}

static u32 emac_rd(struct emac_priv *priv, u32 reg)
{
        return readl(priv->iobase + reg);
}

static int emac_phy_interface_config(struct emac_priv *priv)
{
        u32 val = 0, mask = REF_CLK_SEL | RGMII_TX_CLK_SEL | PHY_INTF_RGMII;

        if (phy_interface_mode_is_rgmii(priv->phy_interface))
                val |= PHY_INTF_RGMII;

        regmap_update_bits(priv->regmap_apmu,
                           priv->regmap_apmu_offset + APMU_EMAC_CTRL_REG,
                           mask, val);

        return 0;
}

/*
 * Where the hardware expects a MAC address, it is laid out in this high, med,
 * low order in three consecutive registers and in this format.
 */

static void emac_set_mac_addr_reg(struct emac_priv *priv,
                                  const unsigned char *addr,
                                  u32 reg)
{
        emac_wr(priv, reg + sizeof(u32) * 0, addr[1] << 8 | addr[0]);
        emac_wr(priv, reg + sizeof(u32) * 1, addr[3] << 8 | addr[2]);
        emac_wr(priv, reg + sizeof(u32) * 2, addr[5] << 8 | addr[4]);
}

static void emac_set_mac_addr(struct emac_priv *priv, const unsigned char *addr)
{
        emac_set_mac_addr_reg(priv, addr, MAC_ADDRESS1_HIGH);
}

static void emac_reset_hw(struct emac_priv *priv)
{
        /* Disable all interrupts */
        emac_wr(priv, MAC_INTERRUPT_ENABLE, 0x0);
        emac_wr(priv, DMA_INTERRUPT_ENABLE, 0x0);

        /* Disable transmit and receive units */
        emac_wr(priv, MAC_RECEIVE_CONTROL, 0x0);
        emac_wr(priv, MAC_TRANSMIT_CONTROL, 0x0);

        /* Disable DMA */
        emac_wr(priv, DMA_CONTROL, 0x0);
}

static void emac_init_hw(struct emac_priv *priv)
{
        u32 rxirq = 0, dma = 0, frame_sz;

        regmap_set_bits(priv->regmap_apmu,
                        priv->regmap_apmu_offset + APMU_EMAC_CTRL_REG,
                        AXI_SINGLE_ID);

        /* Disable transmit and receive units */
        emac_wr(priv, MAC_RECEIVE_CONTROL, 0x0);
        emac_wr(priv, MAC_TRANSMIT_CONTROL, 0x0);

        /* Enable MAC address 1 filtering */
        emac_wr(priv, MAC_ADDRESS_CONTROL, MREGBIT_MAC_ADDRESS1_ENABLE);

        /* Zero initialize the multicast hash table */
        emac_wr(priv, MAC_MULTICAST_HASH_TABLE1, 0x0);
        emac_wr(priv, MAC_MULTICAST_HASH_TABLE2, 0x0);
        emac_wr(priv, MAC_MULTICAST_HASH_TABLE3, 0x0);
        emac_wr(priv, MAC_MULTICAST_HASH_TABLE4, 0x0);

        /* Configure thresholds */
        emac_wr(priv, MAC_TRANSMIT_FIFO_ALMOST_FULL, DEFAULT_TX_ALMOST_FULL);
        emac_wr(priv, MAC_TRANSMIT_PACKET_START_THRESHOLD,
                DEFAULT_TX_THRESHOLD);
        emac_wr(priv, MAC_RECEIVE_PACKET_START_THRESHOLD, DEFAULT_RX_THRESHOLD);

        /* Set maximum frame size and jabber size based on configured MTU,
         * accounting for Ethernet header, double VLAN tags, and FCS.
         */
        frame_sz = priv->ndev->mtu + ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN;

        emac_wr(priv, MAC_MAXIMUM_FRAME_SIZE, frame_sz);
        emac_wr(priv, MAC_TRANSMIT_JABBER_SIZE, frame_sz);
        emac_wr(priv, MAC_RECEIVE_JABBER_SIZE, frame_sz);

        /* RX IRQ mitigation */
        rxirq = FIELD_PREP(MREGBIT_RECEIVE_IRQ_FRAME_COUNTER_MASK,
                           EMAC_RX_FRAMES);
        rxirq |= FIELD_PREP(MREGBIT_RECEIVE_IRQ_TIMEOUT_COUNTER_MASK,
                            EMAC_RX_COAL_TIMEOUT);
        rxirq |= MREGBIT_RECEIVE_IRQ_MITIGATION_ENABLE;
        emac_wr(priv, DMA_RECEIVE_IRQ_MITIGATION_CTRL, rxirq);

        /* Disable and set DMA config */
        emac_wr(priv, DMA_CONTROL, 0x0);

        emac_wr(priv, DMA_CONFIGURATION, MREGBIT_SOFTWARE_RESET);
        usleep_range(9000, 10000);
        emac_wr(priv, DMA_CONFIGURATION, 0x0);
        usleep_range(9000, 10000);

        dma |= MREGBIT_STRICT_BURST;
        dma |= MREGBIT_DMA_64BIT_MODE;
        dma |= DEFAULT_DMA_BURST;

        emac_wr(priv, DMA_CONFIGURATION, dma);
}

static void emac_dma_start_transmit(struct emac_priv *priv)
{
        /* The actual value written does not matter */
        emac_wr(priv, DMA_TRANSMIT_POLL_DEMAND, 1);
}

static void emac_enable_interrupt(struct emac_priv *priv)
{
        u32 val;

        val = emac_rd(priv, DMA_INTERRUPT_ENABLE);
        val |= MREGBIT_TRANSMIT_TRANSFER_DONE_INTR_ENABLE;
        val |= MREGBIT_RECEIVE_TRANSFER_DONE_INTR_ENABLE;
        emac_wr(priv, DMA_INTERRUPT_ENABLE, val);
}

static void emac_disable_interrupt(struct emac_priv *priv)
{
        u32 val;

        val = emac_rd(priv, DMA_INTERRUPT_ENABLE);
        val &= ~MREGBIT_TRANSMIT_TRANSFER_DONE_INTR_ENABLE;
        val &= ~MREGBIT_RECEIVE_TRANSFER_DONE_INTR_ENABLE;
        emac_wr(priv, DMA_INTERRUPT_ENABLE, val);
}

static u32 emac_tx_avail(struct emac_priv *priv)
{
        struct emac_desc_ring *tx_ring = &priv->tx_ring;
        u32 avail;

        if (tx_ring->tail > tx_ring->head)
                avail = tx_ring->tail - tx_ring->head - 1;
        else
                avail = tx_ring->total_cnt - tx_ring->head + tx_ring->tail - 1;

        return avail;
}

static void emac_tx_coal_timer_resched(struct emac_priv *priv)
{
        mod_timer(&priv->txtimer,
                  jiffies + usecs_to_jiffies(priv->tx_coal_timeout));
}

static void emac_tx_coal_timer(struct timer_list *t)
{
        struct emac_priv *priv = timer_container_of(priv, t, txtimer);

        napi_schedule(&priv->napi);
}

static bool emac_tx_should_interrupt(struct emac_priv *priv, u32 pkt_num)
{
        priv->tx_count_frames += pkt_num;
        if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
                emac_tx_coal_timer_resched(priv);
                return false;
        }

        priv->tx_count_frames = 0;
        return true;
}

static void emac_free_tx_buf(struct emac_priv *priv, int i)
{
        struct emac_tx_desc_buffer *tx_buf;
        struct emac_desc_ring *tx_ring;
        struct desc_buf *buf;
        int j;

        tx_ring = &priv->tx_ring;
        tx_buf = &tx_ring->tx_desc_buf[i];

        for (j = 0; j < 2; j++) {
                buf = &tx_buf->buf[j];
                if (!buf->dma_addr)
                        continue;

                if (buf->map_as_page)
                        dma_unmap_page(&priv->pdev->dev, buf->dma_addr,
                                       buf->dma_len, DMA_TO_DEVICE);
                else
                        dma_unmap_single(&priv->pdev->dev,
                                         buf->dma_addr, buf->dma_len,
                                         DMA_TO_DEVICE);

                buf->dma_addr = 0;
                buf->map_as_page = false;
                buf->buff_addr = NULL;
        }

        if (tx_buf->skb) {
                dev_kfree_skb_any(tx_buf->skb);
                tx_buf->skb = NULL;
        }
}

static void emac_clean_tx_desc_ring(struct emac_priv *priv)
{
        struct emac_desc_ring *tx_ring = &priv->tx_ring;
        u32 i;

        for (i = 0; i < tx_ring->total_cnt; i++)
                emac_free_tx_buf(priv, i);

        tx_ring->head = 0;
        tx_ring->tail = 0;
}

static void emac_clean_rx_desc_ring(struct emac_priv *priv)
{
        struct emac_rx_desc_buffer *rx_buf;
        struct emac_desc_ring *rx_ring;
        u32 i;

        rx_ring = &priv->rx_ring;

        for (i = 0; i < rx_ring->total_cnt; i++) {
                rx_buf = &rx_ring->rx_desc_buf[i];

                if (!rx_buf->skb)
                        continue;

                dma_unmap_single(&priv->pdev->dev, rx_buf->dma_addr,
                                 rx_buf->dma_len, DMA_FROM_DEVICE);

                dev_kfree_skb(rx_buf->skb);
                rx_buf->skb = NULL;
        }

        rx_ring->tail = 0;
        rx_ring->head = 0;
}

static int emac_alloc_tx_resources(struct emac_priv *priv)
{
        struct emac_desc_ring *tx_ring = &priv->tx_ring;
        struct platform_device *pdev = priv->pdev;

        tx_ring->tx_desc_buf = kzalloc_objs(*tx_ring->tx_desc_buf,
                                            tx_ring->total_cnt);

        if (!tx_ring->tx_desc_buf)
                return -ENOMEM;

        tx_ring->total_size = tx_ring->total_cnt * sizeof(struct emac_desc);
        tx_ring->total_size = ALIGN(tx_ring->total_size, PAGE_SIZE);

        tx_ring->desc_addr = dma_alloc_coherent(&pdev->dev, tx_ring->total_size,
                                                &tx_ring->desc_dma_addr,
                                                GFP_KERNEL);
        if (!tx_ring->desc_addr) {
                kfree(tx_ring->tx_desc_buf);
                return -ENOMEM;
        }

        tx_ring->head = 0;
        tx_ring->tail = 0;

        return 0;
}

static int emac_alloc_rx_resources(struct emac_priv *priv)
{
        struct emac_desc_ring *rx_ring = &priv->rx_ring;
        struct platform_device *pdev = priv->pdev;

        rx_ring->rx_desc_buf = kzalloc_objs(*rx_ring->rx_desc_buf,
                                            rx_ring->total_cnt);
        if (!rx_ring->rx_desc_buf)
                return -ENOMEM;

        rx_ring->total_size = rx_ring->total_cnt * sizeof(struct emac_desc);

        rx_ring->total_size = ALIGN(rx_ring->total_size, PAGE_SIZE);

        rx_ring->desc_addr = dma_alloc_coherent(&pdev->dev, rx_ring->total_size,
                                                &rx_ring->desc_dma_addr,
                                                GFP_KERNEL);
        if (!rx_ring->desc_addr) {
                kfree(rx_ring->rx_desc_buf);
                return -ENOMEM;
        }

        rx_ring->head = 0;
        rx_ring->tail = 0;

        return 0;
}

static void emac_free_tx_resources(struct emac_priv *priv)
{
        struct emac_desc_ring *tr = &priv->tx_ring;
        struct device *dev = &priv->pdev->dev;

        emac_clean_tx_desc_ring(priv);

        kfree(tr->tx_desc_buf);
        tr->tx_desc_buf = NULL;

        dma_free_coherent(dev, tr->total_size, tr->desc_addr,
                          tr->desc_dma_addr);
        tr->desc_addr = NULL;
}

static void emac_free_rx_resources(struct emac_priv *priv)
{
        struct emac_desc_ring *rr = &priv->rx_ring;
        struct device *dev = &priv->pdev->dev;

        emac_clean_rx_desc_ring(priv);

        kfree(rr->rx_desc_buf);
        rr->rx_desc_buf = NULL;

        dma_free_coherent(dev, rr->total_size, rr->desc_addr,
                          rr->desc_dma_addr);
        rr->desc_addr = NULL;
}

static int emac_tx_clean_desc(struct emac_priv *priv)
{
        struct net_device *ndev = priv->ndev;
        struct emac_desc_ring *tx_ring;
        struct emac_desc *tx_desc;
        u32 i;

        netif_tx_lock(ndev);

        tx_ring = &priv->tx_ring;

        i = tx_ring->tail;

        while (i != tx_ring->head) {
                tx_desc = &((struct emac_desc *)tx_ring->desc_addr)[i];

                /* Stop checking if desc still own by DMA */
                if (READ_ONCE(tx_desc->desc0) & TX_DESC_0_OWN)
                        break;

                emac_free_tx_buf(priv, i);
                memset(tx_desc, 0, sizeof(struct emac_desc));

                if (++i == tx_ring->total_cnt)
                        i = 0;
        }

        tx_ring->tail = i;

        if (unlikely(netif_queue_stopped(ndev) &&
                     emac_tx_avail(priv) > tx_ring->total_cnt / 4))
                netif_wake_queue(ndev);

        netif_tx_unlock(ndev);

        return 0;
}

static bool emac_rx_frame_good(struct emac_priv *priv, struct emac_desc *desc)
{
        const char *msg;
        u32 len;

        len = FIELD_GET(RX_DESC_0_FRAME_PACKET_LENGTH_MASK, desc->desc0);

        if (WARN_ON_ONCE(!(desc->desc0 & RX_DESC_0_LAST_DESCRIPTOR)))
                msg = "Not last descriptor"; /* This would be a bug */
        else if (desc->desc0 & RX_DESC_0_FRAME_RUNT)
                msg = "Runt frame";
        else if (desc->desc0 & RX_DESC_0_FRAME_CRC_ERR)
                msg = "Frame CRC error";
        else if (desc->desc0 & RX_DESC_0_FRAME_MAX_LEN_ERR)
                msg = "Frame exceeds max length";
        else if (desc->desc0 & RX_DESC_0_FRAME_JABBER_ERR)
                msg = "Frame jabber error";
        else if (desc->desc0 & RX_DESC_0_FRAME_LENGTH_ERR)
                msg = "Frame length error";
        else if (len <= ETH_FCS_LEN || len > priv->dma_buf_sz)
                msg = "Frame length unacceptable";
        else
                return true; /* All good */

        dev_dbg_ratelimited(&priv->ndev->dev, "RX error: %s", msg);

        return false;
}

static void emac_alloc_rx_desc_buffers(struct emac_priv *priv)
{
        struct emac_desc_ring *rx_ring = &priv->rx_ring;
        struct emac_desc rx_desc, *rx_desc_addr;
        struct net_device *ndev = priv->ndev;
        struct emac_rx_desc_buffer *rx_buf;
        struct sk_buff *skb;
        u32 i;

        i = rx_ring->head;
        rx_buf = &rx_ring->rx_desc_buf[i];

        while (!rx_buf->skb) {
                skb = netdev_alloc_skb_ip_align(ndev, priv->dma_buf_sz);
                if (!skb)
                        break;

                skb->dev = ndev;

                rx_buf->skb = skb;
                rx_buf->dma_len = priv->dma_buf_sz;
                rx_buf->dma_addr = dma_map_single(&priv->pdev->dev, skb->data,
                                                  priv->dma_buf_sz,
                                                  DMA_FROM_DEVICE);
                if (dma_mapping_error(&priv->pdev->dev, rx_buf->dma_addr)) {
                        dev_err_ratelimited(&ndev->dev, "Mapping skb failed\n");
                        dev_kfree_skb_any(skb);
                        rx_buf->skb = NULL;
                        break;
                }

                rx_desc_addr = &((struct emac_desc *)rx_ring->desc_addr)[i];

                memset(&rx_desc, 0, sizeof(rx_desc));

                rx_desc.buffer_addr_1 = rx_buf->dma_addr;
                rx_desc.desc1 = FIELD_PREP(RX_DESC_1_BUFFER_SIZE_1_MASK,
                                           rx_buf->dma_len);

                if (++i == rx_ring->total_cnt) {
                        rx_desc.desc1 |= RX_DESC_1_END_RING;
                        i = 0;
                }

                *rx_desc_addr = rx_desc;
                dma_wmb();
                WRITE_ONCE(rx_desc_addr->desc0, rx_desc.desc0 | RX_DESC_0_OWN);

                rx_buf = &rx_ring->rx_desc_buf[i];
        }

        rx_ring->head = i;
        return;
}

/* Returns number of packets received */
static int emac_rx_clean_desc(struct emac_priv *priv, int budget)
{
        struct net_device *ndev = priv->ndev;
        struct emac_rx_desc_buffer *rx_buf;
        struct emac_desc_ring *rx_ring;
        struct sk_buff *skb = NULL;
        struct emac_desc *rx_desc;
        u32 got = 0, skb_len, i;

        rx_ring = &priv->rx_ring;

        i = rx_ring->tail;

        while (budget--) {
                rx_desc = &((struct emac_desc *)rx_ring->desc_addr)[i];

                /* Stop checking if rx_desc still owned by DMA */
                if (READ_ONCE(rx_desc->desc0) & RX_DESC_0_OWN)
                        break;

                dma_rmb();

                rx_buf = &rx_ring->rx_desc_buf[i];

                if (!rx_buf->skb)
                        break;

                got++;

                dma_unmap_single(&priv->pdev->dev, rx_buf->dma_addr,
                                 rx_buf->dma_len, DMA_FROM_DEVICE);

                if (likely(emac_rx_frame_good(priv, rx_desc))) {
                        skb = rx_buf->skb;

                        skb_len = FIELD_GET(RX_DESC_0_FRAME_PACKET_LENGTH_MASK,
                                            rx_desc->desc0);
                        skb_len -= ETH_FCS_LEN;

                        skb_put(skb, skb_len);
                        skb->dev = ndev;
                        ndev->hard_header_len = ETH_HLEN;

                        skb->protocol = eth_type_trans(skb, ndev);

                        skb->ip_summed = CHECKSUM_NONE;

                        napi_gro_receive(&priv->napi, skb);

                        memset(rx_desc, 0, sizeof(struct emac_desc));
                        rx_buf->skb = NULL;
                } else {
                        dev_kfree_skb_irq(rx_buf->skb);
                        rx_buf->skb = NULL;
                }

                if (++i == rx_ring->total_cnt)
                        i = 0;
        }

        rx_ring->tail = i;

        emac_alloc_rx_desc_buffers(priv);

        return got;
}

static int emac_rx_poll(struct napi_struct *napi, int budget)
{
        struct emac_priv *priv = container_of(napi, struct emac_priv, napi);
        int work_done;

        emac_tx_clean_desc(priv);

        work_done = emac_rx_clean_desc(priv, budget);
        if (work_done < budget && napi_complete_done(napi, work_done))
                emac_enable_interrupt(priv);

        return work_done;
}

/*
 * For convenience, skb->data is fragment 0, frags[0] is fragment 1, etc.
 *
 * Each descriptor can hold up to two fragments, called buffer 1 and 2. For each
 * fragment f, if f % 2 == 0, it uses buffer 1, otherwise it uses buffer 2.
 */

static int emac_tx_map_frag(struct device *dev, struct emac_desc *tx_desc,
                            struct emac_tx_desc_buffer *tx_buf,
                            struct sk_buff *skb, u32 frag_idx)
{
        bool map_as_page, buf_idx;
        const skb_frag_t *frag;
        phys_addr_t addr;
        u32 len;
        int ret;

        buf_idx = frag_idx % 2;

        if (frag_idx == 0) {
                /* Non-fragmented part */
                len = skb_headlen(skb);
                addr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
                map_as_page = false;
        } else {
                /* Fragment */
                frag = &skb_shinfo(skb)->frags[frag_idx - 1];
                len = skb_frag_size(frag);
                addr = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE);
                map_as_page = true;
        }

        ret = dma_mapping_error(dev, addr);
        if (ret)
                return ret;

        tx_buf->buf[buf_idx].dma_addr = addr;
        tx_buf->buf[buf_idx].dma_len = len;
        tx_buf->buf[buf_idx].map_as_page = map_as_page;

        if (buf_idx == 0) {
                tx_desc->buffer_addr_1 = addr;
                tx_desc->desc1 |= FIELD_PREP(TX_DESC_1_BUFFER_SIZE_1_MASK, len);
        } else {
                tx_desc->buffer_addr_2 = addr;
                tx_desc->desc1 |= FIELD_PREP(TX_DESC_1_BUFFER_SIZE_2_MASK, len);
        }

        return 0;
}

static void emac_tx_mem_map(struct emac_priv *priv, struct sk_buff *skb)
{
        struct emac_desc_ring *tx_ring = &priv->tx_ring;
        struct emac_desc tx_desc, *tx_desc_addr;
        struct device *dev = &priv->pdev->dev;
        struct emac_tx_desc_buffer *tx_buf;
        u32 head, old_head, frag_num, f, i;
        bool buf_idx;

        frag_num = skb_shinfo(skb)->nr_frags;
        head = tx_ring->head;
        old_head = head;

        for (f = 0; f < frag_num + 1; f++) {
                buf_idx = f % 2;

                /*
                 * If using buffer 1, initialize a new desc. Otherwise, use
                 * buffer 2 of previous fragment's desc.
                 */
                if (!buf_idx) {
                        tx_buf = &tx_ring->tx_desc_buf[head];
                        tx_desc_addr =
                                &((struct emac_desc *)tx_ring->desc_addr)[head];
                        memset(&tx_desc, 0, sizeof(tx_desc));

                        /*
                         * Give ownership for all but first desc initially. For
                         * first desc, give at the end so DMA cannot start
                         * reading uninitialized descs.
                         */
                        if (head != old_head)
                                tx_desc.desc0 |= TX_DESC_0_OWN;

                        if (++head == tx_ring->total_cnt) {
                                /* Just used last desc in ring */
                                tx_desc.desc1 |= TX_DESC_1_END_RING;
                                head = 0;
                        }
                }

                if (emac_tx_map_frag(dev, &tx_desc, tx_buf, skb, f)) {
                        dev_err_ratelimited(&priv->ndev->dev,
                                            "Map TX frag %d failed\n", f);
                        goto err_free_skb;
                }

                if (f == 0)
                        tx_desc.desc1 |= TX_DESC_1_FIRST_SEGMENT;

                if (f == frag_num) {
                        tx_desc.desc1 |= TX_DESC_1_LAST_SEGMENT;
                        tx_buf->skb = skb;
                        if (emac_tx_should_interrupt(priv, frag_num + 1))
                                tx_desc.desc1 |=
                                        TX_DESC_1_INTERRUPT_ON_COMPLETION;
                }

                *tx_desc_addr = tx_desc;
        }

        /* All descriptors are ready, give ownership for first desc */
        tx_desc_addr = &((struct emac_desc *)tx_ring->desc_addr)[old_head];
        dma_wmb();
        WRITE_ONCE(tx_desc_addr->desc0, tx_desc_addr->desc0 | TX_DESC_0_OWN);

        emac_dma_start_transmit(priv);

        tx_ring->head = head;

        return;

err_free_skb:
        dev_dstats_tx_dropped(priv->ndev);

        i = old_head;
        while (i != head) {
                emac_free_tx_buf(priv, i);

                if (++i == tx_ring->total_cnt)
                        i = 0;
        }

        dev_kfree_skb_any(skb);
}

static netdev_tx_t emac_start_xmit(struct sk_buff *skb, struct net_device *ndev)
{
        struct emac_priv *priv = netdev_priv(ndev);
        int nfrags = skb_shinfo(skb)->nr_frags;
        struct device *dev = &priv->pdev->dev;

        if (unlikely(emac_tx_avail(priv) < nfrags + 1)) {
                if (!netif_queue_stopped(ndev)) {
                        netif_stop_queue(ndev);
                        dev_err_ratelimited(dev, "TX ring full, stop TX queue\n");
                }
                return NETDEV_TX_BUSY;
        }

        emac_tx_mem_map(priv, skb);

        /* Make sure there is space in the ring for the next TX. */
        if (unlikely(emac_tx_avail(priv) <= MAX_SKB_FRAGS + 2))
                netif_stop_queue(ndev);

        return NETDEV_TX_OK;
}

static int emac_set_mac_address(struct net_device *ndev, void *addr)
{
        struct emac_priv *priv = netdev_priv(ndev);
        int ret = eth_mac_addr(ndev, addr);

        if (ret)
                return ret;

        /* If running, set now; if not running it will be set in emac_up. */
        if (netif_running(ndev))
                emac_set_mac_addr(priv, ndev->dev_addr);

        return 0;
}

static void emac_mac_multicast_filter_clear(struct emac_priv *priv)
{
        emac_wr(priv, MAC_MULTICAST_HASH_TABLE1, 0x0);
        emac_wr(priv, MAC_MULTICAST_HASH_TABLE2, 0x0);
        emac_wr(priv, MAC_MULTICAST_HASH_TABLE3, 0x0);
        emac_wr(priv, MAC_MULTICAST_HASH_TABLE4, 0x0);
}

/*
 * The upper 6 bits of the Ethernet CRC of the MAC address is used as the hash
 * when matching multicast addresses.
 */
static u32 emac_ether_addr_hash(u8 addr[ETH_ALEN])
{
        u32 crc32 = ether_crc(ETH_ALEN, addr);

        return crc32 >> 26;
}

/* Configure Multicast and Promiscuous modes */
static void emac_set_rx_mode(struct net_device *ndev)
{
        struct emac_priv *priv = netdev_priv(ndev);
        struct netdev_hw_addr *ha;
        u32 mc_filter[4] = { 0 };
        u32 hash, reg, bit, val;

        val = emac_rd(priv, MAC_ADDRESS_CONTROL);

        val &= ~MREGBIT_PROMISCUOUS_MODE;

        if (ndev->flags & IFF_PROMISC) {
                /* Enable promisc mode */
                val |= MREGBIT_PROMISCUOUS_MODE;
        } else if ((ndev->flags & IFF_ALLMULTI) ||
                   (netdev_mc_count(ndev) > HASH_TABLE_SIZE)) {
                /* Accept all multicast frames by setting every bit */
                emac_wr(priv, MAC_MULTICAST_HASH_TABLE1, 0xffff);
                emac_wr(priv, MAC_MULTICAST_HASH_TABLE2, 0xffff);
                emac_wr(priv, MAC_MULTICAST_HASH_TABLE3, 0xffff);
                emac_wr(priv, MAC_MULTICAST_HASH_TABLE4, 0xffff);
        } else if (!netdev_mc_empty(ndev)) {
                emac_mac_multicast_filter_clear(priv);
                netdev_for_each_mc_addr(ha, ndev) {
                        /*
                         * The hash table is an array of 4 16-bit registers. It
                         * is treated like an array of 64 bits (bits[hash]).
                         */
                        hash = emac_ether_addr_hash(ha->addr);
                        reg = hash / 16;
                        bit = hash % 16;
                        mc_filter[reg] |= BIT(bit);
                }
                emac_wr(priv, MAC_MULTICAST_HASH_TABLE1, mc_filter[0]);
                emac_wr(priv, MAC_MULTICAST_HASH_TABLE2, mc_filter[1]);
                emac_wr(priv, MAC_MULTICAST_HASH_TABLE3, mc_filter[2]);
                emac_wr(priv, MAC_MULTICAST_HASH_TABLE4, mc_filter[3]);
        }

        emac_wr(priv, MAC_ADDRESS_CONTROL, val);
}

static int emac_change_mtu(struct net_device *ndev, int mtu)
{
        struct emac_priv *priv = netdev_priv(ndev);
        u32 frame_len;

        if (netif_running(ndev)) {
                netdev_err(ndev, "must be stopped to change MTU\n");
                return -EBUSY;
        }

        frame_len = mtu + ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN;

        if (frame_len <= EMAC_DEFAULT_BUFSIZE)
                priv->dma_buf_sz = EMAC_DEFAULT_BUFSIZE;
        else if (frame_len <= EMAC_RX_BUF_2K)
                priv->dma_buf_sz = EMAC_RX_BUF_2K;
        else
                priv->dma_buf_sz = EMAC_RX_BUF_MAX;

        ndev->mtu = mtu;

        return 0;
}

static void emac_tx_timeout(struct net_device *ndev, unsigned int txqueue)
{
        struct emac_priv *priv = netdev_priv(ndev);

        schedule_work(&priv->tx_timeout_task);
}

static int emac_mii_read(struct mii_bus *bus, int phy_addr, int regnum)
{
        struct emac_priv *priv = bus->priv;
        u32 cmd = 0, val;
        int ret;

        cmd |= FIELD_PREP(MREGBIT_PHY_ADDRESS, phy_addr);
        cmd |= FIELD_PREP(MREGBIT_REGISTER_ADDRESS, regnum);
        cmd |= MREGBIT_START_MDIO_TRANS | MREGBIT_MDIO_READ_WRITE;

        emac_wr(priv, MAC_MDIO_DATA, 0x0);
        emac_wr(priv, MAC_MDIO_CONTROL, cmd);

        ret = readl_poll_timeout(priv->iobase + MAC_MDIO_CONTROL, val,
                                 !(val & MREGBIT_START_MDIO_TRANS), 100, 10000);

        if (ret)
                return ret;

        val = emac_rd(priv, MAC_MDIO_DATA);
        return FIELD_GET(MREGBIT_MDIO_DATA, val);
}

static int emac_mii_write(struct mii_bus *bus, int phy_addr, int regnum,
                          u16 value)
{
        struct emac_priv *priv = bus->priv;
        u32 cmd = 0, val;
        int ret;

        emac_wr(priv, MAC_MDIO_DATA, value);

        cmd |= FIELD_PREP(MREGBIT_PHY_ADDRESS, phy_addr);
        cmd |= FIELD_PREP(MREGBIT_REGISTER_ADDRESS, regnum);
        cmd |= MREGBIT_START_MDIO_TRANS;

        emac_wr(priv, MAC_MDIO_CONTROL, cmd);

        ret = readl_poll_timeout(priv->iobase + MAC_MDIO_CONTROL, val,
                                 !(val & MREGBIT_START_MDIO_TRANS), 100, 10000);

        return ret;
}

static int emac_mdio_init(struct emac_priv *priv)
{
        struct device *dev = &priv->pdev->dev;
        struct device_node *mii_np;
        struct mii_bus *mii;
        int ret;

        mii = devm_mdiobus_alloc(dev);
        if (!mii)
                return -ENOMEM;

        mii->priv = priv;
        mii->name = "k1_emac_mii";
        mii->read = emac_mii_read;
        mii->write = emac_mii_write;
        mii->parent = dev;
        mii->phy_mask = ~0;
        snprintf(mii->id, MII_BUS_ID_SIZE, "%s", priv->pdev->name);

        mii_np = of_get_available_child_by_name(dev->of_node, "mdio-bus");

        ret = devm_of_mdiobus_register(dev, mii, mii_np);
        if (ret)
                dev_err_probe(dev, ret, "Failed to register mdio bus\n");

        of_node_put(mii_np);
        return ret;
}

/*
 * Even though this MAC supports gigabit operation, it only provides 32-bit
 * statistics counters. The most overflow-prone counters are the "bytes" ones,
 * which at gigabit overflow about twice a minute.
 *
 * Therefore, we maintain the high 32 bits of counters ourselves, incrementing
 * every time statistics seem to go backwards. Also, update periodically to
 * catch overflows when we are not otherwise checking the statistics often
 * enough.
 */

#define EMAC_STATS_TIMER_PERIOD         20

static int emac_read_stat_cnt(struct emac_priv *priv, u8 cnt, u32 *res,
                              u32 control_reg, u32 high_reg, u32 low_reg)
{
        u32 val, high, low;
        int ret;

        /* The "read" bit is the same for TX and RX */

        val = MREGBIT_START_TX_COUNTER_READ | cnt;
        emac_wr(priv, control_reg, val);
        val = emac_rd(priv, control_reg);

        ret = readl_poll_timeout_atomic(priv->iobase + control_reg, val,
                                        !(val & MREGBIT_START_TX_COUNTER_READ),
                                        100, 10000);

        if (ret) {
                /*
                 * This could be caused by the PHY stopping its refclk even when
                 * the link is up, for power saving. See also comments in
                 * emac_stats_update().
                 */
                dev_err_ratelimited(&priv->ndev->dev,
                                    "Read stat timeout. PHY clock stopped?\n");
                return ret;
        }

        high = emac_rd(priv, high_reg);
        low = emac_rd(priv, low_reg);
        *res = high << 16 | lower_16_bits(low);

        return 0;
}

static int emac_tx_read_stat_cnt(struct emac_priv *priv, u8 cnt, u32 *res)
{
        return emac_read_stat_cnt(priv, cnt, res, MAC_TX_STATCTR_CONTROL,
                                  MAC_TX_STATCTR_DATA_HIGH,
                                  MAC_TX_STATCTR_DATA_LOW);
}

static int emac_rx_read_stat_cnt(struct emac_priv *priv, u8 cnt, u32 *res)
{
        return emac_read_stat_cnt(priv, cnt, res, MAC_RX_STATCTR_CONTROL,
                                  MAC_RX_STATCTR_DATA_HIGH,
                                  MAC_RX_STATCTR_DATA_LOW);
}

static void emac_update_counter(u64 *counter, u32 new_low)
{
        u32 old_low = lower_32_bits(*counter);
        u64 high = upper_32_bits(*counter);

        if (old_low > new_low) {
                /* Overflowed, increment high 32 bits */
                high++;
        }

        *counter = (high << 32) | new_low;
}

static void emac_stats_update(struct emac_priv *priv)
{
        u64 *tx_stats_off = priv->tx_stats_off.array;
        u64 *rx_stats_off = priv->rx_stats_off.array;
        u64 *tx_stats = priv->tx_stats.array;
        u64 *rx_stats = priv->rx_stats.array;
        u32 i, res, offset;

        assert_spin_locked(&priv->stats_lock);

        /*
         * We can't read statistics if the interface is not up. Also, some PHYs
         * stop their reference clocks for link down power saving, which also
         * causes reading statistics to time out. Don't update and don't
         * reschedule in these cases.
         */
        if (!netif_running(priv->ndev) ||
            !netif_carrier_ok(priv->ndev) ||
            !netif_device_present(priv->ndev)) {
                return;
        }

        for (i = 0; i < sizeof(priv->tx_stats) / sizeof(*tx_stats); i++) {
                /*
                 * If reading stats times out anyway, the stat registers will be
                 * stuck, and we can't really recover from that.
                 *
                 * Reading statistics also can't return an error, so just return
                 * without updating and without rescheduling.
                 */
                if (emac_tx_read_stat_cnt(priv, i, &res))
                        return;

                /*
                 * Re-initializing while bringing interface up resets counters
                 * to zero, so to provide continuity, we add the values saved
                 * last time we did emac_down() to the new hardware-provided
                 * value.
                 */
                offset = lower_32_bits(tx_stats_off[i]);
                emac_update_counter(&tx_stats[i], res + offset);
        }

        /* Similar remarks as TX stats */
        for (i = 0; i < sizeof(priv->rx_stats) / sizeof(*rx_stats); i++) {
                if (emac_rx_read_stat_cnt(priv, i, &res))
                        return;
                offset = lower_32_bits(rx_stats_off[i]);
                emac_update_counter(&rx_stats[i], res + offset);
        }

        mod_timer(&priv->stats_timer, jiffies + EMAC_STATS_TIMER_PERIOD * HZ);
}

static void emac_stats_timer(struct timer_list *t)
{
        struct emac_priv *priv = timer_container_of(priv, t, stats_timer);

        spin_lock(&priv->stats_lock);

        emac_stats_update(priv);

        spin_unlock(&priv->stats_lock);
}

static const struct ethtool_rmon_hist_range emac_rmon_hist_ranges[] = {
        {   64,   64 },
        {   65,  127 },
        {  128,  255 },
        {  256,  511 },
        {  512, 1023 },
        { 1024, 1518 },
        { 1519, 4096 },
        { /* sentinel */ },
};

/* Like dev_fetch_dstats(), but we only use tx_drops */
static u64 emac_get_stat_tx_drops(struct emac_priv *priv)
{
        const struct pcpu_dstats *stats;
        u64 tx_drops, total = 0;
        unsigned int start;
        int cpu;

        for_each_possible_cpu(cpu) {
                stats = per_cpu_ptr(priv->ndev->dstats, cpu);
                do {
                        start = u64_stats_fetch_begin(&stats->syncp);
                        tx_drops = u64_stats_read(&stats->tx_drops);
                } while (u64_stats_fetch_retry(&stats->syncp, start));

                total += tx_drops;
        }

        return total;
}

static void emac_get_stats64(struct net_device *dev,
                             struct rtnl_link_stats64 *storage)
{
        struct emac_priv *priv = netdev_priv(dev);
        union emac_hw_tx_stats *tx_stats;
        union emac_hw_rx_stats *rx_stats;

        tx_stats = &priv->tx_stats;
        rx_stats = &priv->rx_stats;

        /* This is the only software counter */
        storage->tx_dropped = emac_get_stat_tx_drops(priv);

        spin_lock_bh(&priv->stats_lock);

        emac_stats_update(priv);

        storage->tx_packets = tx_stats->stats.tx_ok_pkts;
        storage->tx_bytes = tx_stats->stats.tx_ok_bytes;
        storage->tx_errors = tx_stats->stats.tx_err_pkts;

        storage->rx_packets = rx_stats->stats.rx_ok_pkts;
        storage->rx_bytes = rx_stats->stats.rx_ok_bytes;
        storage->rx_errors = rx_stats->stats.rx_err_total_pkts;
        storage->rx_crc_errors = rx_stats->stats.rx_crc_err_pkts;
        storage->rx_frame_errors = rx_stats->stats.rx_align_err_pkts;
        storage->rx_length_errors = rx_stats->stats.rx_len_err_pkts;

        storage->collisions = tx_stats->stats.tx_singleclsn_pkts;
        storage->collisions += tx_stats->stats.tx_multiclsn_pkts;
        storage->collisions += tx_stats->stats.tx_excessclsn_pkts;

        storage->rx_missed_errors = rx_stats->stats.rx_drp_fifo_full_pkts;
        storage->rx_missed_errors += rx_stats->stats.rx_truncate_fifo_full_pkts;

        spin_unlock_bh(&priv->stats_lock);
}

static void emac_get_rmon_stats(struct net_device *dev,
                                struct ethtool_rmon_stats *rmon_stats,
                                const struct ethtool_rmon_hist_range **ranges)
{
        struct emac_priv *priv = netdev_priv(dev);
        union emac_hw_rx_stats *rx_stats;

        rx_stats = &priv->rx_stats;

        *ranges = emac_rmon_hist_ranges;

        spin_lock_bh(&priv->stats_lock);

        emac_stats_update(priv);

        rmon_stats->undersize_pkts = rx_stats->stats.rx_len_undersize_pkts;
        rmon_stats->oversize_pkts = rx_stats->stats.rx_len_oversize_pkts;
        rmon_stats->fragments = rx_stats->stats.rx_len_fragment_pkts;
        rmon_stats->jabbers = rx_stats->stats.rx_len_jabber_pkts;

        /* Only RX has histogram stats */

        rmon_stats->hist[0] = rx_stats->stats.rx_64_pkts;
        rmon_stats->hist[1] = rx_stats->stats.rx_65_127_pkts;
        rmon_stats->hist[2] = rx_stats->stats.rx_128_255_pkts;
        rmon_stats->hist[3] = rx_stats->stats.rx_256_511_pkts;
        rmon_stats->hist[4] = rx_stats->stats.rx_512_1023_pkts;
        rmon_stats->hist[5] = rx_stats->stats.rx_1024_1518_pkts;
        rmon_stats->hist[6] = rx_stats->stats.rx_1519_plus_pkts;

        spin_unlock_bh(&priv->stats_lock);
}

static void emac_get_eth_mac_stats(struct net_device *dev,
                                   struct ethtool_eth_mac_stats *mac_stats)
{
        struct emac_priv *priv = netdev_priv(dev);
        union emac_hw_tx_stats *tx_stats;
        union emac_hw_rx_stats *rx_stats;

        tx_stats = &priv->tx_stats;
        rx_stats = &priv->rx_stats;

        spin_lock_bh(&priv->stats_lock);

        emac_stats_update(priv);

        mac_stats->MulticastFramesXmittedOK = tx_stats->stats.tx_multicast_pkts;
        mac_stats->BroadcastFramesXmittedOK = tx_stats->stats.tx_broadcast_pkts;

        mac_stats->MulticastFramesReceivedOK =
                rx_stats->stats.rx_multicast_pkts;
        mac_stats->BroadcastFramesReceivedOK =
                rx_stats->stats.rx_broadcast_pkts;

        mac_stats->SingleCollisionFrames = tx_stats->stats.tx_singleclsn_pkts;
        mac_stats->MultipleCollisionFrames = tx_stats->stats.tx_multiclsn_pkts;
        mac_stats->LateCollisions = tx_stats->stats.tx_lateclsn_pkts;
        mac_stats->FramesAbortedDueToXSColls =
                tx_stats->stats.tx_excessclsn_pkts;

        spin_unlock_bh(&priv->stats_lock);
}

static void emac_get_pause_stats(struct net_device *dev,
                                 struct ethtool_pause_stats *pause_stats)
{
        struct emac_priv *priv = netdev_priv(dev);
        union emac_hw_tx_stats *tx_stats;
        union emac_hw_rx_stats *rx_stats;

        tx_stats = &priv->tx_stats;
        rx_stats = &priv->rx_stats;

        spin_lock_bh(&priv->stats_lock);

        emac_stats_update(priv);

        pause_stats->tx_pause_frames = tx_stats->stats.tx_pause_pkts;
        pause_stats->rx_pause_frames = rx_stats->stats.rx_pause_pkts;

        spin_unlock_bh(&priv->stats_lock);
}

/* Other statistics that are not derivable from standard statistics */

#define EMAC_ETHTOOL_STAT(type, name) \
        { offsetof(type, stats.name) / sizeof(u64), #name }

static const struct emac_ethtool_stats {
        size_t offset;
        char str[ETH_GSTRING_LEN];
} emac_ethtool_rx_stats[] = {
        EMAC_ETHTOOL_STAT(union emac_hw_rx_stats, rx_drp_fifo_full_pkts),
        EMAC_ETHTOOL_STAT(union emac_hw_rx_stats, rx_truncate_fifo_full_pkts),
};

static int emac_get_sset_count(struct net_device *dev, int sset)
{
        switch (sset) {
        case ETH_SS_STATS:
                return ARRAY_SIZE(emac_ethtool_rx_stats);
        default:
                return -EOPNOTSUPP;
        }
}

static void emac_get_strings(struct net_device *dev, u32 stringset, u8 *data)
{
        int i;

        switch (stringset) {
        case ETH_SS_STATS:
                for (i = 0; i < ARRAY_SIZE(emac_ethtool_rx_stats); i++) {
                        memcpy(data, emac_ethtool_rx_stats[i].str,
                               ETH_GSTRING_LEN);
                        data += ETH_GSTRING_LEN;
                }
                break;
        }
}

static void emac_get_ethtool_stats(struct net_device *dev,
                                   struct ethtool_stats *stats, u64 *data)
{
        struct emac_priv *priv = netdev_priv(dev);
        u64 *rx_stats = (u64 *)&priv->rx_stats;
        int i;

        spin_lock_bh(&priv->stats_lock);

        emac_stats_update(priv);

        for (i = 0; i < ARRAY_SIZE(emac_ethtool_rx_stats); i++)
                data[i] = rx_stats[emac_ethtool_rx_stats[i].offset];

        spin_unlock_bh(&priv->stats_lock);
}

static int emac_ethtool_get_regs_len(struct net_device *dev)
{
        return (EMAC_DMA_REG_CNT + EMAC_MAC_REG_CNT) * sizeof(u32);
}

static void emac_ethtool_get_regs(struct net_device *dev,
                                  struct ethtool_regs *regs, void *space)
{
        struct emac_priv *priv = netdev_priv(dev);
        u32 *reg_space = space;
        int i;

        regs->version = 1;

        for (i = 0; i < EMAC_DMA_REG_CNT; i++)
                reg_space[i] = emac_rd(priv, DMA_CONFIGURATION + i * 4);

        for (i = 0; i < EMAC_MAC_REG_CNT; i++)
                reg_space[i + EMAC_DMA_REG_CNT] =
                        emac_rd(priv, MAC_GLOBAL_CONTROL + i * 4);
}

static void emac_get_drvinfo(struct net_device *dev,
                             struct ethtool_drvinfo *info)
{
        strscpy(info->driver, DRIVER_NAME, sizeof(info->driver));
        info->n_stats = ARRAY_SIZE(emac_ethtool_rx_stats);
}

static void emac_tx_timeout_task(struct work_struct *work)
{
        struct net_device *ndev;
        struct emac_priv *priv;

        priv = container_of(work, struct emac_priv, tx_timeout_task);
        ndev = priv->ndev;

        rtnl_lock();

        /* No need to reset if already down */
        if (!netif_running(ndev)) {
                rtnl_unlock();
                return;
        }

        netdev_err(ndev, "MAC reset due to TX timeout\n");

        netif_trans_update(ndev); /* prevent tx timeout */
        dev_close(ndev);
        dev_open(ndev, NULL);

        rtnl_unlock();
}

static void emac_sw_init(struct emac_priv *priv)
{
        priv->dma_buf_sz = EMAC_DEFAULT_BUFSIZE;

        priv->tx_ring.total_cnt = DEFAULT_TX_RING_NUM;
        priv->rx_ring.total_cnt = DEFAULT_RX_RING_NUM;

        spin_lock_init(&priv->stats_lock);

        INIT_WORK(&priv->tx_timeout_task, emac_tx_timeout_task);

        priv->tx_coal_frames = EMAC_TX_FRAMES;
        priv->tx_coal_timeout = EMAC_TX_COAL_TIMEOUT;

        timer_setup(&priv->txtimer, emac_tx_coal_timer, 0);
        timer_setup(&priv->stats_timer, emac_stats_timer, 0);
}

static irqreturn_t emac_interrupt_handler(int irq, void *dev_id)
{
        struct net_device *ndev = (struct net_device *)dev_id;
        struct emac_priv *priv = netdev_priv(ndev);
        bool should_schedule = false;
        u32 clr = 0;
        u32 status;

        status = emac_rd(priv, DMA_STATUS_IRQ);

        if (status & MREGBIT_TRANSMIT_TRANSFER_DONE_IRQ) {
                clr |= MREGBIT_TRANSMIT_TRANSFER_DONE_IRQ;
                should_schedule = true;
        }

        if (status & MREGBIT_TRANSMIT_DES_UNAVAILABLE_IRQ)
                clr |= MREGBIT_TRANSMIT_DES_UNAVAILABLE_IRQ;

        if (status & MREGBIT_TRANSMIT_DMA_STOPPED_IRQ)
                clr |= MREGBIT_TRANSMIT_DMA_STOPPED_IRQ;

        if (status & MREGBIT_RECEIVE_TRANSFER_DONE_IRQ) {
                clr |= MREGBIT_RECEIVE_TRANSFER_DONE_IRQ;
                should_schedule = true;
        }

        if (status & MREGBIT_RECEIVE_DES_UNAVAILABLE_IRQ)
                clr |= MREGBIT_RECEIVE_DES_UNAVAILABLE_IRQ;

        if (status & MREGBIT_RECEIVE_DMA_STOPPED_IRQ)
                clr |= MREGBIT_RECEIVE_DMA_STOPPED_IRQ;

        if (status & MREGBIT_RECEIVE_MISSED_FRAME_IRQ)
                clr |= MREGBIT_RECEIVE_MISSED_FRAME_IRQ;

        if (should_schedule) {
                if (napi_schedule_prep(&priv->napi)) {
                        emac_disable_interrupt(priv);
                        __napi_schedule_irqoff(&priv->napi);
                }
        }

        emac_wr(priv, DMA_STATUS_IRQ, clr);

        return IRQ_HANDLED;
}

static void emac_configure_tx(struct emac_priv *priv)
{
        u32 val;

        /* Set base address */
        val = (u32)priv->tx_ring.desc_dma_addr;
        emac_wr(priv, DMA_TRANSMIT_BASE_ADDRESS, val);

        /* Set TX inter-frame gap value, enable transmit */
        val = emac_rd(priv, MAC_TRANSMIT_CONTROL);
        val &= ~MREGBIT_IFG_LEN;
        val |= MREGBIT_TRANSMIT_ENABLE;
        val |= MREGBIT_TRANSMIT_AUTO_RETRY;
        emac_wr(priv, MAC_TRANSMIT_CONTROL, val);

        emac_wr(priv, DMA_TRANSMIT_AUTO_POLL_COUNTER, 0x0);

        /* Start TX DMA */
        val = emac_rd(priv, DMA_CONTROL);
        val |= MREGBIT_START_STOP_TRANSMIT_DMA;
        emac_wr(priv, DMA_CONTROL, val);
}

static void emac_configure_rx(struct emac_priv *priv)
{
        u32 val;

        /* Set base address */
        val = (u32)priv->rx_ring.desc_dma_addr;
        emac_wr(priv, DMA_RECEIVE_BASE_ADDRESS, val);

        /* Enable receive */
        val = emac_rd(priv, MAC_RECEIVE_CONTROL);
        val |= MREGBIT_RECEIVE_ENABLE;
        val |= MREGBIT_STORE_FORWARD;
        emac_wr(priv, MAC_RECEIVE_CONTROL, val);

        /* Start RX DMA */
        val = emac_rd(priv, DMA_CONTROL);
        val |= MREGBIT_START_STOP_RECEIVE_DMA;
        emac_wr(priv, DMA_CONTROL, val);
}

static void emac_adjust_link(struct net_device *dev)
{
        struct emac_priv *priv = netdev_priv(dev);
        struct phy_device *phydev = dev->phydev;
        u32 ctrl;

        if (phydev->link) {
                ctrl = emac_rd(priv, MAC_GLOBAL_CONTROL);

                /* Update duplex and speed from PHY */

                FIELD_MODIFY(MREGBIT_FULL_DUPLEX_MODE, &ctrl,
                             phydev->duplex == DUPLEX_FULL);

                ctrl &= ~MREGBIT_SPEED;

                switch (phydev->speed) {
                case SPEED_1000:
                        ctrl |= MREGBIT_SPEED_1000M;
                        break;
                case SPEED_100:
                        ctrl |= MREGBIT_SPEED_100M;
                        break;
                case SPEED_10:
                        ctrl |= MREGBIT_SPEED_10M;
                        break;
                default:
                        netdev_err(dev, "Unknown speed: %d\n", phydev->speed);
                        phydev->speed = SPEED_UNKNOWN;
                        break;
                }

                emac_wr(priv, MAC_GLOBAL_CONTROL, ctrl);

                /*
                 * Reschedule stats updates now that link is up. See comments in
                 * emac_stats_update().
                 */
                mod_timer(&priv->stats_timer, jiffies);
        }

        phy_print_status(phydev);
}

static void emac_update_delay_line(struct emac_priv *priv)
{
        u32 mask = 0, val = 0;

        mask |= EMAC_RX_DLINE_EN;
        mask |= EMAC_RX_DLINE_STEP_MASK | EMAC_RX_DLINE_CODE_MASK;
        mask |= EMAC_TX_DLINE_EN;
        mask |= EMAC_TX_DLINE_STEP_MASK | EMAC_TX_DLINE_CODE_MASK;

        if (phy_interface_mode_is_rgmii(priv->phy_interface)) {
                val |= EMAC_RX_DLINE_EN;
                val |= FIELD_PREP(EMAC_RX_DLINE_STEP_MASK,
                                  EMAC_DLINE_STEP_15P6);
                val |= FIELD_PREP(EMAC_RX_DLINE_CODE_MASK, priv->rx_delay);

                val |= EMAC_TX_DLINE_EN;
                val |= FIELD_PREP(EMAC_TX_DLINE_STEP_MASK,
                                  EMAC_DLINE_STEP_15P6);
                val |= FIELD_PREP(EMAC_TX_DLINE_CODE_MASK, priv->tx_delay);
        }

        regmap_update_bits(priv->regmap_apmu,
                           priv->regmap_apmu_offset + APMU_EMAC_DLINE_REG,
                           mask, val);
}

static int emac_phy_connect(struct net_device *ndev)
{
        struct emac_priv *priv = netdev_priv(ndev);
        struct device *dev = &priv->pdev->dev;
        struct phy_device *phydev;
        struct device_node *np;
        int ret;

        ret = of_get_phy_mode(dev->of_node, &priv->phy_interface);
        if (ret) {
                netdev_err(ndev, "No phy-mode found");
                return ret;
        }

        switch (priv->phy_interface) {
        case PHY_INTERFACE_MODE_RMII:
        case PHY_INTERFACE_MODE_RGMII:
        case PHY_INTERFACE_MODE_RGMII_ID:
        case PHY_INTERFACE_MODE_RGMII_RXID:
        case PHY_INTERFACE_MODE_RGMII_TXID:
                break;
        default:
                netdev_err(ndev, "Unsupported PHY interface %s",
                           phy_modes(priv->phy_interface));
                return -EINVAL;
        }

        np = of_parse_phandle(dev->of_node, "phy-handle", 0);
        if (!np && of_phy_is_fixed_link(dev->of_node))
                np = of_node_get(dev->of_node);

        if (!np) {
                netdev_err(ndev, "No PHY specified");
                return -ENODEV;
        }

        ret = emac_phy_interface_config(priv);
        if (ret)
                goto err_node_put;

        phydev = of_phy_connect(ndev, np, &emac_adjust_link, 0,
                                priv->phy_interface);
        if (!phydev) {
                netdev_err(ndev, "Could not attach to PHY\n");
                ret = -ENODEV;
                goto err_node_put;
        }

        phydev->mac_managed_pm = true;

        emac_update_delay_line(priv);

        phy_attached_info(phydev);

err_node_put:
        of_node_put(np);
        return ret;
}

static int emac_up(struct emac_priv *priv)
{
        struct platform_device *pdev = priv->pdev;
        struct net_device *ndev = priv->ndev;
        int ret;

        pm_runtime_get_sync(&pdev->dev);

        ret = emac_phy_connect(ndev);
        if (ret) {
                dev_err(&pdev->dev, "emac_phy_connect failed\n");
                goto err_pm_put;
        }

        emac_init_hw(priv);

        emac_set_mac_addr(priv, ndev->dev_addr);
        emac_configure_tx(priv);
        emac_configure_rx(priv);

        emac_alloc_rx_desc_buffers(priv);

        phy_start(ndev->phydev);

        ret = request_irq(priv->irq, emac_interrupt_handler, IRQF_SHARED,
                          ndev->name, ndev);
        if (ret) {
                dev_err(&pdev->dev, "request_irq failed\n");
                goto err_reset_disconnect_phy;
        }

        /* Don't enable MAC interrupts */
        emac_wr(priv, MAC_INTERRUPT_ENABLE, 0x0);

        /* Enable DMA interrupts */
        emac_wr(priv, DMA_INTERRUPT_ENABLE,
                MREGBIT_TRANSMIT_TRANSFER_DONE_INTR_ENABLE |
                        MREGBIT_TRANSMIT_DMA_STOPPED_INTR_ENABLE |
                        MREGBIT_RECEIVE_TRANSFER_DONE_INTR_ENABLE |
                        MREGBIT_RECEIVE_DMA_STOPPED_INTR_ENABLE |
                        MREGBIT_RECEIVE_MISSED_FRAME_INTR_ENABLE);

        napi_enable(&priv->napi);

        netif_start_queue(ndev);

        mod_timer(&priv->stats_timer, jiffies);

        return 0;

err_reset_disconnect_phy:
        emac_reset_hw(priv);
        phy_disconnect(ndev->phydev);

err_pm_put:
        pm_runtime_put_sync(&pdev->dev);
        return ret;
}

static int emac_down(struct emac_priv *priv)
{
        struct platform_device *pdev = priv->pdev;
        struct net_device *ndev = priv->ndev;

        netif_stop_queue(ndev);

        phy_disconnect(ndev->phydev);

        emac_wr(priv, MAC_INTERRUPT_ENABLE, 0x0);
        emac_wr(priv, DMA_INTERRUPT_ENABLE, 0x0);

        free_irq(priv->irq, ndev);

        napi_disable(&priv->napi);

        timer_delete_sync(&priv->txtimer);
        cancel_work_sync(&priv->tx_timeout_task);

        timer_delete_sync(&priv->stats_timer);

        emac_reset_hw(priv);

        /* Update and save current stats, see emac_stats_update() for usage */

        spin_lock_bh(&priv->stats_lock);

        emac_stats_update(priv);

        priv->tx_stats_off = priv->tx_stats;
        priv->rx_stats_off = priv->rx_stats;

        spin_unlock_bh(&priv->stats_lock);

        pm_runtime_put_sync(&pdev->dev);
        return 0;
}

/* Called when net interface is brought up. */
static int emac_open(struct net_device *ndev)
{
        struct emac_priv *priv = netdev_priv(ndev);
        struct device *dev = &priv->pdev->dev;
        int ret;

        ret = emac_alloc_tx_resources(priv);
        if (ret) {
                dev_err(dev, "Cannot allocate TX resources\n");
                return ret;
        }

        ret = emac_alloc_rx_resources(priv);
        if (ret) {
                dev_err(dev, "Cannot allocate RX resources\n");
                goto err_free_tx;
        }

        ret = emac_up(priv);
        if (ret) {
                dev_err(dev, "Error when bringing interface up\n");
                goto err_free_rx;
        }
        return 0;

err_free_rx:
        emac_free_rx_resources(priv);
err_free_tx:
        emac_free_tx_resources(priv);

        return ret;
}

/* Called when interface is brought down. */
static int emac_stop(struct net_device *ndev)
{
        struct emac_priv *priv = netdev_priv(ndev);

        emac_down(priv);
        emac_free_tx_resources(priv);
        emac_free_rx_resources(priv);

        return 0;
}

static const struct ethtool_ops emac_ethtool_ops = {
        .get_link_ksettings     = phy_ethtool_get_link_ksettings,
        .set_link_ksettings     = phy_ethtool_set_link_ksettings,
        .nway_reset             = phy_ethtool_nway_reset,
        .get_drvinfo            = emac_get_drvinfo,
        .get_link               = ethtool_op_get_link,

        .get_regs               = emac_ethtool_get_regs,
        .get_regs_len           = emac_ethtool_get_regs_len,

        .get_rmon_stats         = emac_get_rmon_stats,
        .get_pause_stats        = emac_get_pause_stats,
        .get_eth_mac_stats      = emac_get_eth_mac_stats,

        .get_sset_count         = emac_get_sset_count,
        .get_strings            = emac_get_strings,
        .get_ethtool_stats      = emac_get_ethtool_stats,
};

static const struct net_device_ops emac_netdev_ops = {
        .ndo_open               = emac_open,
        .ndo_stop               = emac_stop,
        .ndo_start_xmit         = emac_start_xmit,
        .ndo_validate_addr      = eth_validate_addr,
        .ndo_set_mac_address    = emac_set_mac_address,
        .ndo_eth_ioctl          = phy_do_ioctl_running,
        .ndo_change_mtu         = emac_change_mtu,
        .ndo_tx_timeout         = emac_tx_timeout,
        .ndo_set_rx_mode        = emac_set_rx_mode,
        .ndo_get_stats64        = emac_get_stats64,
};

/* Currently we always use 15.6 ps/step for the delay line */

static u32 delay_ps_to_unit(u32 ps)
{
        return DIV_ROUND_CLOSEST(ps * 10, 156);
}

static u32 delay_unit_to_ps(u32 unit)
{
        return DIV_ROUND_CLOSEST(unit * 156, 10);
}

#define EMAC_MAX_DELAY_UNIT     FIELD_MAX(EMAC_TX_DLINE_CODE_MASK)

/* Minus one just to be safe from rounding errors */
#define EMAC_MAX_DELAY_PS       (delay_unit_to_ps(EMAC_MAX_DELAY_UNIT - 1))

static int emac_config_dt(struct platform_device *pdev, struct emac_priv *priv)
{
        struct device_node *np = pdev->dev.of_node;
        struct device *dev = &pdev->dev;
        u8 mac_addr[ETH_ALEN] = { 0 };
        int ret;

        priv->iobase = devm_platform_ioremap_resource(pdev, 0);
        if (IS_ERR(priv->iobase))
                return dev_err_probe(dev, PTR_ERR(priv->iobase),
                                     "ioremap failed\n");

        priv->regmap_apmu =
                syscon_regmap_lookup_by_phandle_args(np, "spacemit,apmu", 1,
                                                     &priv->regmap_apmu_offset);

        if (IS_ERR(priv->regmap_apmu))
                return dev_err_probe(dev, PTR_ERR(priv->regmap_apmu),
                                     "failed to get syscon\n");

        priv->irq = platform_get_irq(pdev, 0);
        if (priv->irq < 0)
                return priv->irq;

        ret = of_get_mac_address(np, mac_addr);
        if (ret) {
                if (ret == -EPROBE_DEFER)
                        return dev_err_probe(dev, ret,
                                             "Can't get MAC address\n");

                dev_info(&pdev->dev, "Using random MAC address\n");
                eth_hw_addr_random(priv->ndev);
        } else {
                eth_hw_addr_set(priv->ndev, mac_addr);
        }

        priv->tx_delay = 0;
        priv->rx_delay = 0;

        of_property_read_u32(np, "tx-internal-delay-ps", &priv->tx_delay);
        of_property_read_u32(np, "rx-internal-delay-ps", &priv->rx_delay);

        if (priv->tx_delay > EMAC_MAX_DELAY_PS) {
                dev_err(&pdev->dev,
                        "tx-internal-delay-ps too large: max %d, got %d",
                        EMAC_MAX_DELAY_PS, priv->tx_delay);
                return -EINVAL;
        }

        if (priv->rx_delay > EMAC_MAX_DELAY_PS) {
                dev_err(&pdev->dev,
                        "rx-internal-delay-ps too large: max %d, got %d",
                        EMAC_MAX_DELAY_PS, priv->rx_delay);
                return -EINVAL;
        }

        priv->tx_delay = delay_ps_to_unit(priv->tx_delay);
        priv->rx_delay = delay_ps_to_unit(priv->rx_delay);

        return 0;
}

static void emac_phy_deregister_fixed_link(void *data)
{
        struct device_node *of_node = data;

        of_phy_deregister_fixed_link(of_node);
}

static int emac_probe(struct platform_device *pdev)
{
        struct device *dev = &pdev->dev;
        struct reset_control *reset;
        struct net_device *ndev;
        struct emac_priv *priv;
        int ret;

        ndev = devm_alloc_etherdev(dev, sizeof(struct emac_priv));
        if (!ndev)
                return -ENOMEM;

        ndev->hw_features = NETIF_F_SG;
        ndev->features |= ndev->hw_features;

        ndev->max_mtu = EMAC_RX_BUF_MAX - (ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN);
        ndev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS;

        priv = netdev_priv(ndev);
        priv->ndev = ndev;
        priv->pdev = pdev;
        platform_set_drvdata(pdev, priv);

        ret = emac_config_dt(pdev, priv);
        if (ret < 0)
                return dev_err_probe(dev, ret, "Configuration failed\n");

        ndev->watchdog_timeo = 5 * HZ;
        ndev->base_addr = (unsigned long)priv->iobase;
        ndev->irq = priv->irq;

        ndev->ethtool_ops = &emac_ethtool_ops;
        ndev->netdev_ops = &emac_netdev_ops;

        devm_pm_runtime_enable(&pdev->dev);

        priv->bus_clk = devm_clk_get_enabled(&pdev->dev, NULL);
        if (IS_ERR(priv->bus_clk))
                return dev_err_probe(dev, PTR_ERR(priv->bus_clk),
                                     "Failed to get clock\n");

        reset = devm_reset_control_get_optional_exclusive_deasserted(&pdev->dev,
                                                                     NULL);
        if (IS_ERR(reset))
                return dev_err_probe(dev, PTR_ERR(reset),
                                     "Failed to get reset\n");

        if (of_phy_is_fixed_link(dev->of_node)) {
                ret = of_phy_register_fixed_link(dev->of_node);
                if (ret)
                        return dev_err_probe(dev, ret,
                                             "Failed to register fixed-link\n");

                ret = devm_add_action_or_reset(dev,
                                               emac_phy_deregister_fixed_link,
                                               dev->of_node);

                if (ret) {
                        dev_err(dev, "devm_add_action_or_reset failed\n");
                        return ret;
                }
        }

        emac_sw_init(priv);

        ret = emac_mdio_init(priv);
        if (ret)
                goto err_timer_delete;

        SET_NETDEV_DEV(ndev, &pdev->dev);

        ret = devm_register_netdev(dev, ndev);
        if (ret) {
                dev_err(dev, "devm_register_netdev failed\n");
                goto err_timer_delete;
        }

        netif_napi_add(ndev, &priv->napi, emac_rx_poll);
        netif_carrier_off(ndev);

        return 0;

err_timer_delete:
        timer_delete_sync(&priv->txtimer);
        timer_delete_sync(&priv->stats_timer);

        return ret;
}

static void emac_remove(struct platform_device *pdev)
{
        struct emac_priv *priv = platform_get_drvdata(pdev);

        timer_shutdown_sync(&priv->txtimer);
        cancel_work_sync(&priv->tx_timeout_task);

        timer_shutdown_sync(&priv->stats_timer);

        emac_reset_hw(priv);
}

static int emac_resume(struct device *dev)
{
        struct emac_priv *priv = dev_get_drvdata(dev);
        struct net_device *ndev = priv->ndev;
        int ret;

        ret = clk_prepare_enable(priv->bus_clk);
        if (ret < 0) {
                dev_err(dev, "Failed to enable bus clock: %d\n", ret);
                return ret;
        }

        if (!netif_running(ndev))
                return 0;

        ret = emac_open(ndev);
        if (ret) {
                clk_disable_unprepare(priv->bus_clk);
                return ret;
        }

        netif_device_attach(ndev);

        mod_timer(&priv->stats_timer, jiffies);

        return 0;
}

static int emac_suspend(struct device *dev)
{
        struct emac_priv *priv = dev_get_drvdata(dev);
        struct net_device *ndev = priv->ndev;

        if (!ndev || !netif_running(ndev)) {
                clk_disable_unprepare(priv->bus_clk);
                return 0;
        }

        emac_stop(ndev);

        clk_disable_unprepare(priv->bus_clk);
        netif_device_detach(ndev);
        return 0;
}

static const struct dev_pm_ops emac_pm_ops = {
        SYSTEM_SLEEP_PM_OPS(emac_suspend, emac_resume)
};

static const struct of_device_id emac_of_match[] = {
        { .compatible = "spacemit,k1-emac" },
        { /* sentinel */ },
};
MODULE_DEVICE_TABLE(of, emac_of_match);

static struct platform_driver emac_driver = {
        .probe = emac_probe,
        .remove = emac_remove,
        .driver = {
                .name = DRIVER_NAME,
                .of_match_table = of_match_ptr(emac_of_match),
                .pm = &emac_pm_ops,
        },
};
module_platform_driver(emac_driver);

MODULE_DESCRIPTION("SpacemiT K1 Ethernet driver");
MODULE_AUTHOR("Vivian Wang <wangruikang@iscas.ac.cn>");
MODULE_LICENSE("GPL");