/*-
 * Copyright (c) 2020 Michael J Karels
 * Copyright (c) 2016, 2020 Jared McNeill <jmcneill@invisible.ca>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * RPi4 (BCM 2711) Gigabit Ethernet ("GENET") controller
 *
 * This driver is derived in large part from bcmgenet.c from NetBSD by
 * Jared McNeill.  Parts of the structure and other common code in
 * this driver have been copied from if_awg.c for the Allwinner EMAC,
 * also by Jared McNeill.
 */

#include "opt_device_polling.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <sys/kernel.h>
#include <sys/endian.h>
#include <sys/mbuf.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/sysctl.h>
#include <sys/module.h>
#include <sys/taskqueue.h>
#include <sys/gpio.h>

#include <net/bpf.h>
#include <net/if.h>
#include <net/ethernet.h>
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/if_types.h>
#include <net/if_var.h>

#include <machine/bus.h>

#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>

#define __BIT(_x)       (1 << (_x))
#include "if_genetreg.h"

#include <dev/mii/mii.h>
#include <dev/mii/miivar.h>
#include <dev/mii/mii_fdt.h>

#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/ip6.h>

#include "syscon_if.h"
#include "miibus_if.h"
#include "gpio_if.h"

#define RD4(sc, reg)            bus_read_4((sc)->res[_RES_MAC], (reg))
#define WR4(sc, reg, val)       bus_write_4((sc)->res[_RES_MAC], (reg), (val))

#define GEN_LOCK(sc)            mtx_lock(&(sc)->mtx)
#define GEN_UNLOCK(sc)          mtx_unlock(&(sc)->mtx)
#define GEN_ASSERT_LOCKED(sc)   mtx_assert(&(sc)->mtx, MA_OWNED)
#define GEN_ASSERT_UNLOCKED(sc) mtx_assert(&(sc)->mtx, MA_NOTOWNED)

#define TX_DESC_COUNT           GENET_DMA_DESC_COUNT
#define RX_DESC_COUNT           GENET_DMA_DESC_COUNT

#define TX_NEXT(n, count)               (((n) + 1) & ((count) - 1))
#define RX_NEXT(n, count)               (((n) + 1) & ((count) - 1))

#define TX_MAX_SEGS             20

static SYSCTL_NODE(_hw, OID_AUTO, genet, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
    "genet driver parameters");

/* Maximum number of mbufs to pass per call to if_input */
static int gen_rx_batch = 16 /* RX_BATCH_DEFAULT */;
SYSCTL_INT(_hw_genet, OID_AUTO, rx_batch, CTLFLAG_RDTUN,
    &gen_rx_batch, 0, "max mbufs per call to if_input");

TUNABLE_INT("hw.gen.rx_batch", &gen_rx_batch);  /* old name/interface */

/*
 * Transmitting packets with only an Ethernet header in the first mbuf
 * fails.  Examples include reflected ICMPv6 packets, e.g. echo replies;
 * forwarded IPv6/TCP packets; and forwarded IPv4/TCP packets that use NAT
 * with IPFW.  Pulling up the sizes of ether_header + ip6_hdr + icmp6_hdr
 * seems to work for both ICMPv6 and TCP over IPv6, as well as the IPv4/TCP
 * case.
 */
static int gen_tx_hdr_min = 56;         /* ether_header + ip6_hdr + icmp6_hdr */
SYSCTL_INT(_hw_genet, OID_AUTO, tx_hdr_min, CTLFLAG_RW,
    &gen_tx_hdr_min, 0, "header to add to packets with ether header only");

static struct ofw_compat_data compat_data[] = {
        { "brcm,genet-v1",              1 },
        { "brcm,genet-v2",              2 },
        { "brcm,genet-v3",              3 },
        { "brcm,genet-v4",              4 },
        { "brcm,genet-v5",              5 },
        { "brcm,bcm2711-genet-v5",      5 },
        { NULL,                         0 }
};

enum {
        _RES_MAC,               /* what to call this? */
        _RES_IRQ1,
        _RES_IRQ2,
        _RES_NITEMS
};

static struct resource_spec gen_spec[] = {
        { SYS_RES_MEMORY,       0,      RF_ACTIVE },
        { SYS_RES_IRQ,          0,      RF_ACTIVE },
        { SYS_RES_IRQ,          1,      RF_ACTIVE },
        { -1, 0 }
};

/* structure per ring entry */
struct gen_ring_ent {
        bus_dmamap_t            map;
        struct mbuf             *mbuf;
};

struct tx_queue {
        int                     hwindex;                /* hardware index */
        int                     nentries;
        u_int                   queued;                 /* or avail? */
        u_int                   cur;
        u_int                   next;
        u_int                   prod_idx;
        u_int                   cons_idx;
        struct gen_ring_ent     *entries;
};

struct rx_queue {
        int                     hwindex;                /* hardware index */
        int                     nentries;
        u_int                   cur;
        u_int                   prod_idx;
        u_int                   cons_idx;
        struct gen_ring_ent     *entries;
};

struct gen_softc {
        struct resource         *res[_RES_NITEMS];
        struct mtx              mtx;
        if_t                    ifp;
        device_t                dev;
        device_t                miibus;
        mii_contype_t           phy_mode;

        struct callout          stat_ch;
        struct task             link_task;
        void                    *ih;
        void                    *ih2;
        int                     type;
        int                     if_flags;
        int                     link;
        bus_dma_tag_t           tx_buf_tag;
        /*
         * The genet chip has multiple queues for transmit and receive.
         * This driver uses only one (queue 16, the default), but is cast
         * with multiple rings.  The additional rings are used for different
         * priorities.
         */
#define DEF_TXQUEUE     0
#define NTXQUEUE        1
        struct tx_queue         tx_queue[NTXQUEUE];
        struct gen_ring_ent     tx_ring_ent[TX_DESC_COUNT];  /* ring entries */

        bus_dma_tag_t           rx_buf_tag;
#define DEF_RXQUEUE     0
#define NRXQUEUE        1
        struct rx_queue         rx_queue[NRXQUEUE];
        struct gen_ring_ent     rx_ring_ent[RX_DESC_COUNT];  /* ring entries */
};

static void gen_init(void *softc);
static void gen_start(if_t ifp);
static void gen_stop(struct gen_softc *sc);
static void gen_destroy(struct gen_softc *sc);
static int gen_encap(struct gen_softc *sc, struct mbuf **mp);
static int gen_parse_tx(struct mbuf *m, int csum_flags);
static int gen_ioctl(if_t ifp, u_long cmd, caddr_t data);
static int gen_get_phy_mode(device_t dev);
static bool gen_get_eaddr(device_t dev, struct ether_addr *eaddr);
static void gen_set_enaddr(struct gen_softc *sc);
static void gen_setup_rxfilter(struct gen_softc *sc);
static void gen_reset(struct gen_softc *sc);
static void gen_enable(struct gen_softc *sc);
static void gen_dma_disable(struct gen_softc *sc);
static int gen_bus_dma_init(struct gen_softc *sc);
static void gen_bus_dma_teardown(struct gen_softc *sc);
static void gen_enable_intr(struct gen_softc *sc);
static void gen_init_txrings(struct gen_softc *sc);
static void gen_init_rxrings(struct gen_softc *sc);
static void gen_intr(void *softc);
static int gen_rxintr(struct gen_softc *sc, struct rx_queue *q);
static void gen_txintr(struct gen_softc *sc, struct tx_queue *q);
static void gen_intr2(void *softc);
static int gen_newbuf_rx(struct gen_softc *sc, struct rx_queue *q, int index);
static int gen_mapbuf_rx(struct gen_softc *sc, struct rx_queue *q, int index,
    struct mbuf *m);
static void gen_link_task(void *arg, int pending);
static void gen_media_status(if_t ifp, struct ifmediareq *ifmr);
static int gen_media_change(if_t ifp);
static void gen_tick(void *softc);

static int
gen_probe(device_t dev)
{
        if (!ofw_bus_status_okay(dev))
                return (ENXIO);

        if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
                return (ENXIO);

        device_set_desc(dev, "RPi4 Gigabit Ethernet");
        return (BUS_PROBE_DEFAULT);
}

static int
gen_attach(device_t dev)
{
        struct ether_addr eaddr;
        struct gen_softc *sc;
        int major, minor, error, mii_flags;
        bool eaddr_found;

        sc = device_get_softc(dev);
        sc->dev = dev;
        sc->type = ofw_bus_search_compatible(dev, compat_data)->ocd_data;

        if (bus_alloc_resources(dev, gen_spec, sc->res) != 0) {
                device_printf(dev, "cannot allocate resources for device\n");
                error = ENXIO;
                goto fail;
        }

        major = (RD4(sc, GENET_SYS_REV_CTRL) & REV_MAJOR) >> REV_MAJOR_SHIFT;
        if (major != REV_MAJOR_V5) {
                device_printf(dev, "version %d is not supported\n", major);
                error = ENXIO;
                goto fail;
        }
        minor = (RD4(sc, GENET_SYS_REV_CTRL) & REV_MINOR) >> REV_MINOR_SHIFT;
        device_printf(dev, "GENET version 5.%d phy 0x%04x\n", minor,
                RD4(sc, GENET_SYS_REV_CTRL) & REV_PHY);

        mtx_init(&sc->mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK, MTX_DEF);
        callout_init_mtx(&sc->stat_ch, &sc->mtx, 0);
        TASK_INIT(&sc->link_task, 0, gen_link_task, sc);

        error = gen_get_phy_mode(dev);
        if (error != 0)
                goto fail;

        bzero(&eaddr, sizeof(eaddr));
        eaddr_found = gen_get_eaddr(dev, &eaddr);

        /* reset core */
        gen_reset(sc);

        gen_dma_disable(sc);

        /* Setup DMA */
        error = gen_bus_dma_init(sc);
        if (error != 0) {
                device_printf(dev, "cannot setup bus dma\n");
                goto fail;
        }

        /* Setup ethernet interface */
        sc->ifp = if_alloc(IFT_ETHER);
        if_setsoftc(sc->ifp, sc);
        if_initname(sc->ifp, device_get_name(dev), device_get_unit(dev));
        if_setflags(sc->ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
        if_setstartfn(sc->ifp, gen_start);
        if_setioctlfn(sc->ifp, gen_ioctl);
        if_setinitfn(sc->ifp, gen_init);
        if_setsendqlen(sc->ifp, TX_DESC_COUNT - 1);
        if_setsendqready(sc->ifp);
#define GEN_CSUM_FEATURES       (CSUM_UDP | CSUM_TCP)
        if_sethwassist(sc->ifp, GEN_CSUM_FEATURES);
        if_setcapabilities(sc->ifp, IFCAP_VLAN_MTU | IFCAP_HWCSUM |
            IFCAP_HWCSUM_IPV6);
        if_setcapenable(sc->ifp, if_getcapabilities(sc->ifp));

        /* Install interrupt handlers */
        error = bus_setup_intr(dev, sc->res[_RES_IRQ1],
            INTR_TYPE_NET | INTR_MPSAFE, NULL, gen_intr, sc, &sc->ih);
        if (error != 0) {
                device_printf(dev, "cannot setup interrupt handler1\n");
                goto fail;
        }

        error = bus_setup_intr(dev, sc->res[_RES_IRQ2],
            INTR_TYPE_NET | INTR_MPSAFE, NULL, gen_intr2, sc, &sc->ih2);
        if (error != 0) {
                device_printf(dev, "cannot setup interrupt handler2\n");
                goto fail;
        }

        /* Attach MII driver */
        mii_flags = 0;
        switch (sc->phy_mode)
        {
        case MII_CONTYPE_RGMII_ID:
                mii_flags |= MIIF_RX_DELAY | MIIF_TX_DELAY;
                break;
        case MII_CONTYPE_RGMII_RXID:
                mii_flags |= MIIF_RX_DELAY;
                break;
        case MII_CONTYPE_RGMII_TXID:
                mii_flags |= MIIF_TX_DELAY;
                break;
        default:
                break;
        }
        error = mii_attach(dev, &sc->miibus, sc->ifp, gen_media_change,
            gen_media_status, BMSR_DEFCAPMASK, MII_PHY_ANY, MII_OFFSET_ANY,
            mii_flags);
        if (error != 0) {
                device_printf(dev, "cannot attach PHY\n");
                goto fail;
        }

        /* If address was not found, create one based on the hostid and name. */
        if (!eaddr_found)
                ether_gen_addr(sc->ifp, &eaddr);
        /* Attach ethernet interface */
        ether_ifattach(sc->ifp, eaddr.octet);

fail:
        if (error)
                gen_destroy(sc);
        return (error);
}

/* Free resources after failed attach.  This is not a complete detach. */
static void
gen_destroy(struct gen_softc *sc)
{

        device_delete_children(sc->dev);
        bus_teardown_intr(sc->dev, sc->res[_RES_IRQ1], sc->ih);
        bus_teardown_intr(sc->dev, sc->res[_RES_IRQ2], sc->ih2);
        gen_bus_dma_teardown(sc);
        callout_drain(&sc->stat_ch);
        if (mtx_initialized(&sc->mtx))
                mtx_destroy(&sc->mtx);
        bus_release_resources(sc->dev, gen_spec, sc->res);
        if (sc->ifp != NULL) {
                if_free(sc->ifp);
                sc->ifp = NULL;
        }
}

static int
gen_detach(device_t dev)
{
        struct gen_softc *sc;
        int error;

        sc = device_get_softc(dev);

        GEN_LOCK(sc);
        gen_stop(sc);
        GEN_UNLOCK(sc);
        callout_drain(&sc->stat_ch);
        ether_ifdetach(sc->ifp);

        /* Detach the miibus */
        error = bus_generic_detach(dev);
        if (error != 0)
                return (error);

        /* clean up dma */
        gen_bus_dma_teardown(sc);

        /* Release bus resources. */
        bus_teardown_intr(sc->dev, sc->res[_RES_IRQ1], sc->ih);
        bus_teardown_intr(sc->dev, sc->res[_RES_IRQ2], sc->ih2);
        bus_release_resources(sc->dev, gen_spec, sc->res);

        if (sc->ifp != NULL)
                if_free(sc->ifp);
        mtx_destroy(&sc->mtx);
        return (0);
}

static int
gen_get_phy_mode(device_t dev)
{
        struct gen_softc *sc;
        phandle_t node;
        mii_contype_t type;
        int error = 0;

        sc = device_get_softc(dev);
        node = ofw_bus_get_node(dev);
        type = mii_fdt_get_contype(node);

        switch (type) {
        case MII_CONTYPE_RGMII:
        case MII_CONTYPE_RGMII_ID:
        case MII_CONTYPE_RGMII_RXID:
        case MII_CONTYPE_RGMII_TXID:
                sc->phy_mode = type;
                break;
        default:
                device_printf(dev, "unknown phy-mode '%s'\n",
                    mii_fdt_contype_to_name(type));
                error = ENXIO;
                break;
        }

        return (error);
}

static bool
gen_get_eaddr(device_t dev, struct ether_addr *eaddr)
{
        struct gen_softc *sc;
        uint32_t maclo, machi, val;
        phandle_t node;

        sc = device_get_softc(dev);

        node = ofw_bus_get_node(dev);
        if (OF_getprop(node, "mac-address", eaddr->octet,
            ETHER_ADDR_LEN) != -1 ||
            OF_getprop(node, "local-mac-address", eaddr->octet,
            ETHER_ADDR_LEN) != -1 ||
            OF_getprop(node, "address", eaddr->octet, ETHER_ADDR_LEN) != -1)
                return (true);

        device_printf(dev, "No Ethernet address found in fdt!\n");
        maclo = machi = 0;

        val = RD4(sc, GENET_SYS_RBUF_FLUSH_CTRL);
        if ((val & GENET_SYS_RBUF_FLUSH_RESET) == 0) {
                maclo = htobe32(RD4(sc, GENET_UMAC_MAC0));
                machi = htobe16(RD4(sc, GENET_UMAC_MAC1) & 0xffff);
        }

        if (maclo == 0 && machi == 0) {
                if (bootverbose)
                        device_printf(dev,
                            "No Ethernet address found in controller\n");
                return (false);
        } else {
                eaddr->octet[0] = maclo & 0xff;
                eaddr->octet[1] = (maclo >> 8) & 0xff;
                eaddr->octet[2] = (maclo >> 16) & 0xff;
                eaddr->octet[3] = (maclo >> 24) & 0xff;
                eaddr->octet[4] = machi & 0xff;
                eaddr->octet[5] = (machi >> 8) & 0xff;
                return (true);
        }
}

static void
gen_reset(struct gen_softc *sc)
{
        uint32_t val;

        val = RD4(sc, GENET_SYS_RBUF_FLUSH_CTRL);
        val |= GENET_SYS_RBUF_FLUSH_RESET;
        WR4(sc, GENET_SYS_RBUF_FLUSH_CTRL, val);
        DELAY(10);

        val &= ~GENET_SYS_RBUF_FLUSH_RESET;
        WR4(sc, GENET_SYS_RBUF_FLUSH_CTRL, val);
        DELAY(10);

        WR4(sc, GENET_SYS_RBUF_FLUSH_CTRL, 0);
        DELAY(10);

        WR4(sc, GENET_UMAC_CMD, 0);
        WR4(sc, GENET_UMAC_CMD,
            GENET_UMAC_CMD_LCL_LOOP_EN | GENET_UMAC_CMD_SW_RESET);
        DELAY(10);
        WR4(sc, GENET_UMAC_CMD, 0);

        WR4(sc, GENET_UMAC_MIB_CTRL, GENET_UMAC_MIB_RESET_RUNT |
            GENET_UMAC_MIB_RESET_RX | GENET_UMAC_MIB_RESET_TX);
        WR4(sc, GENET_UMAC_MIB_CTRL, 0);
}

static void
gen_enable(struct gen_softc *sc)
{
        u_int val;

        WR4(sc, GENET_UMAC_MAX_FRAME_LEN, 1536);

        val = RD4(sc, GENET_RBUF_CTRL);
        val |= GENET_RBUF_ALIGN_2B;
        WR4(sc, GENET_RBUF_CTRL, val);

        WR4(sc, GENET_RBUF_TBUF_SIZE_CTRL, 1);

        /* Enable transmitter and receiver */
        val = RD4(sc, GENET_UMAC_CMD);
        val |= GENET_UMAC_CMD_TXEN;
        val |= GENET_UMAC_CMD_RXEN;
        WR4(sc, GENET_UMAC_CMD, val);

        /* Enable interrupts */
        gen_enable_intr(sc);
        WR4(sc, GENET_INTRL2_CPU_CLEAR_MASK,
            GENET_IRQ_TXDMA_DONE | GENET_IRQ_RXDMA_DONE);
}

static void
gen_disable_intr(struct gen_softc *sc)
{
        /* Disable interrupts */
        WR4(sc, GENET_INTRL2_CPU_SET_MASK, 0xffffffff);
        WR4(sc, GENET_INTRL2_CPU_CLEAR_MASK, 0xffffffff);
}

static void
gen_disable(struct gen_softc *sc)
{
        uint32_t val;

        /* Stop receiver */
        val = RD4(sc, GENET_UMAC_CMD);
        val &= ~GENET_UMAC_CMD_RXEN;
        WR4(sc, GENET_UMAC_CMD, val);

        /* Stop transmitter */
        val = RD4(sc, GENET_UMAC_CMD);
        val &= ~GENET_UMAC_CMD_TXEN;
        WR4(sc, GENET_UMAC_CMD, val);

        /* Disable Interrupt */
        gen_disable_intr(sc);
}

static void
gen_enable_offload(struct gen_softc *sc)
{
        uint32_t check_ctrl, buf_ctrl;

        check_ctrl = RD4(sc, GENET_RBUF_CHECK_CTRL);
        buf_ctrl  = RD4(sc, GENET_RBUF_CTRL);
        if ((if_getcapenable(sc->ifp) & IFCAP_RXCSUM) != 0) {
                check_ctrl |= GENET_RBUF_CHECK_CTRL_EN;
                buf_ctrl |= GENET_RBUF_64B_EN;
        } else {
                check_ctrl &= ~GENET_RBUF_CHECK_CTRL_EN;
                buf_ctrl &= ~GENET_RBUF_64B_EN;
        }
        WR4(sc, GENET_RBUF_CHECK_CTRL, check_ctrl);
        WR4(sc, GENET_RBUF_CTRL, buf_ctrl);

        buf_ctrl  = RD4(sc, GENET_TBUF_CTRL);
        if ((if_getcapenable(sc->ifp) & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6)) !=
            0)
                buf_ctrl |= GENET_RBUF_64B_EN;
        else
                buf_ctrl &= ~GENET_RBUF_64B_EN;
        WR4(sc, GENET_TBUF_CTRL, buf_ctrl);
}

static void
gen_dma_disable(struct gen_softc *sc)
{
        int val;

        val = RD4(sc, GENET_TX_DMA_CTRL);
        val &= ~GENET_TX_DMA_CTRL_EN;
        val &= ~GENET_TX_DMA_CTRL_RBUF_EN(GENET_DMA_DEFAULT_QUEUE);
        WR4(sc, GENET_TX_DMA_CTRL, val);

        val = RD4(sc, GENET_RX_DMA_CTRL);
        val &= ~GENET_RX_DMA_CTRL_EN;
        val &= ~GENET_RX_DMA_CTRL_RBUF_EN(GENET_DMA_DEFAULT_QUEUE);
        WR4(sc, GENET_RX_DMA_CTRL, val);
}

static int
gen_bus_dma_init(struct gen_softc *sc)
{
        device_t dev = sc->dev;
        int i, error;

        error = bus_dma_tag_create(
            bus_get_dma_tag(dev),       /* Parent tag */
            4, 0,                       /* alignment, boundary */
            BUS_SPACE_MAXADDR_40BIT,    /* lowaddr */
            BUS_SPACE_MAXADDR,          /* highaddr */
            NULL, NULL,                 /* filter, filterarg */
            MCLBYTES, TX_MAX_SEGS,      /* maxsize, nsegs */
            MCLBYTES,                   /* maxsegsize */
            0,                          /* flags */
            NULL, NULL,                 /* lockfunc, lockarg */
            &sc->tx_buf_tag);
        if (error != 0) {
                device_printf(dev, "cannot create TX buffer tag\n");
                return (error);
        }

        for (i = 0; i < TX_DESC_COUNT; i++) {
                error = bus_dmamap_create(sc->tx_buf_tag, 0,
                    &sc->tx_ring_ent[i].map);
                if (error != 0) {
                        device_printf(dev, "cannot create TX buffer map\n");
                        return (error);
                }
        }

        error = bus_dma_tag_create(
            bus_get_dma_tag(dev),       /* Parent tag */
            4, 0,                       /* alignment, boundary */
            BUS_SPACE_MAXADDR_40BIT,    /* lowaddr */
            BUS_SPACE_MAXADDR,          /* highaddr */
            NULL, NULL,                 /* filter, filterarg */
            MCLBYTES, 1,                /* maxsize, nsegs */
            MCLBYTES,                   /* maxsegsize */
            0,                          /* flags */
            NULL, NULL,                 /* lockfunc, lockarg */
            &sc->rx_buf_tag);
        if (error != 0) {
                device_printf(dev, "cannot create RX buffer tag\n");
                return (error);
        }

        for (i = 0; i < RX_DESC_COUNT; i++) {
                error = bus_dmamap_create(sc->rx_buf_tag, 0,
                    &sc->rx_ring_ent[i].map);
                if (error != 0) {
                        device_printf(dev, "cannot create RX buffer map\n");
                        return (error);
                }
        }
        return (0);
}

static void
gen_bus_dma_teardown(struct gen_softc *sc)
{
        int i, error;

        if (sc->tx_buf_tag != NULL) {
                for (i = 0; i < TX_DESC_COUNT; i++) {
                        error = bus_dmamap_destroy(sc->tx_buf_tag,
                            sc->tx_ring_ent[i].map);
                        sc->tx_ring_ent[i].map = NULL;
                        if (error)
                                device_printf(sc->dev,
                                    "%s: bus_dmamap_destroy failed: %d\n",
                                    __func__, error);
                }
                error = bus_dma_tag_destroy(sc->tx_buf_tag);
                sc->tx_buf_tag = NULL;
                if (error)
                        device_printf(sc->dev,
                            "%s: bus_dma_tag_destroy failed: %d\n", __func__,
                            error);
        }

        if (sc->rx_buf_tag != NULL) {
                for (i = 0; i < RX_DESC_COUNT; i++) {
                        error = bus_dmamap_destroy(sc->rx_buf_tag,
                            sc->rx_ring_ent[i].map);
                        sc->rx_ring_ent[i].map = NULL;
                        if (error)
                                device_printf(sc->dev,
                                    "%s: bus_dmamap_destroy failed: %d\n",
                                    __func__, error);
                }
                error = bus_dma_tag_destroy(sc->rx_buf_tag);
                sc->rx_buf_tag = NULL;
                if (error)
                        device_printf(sc->dev,
                            "%s: bus_dma_tag_destroy failed: %d\n", __func__,
                            error);
        }
}

static void
gen_enable_intr(struct gen_softc *sc)
{

        WR4(sc, GENET_INTRL2_CPU_CLEAR_MASK,
            GENET_IRQ_TXDMA_DONE | GENET_IRQ_RXDMA_DONE);
}

/*
 * "queue" is the software queue index (0-4); "qid" is the hardware index
 * (0-16).  "base" is the starting index in the ring array.
 */
static void
gen_init_txring(struct gen_softc *sc, int queue, int qid, int base,
    int nentries)
{
        struct tx_queue *q;
        uint32_t val;

        q = &sc->tx_queue[queue];
        q->entries = &sc->tx_ring_ent[base];
        q->hwindex = qid;
        q->nentries = nentries;

        /* TX ring */

        q->queued = 0;
        q->cons_idx = q->prod_idx = 0;

        WR4(sc, GENET_TX_SCB_BURST_SIZE, 0x08);

        WR4(sc, GENET_TX_DMA_READ_PTR_LO(qid), 0);
        WR4(sc, GENET_TX_DMA_READ_PTR_HI(qid), 0);
        WR4(sc, GENET_TX_DMA_CONS_INDEX(qid), 0);
        WR4(sc, GENET_TX_DMA_PROD_INDEX(qid), 0);
        WR4(sc, GENET_TX_DMA_RING_BUF_SIZE(qid),
            (nentries << GENET_TX_DMA_RING_BUF_SIZE_DESC_SHIFT) |
            (MCLBYTES & GENET_TX_DMA_RING_BUF_SIZE_BUF_LEN_MASK));
        WR4(sc, GENET_TX_DMA_START_ADDR_LO(qid), 0);
        WR4(sc, GENET_TX_DMA_START_ADDR_HI(qid), 0);
        WR4(sc, GENET_TX_DMA_END_ADDR_LO(qid),
            TX_DESC_COUNT * GENET_DMA_DESC_SIZE / 4 - 1);
        WR4(sc, GENET_TX_DMA_END_ADDR_HI(qid), 0);
        WR4(sc, GENET_TX_DMA_MBUF_DONE_THRES(qid), 1);
        WR4(sc, GENET_TX_DMA_FLOW_PERIOD(qid), 0);
        WR4(sc, GENET_TX_DMA_WRITE_PTR_LO(qid), 0);
        WR4(sc, GENET_TX_DMA_WRITE_PTR_HI(qid), 0);

        WR4(sc, GENET_TX_DMA_RING_CFG, __BIT(qid));     /* enable */

        /* Enable transmit DMA */
        val = RD4(sc, GENET_TX_DMA_CTRL);
        val |= GENET_TX_DMA_CTRL_EN;
        val |= GENET_TX_DMA_CTRL_RBUF_EN(qid);
        WR4(sc, GENET_TX_DMA_CTRL, val);
}

/*
 * "queue" is the software queue index (0-4); "qid" is the hardware index
 * (0-16).  "base" is the starting index in the ring array.
 */
static void
gen_init_rxring(struct gen_softc *sc, int queue, int qid, int base,
    int nentries)
{
        struct rx_queue *q;
        uint32_t val;
        int i;

        q = &sc->rx_queue[queue];
        q->entries = &sc->rx_ring_ent[base];
        q->hwindex = qid;
        q->nentries = nentries;
        q->cons_idx = q->prod_idx = 0;

        WR4(sc, GENET_RX_SCB_BURST_SIZE, 0x08);

        WR4(sc, GENET_RX_DMA_WRITE_PTR_LO(qid), 0);
        WR4(sc, GENET_RX_DMA_WRITE_PTR_HI(qid), 0);
        WR4(sc, GENET_RX_DMA_PROD_INDEX(qid), 0);
        WR4(sc, GENET_RX_DMA_CONS_INDEX(qid), 0);
        WR4(sc, GENET_RX_DMA_RING_BUF_SIZE(qid),
            (nentries << GENET_RX_DMA_RING_BUF_SIZE_DESC_SHIFT) |
            (MCLBYTES & GENET_RX_DMA_RING_BUF_SIZE_BUF_LEN_MASK));
        WR4(sc, GENET_RX_DMA_START_ADDR_LO(qid), 0);
        WR4(sc, GENET_RX_DMA_START_ADDR_HI(qid), 0);
        WR4(sc, GENET_RX_DMA_END_ADDR_LO(qid),
            RX_DESC_COUNT * GENET_DMA_DESC_SIZE / 4 - 1);
        WR4(sc, GENET_RX_DMA_END_ADDR_HI(qid), 0);
        WR4(sc, GENET_RX_DMA_XON_XOFF_THRES(qid),
            (5 << GENET_RX_DMA_XON_XOFF_THRES_LO_SHIFT) | (RX_DESC_COUNT >> 4));
        WR4(sc, GENET_RX_DMA_READ_PTR_LO(qid), 0);
        WR4(sc, GENET_RX_DMA_READ_PTR_HI(qid), 0);

        WR4(sc, GENET_RX_DMA_RING_CFG, __BIT(qid));     /* enable */

        /* fill ring */
        for (i = 0; i < RX_DESC_COUNT; i++)
                gen_newbuf_rx(sc, &sc->rx_queue[DEF_RXQUEUE], i);

        /* Enable receive DMA */
        val = RD4(sc, GENET_RX_DMA_CTRL);
        val |= GENET_RX_DMA_CTRL_EN;
        val |= GENET_RX_DMA_CTRL_RBUF_EN(qid);
        WR4(sc, GENET_RX_DMA_CTRL, val);
}

static void
gen_init_txrings(struct gen_softc *sc)
{
        int base = 0;
#ifdef PRI_RINGS
        int i;

        /* init priority rings */
        for (i = 0; i < PRI_RINGS; i++) {
                gen_init_txring(sc, i, i, base, TX_DESC_PRICOUNT);
                sc->tx_queue[i].queue = i;
                base += TX_DESC_PRICOUNT;
                dma_ring_conf |= 1 << i;
                dma_control |= DMA_RENABLE(i);
        }
#endif

        /* init GENET_DMA_DEFAULT_QUEUE (16) */
        gen_init_txring(sc, DEF_TXQUEUE, GENET_DMA_DEFAULT_QUEUE, base,
            TX_DESC_COUNT);
        sc->tx_queue[DEF_TXQUEUE].hwindex = GENET_DMA_DEFAULT_QUEUE;
}

static void
gen_init_rxrings(struct gen_softc *sc)
{
        int base = 0;
#ifdef PRI_RINGS
        int i;

        /* init priority rings */
        for (i = 0; i < PRI_RINGS; i++) {
                gen_init_rxring(sc, i, i, base, TX_DESC_PRICOUNT);
                sc->rx_queue[i].queue = i;
                base += TX_DESC_PRICOUNT;
                dma_ring_conf |= 1 << i;
                dma_control |= DMA_RENABLE(i);
        }
#endif

        /* init GENET_DMA_DEFAULT_QUEUE (16) */
        gen_init_rxring(sc, DEF_RXQUEUE, GENET_DMA_DEFAULT_QUEUE, base,
            RX_DESC_COUNT);
        sc->rx_queue[DEF_RXQUEUE].hwindex = GENET_DMA_DEFAULT_QUEUE;

}

static void
gen_stop(struct gen_softc *sc)
{
        int i;
        struct gen_ring_ent *ent;

        GEN_ASSERT_LOCKED(sc);

        callout_stop(&sc->stat_ch);
        if_setdrvflagbits(sc->ifp, 0, IFF_DRV_RUNNING);
        gen_reset(sc);
        gen_disable(sc);
        gen_dma_disable(sc);

        /* Clear the tx/rx ring buffer */
        for (i = 0; i < TX_DESC_COUNT; i++) {
                ent = &sc->tx_ring_ent[i];
                if (ent->mbuf != NULL) {
                        bus_dmamap_sync(sc->tx_buf_tag, ent->map,
                                BUS_DMASYNC_POSTWRITE);
                        bus_dmamap_unload(sc->tx_buf_tag, ent->map);
                        m_freem(ent->mbuf);
                        ent->mbuf = NULL;
                }
        }

        for (i = 0; i < RX_DESC_COUNT; i++) {
                ent = &sc->rx_ring_ent[i];
                if (ent->mbuf != NULL) {
                        bus_dmamap_sync(sc->rx_buf_tag, ent->map,
                            BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
                        bus_dmamap_unload(sc->rx_buf_tag, ent->map);
                        m_freem(ent->mbuf);
                        ent->mbuf = NULL;
                }
        }
}

static void
gen_init_locked(struct gen_softc *sc)
{
        struct mii_data *mii;
        if_t ifp;

        mii = device_get_softc(sc->miibus);
        ifp = sc->ifp;

        GEN_ASSERT_LOCKED(sc);

        if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
                return;

        switch (sc->phy_mode)
        {
        case MII_CONTYPE_RGMII:
        case MII_CONTYPE_RGMII_ID:
        case MII_CONTYPE_RGMII_RXID:
        case MII_CONTYPE_RGMII_TXID:
                WR4(sc, GENET_SYS_PORT_CTRL, GENET_SYS_PORT_MODE_EXT_GPHY);
                break;
        default:
                WR4(sc, GENET_SYS_PORT_CTRL, 0);
        }

        gen_set_enaddr(sc);

        /* Setup RX filter */
        gen_setup_rxfilter(sc);

        gen_init_txrings(sc);
        gen_init_rxrings(sc);
        gen_enable(sc);
        gen_enable_offload(sc);

        if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);

        mii_mediachg(mii);
        callout_reset(&sc->stat_ch, hz, gen_tick, sc);
}

static void
gen_init(void *softc)
{
        struct gen_softc *sc;

        sc = softc;
        GEN_LOCK(sc);
        gen_init_locked(sc);
        GEN_UNLOCK(sc);
}

static uint8_t ether_broadcastaddr[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };

static void
gen_setup_rxfilter_mdf(struct gen_softc *sc, u_int n, const uint8_t *ea)
{
        uint32_t addr0 = (ea[0] << 8) | ea[1];
        uint32_t addr1 = (ea[2] << 24) | (ea[3] << 16) | (ea[4] << 8) | ea[5];

        WR4(sc, GENET_UMAC_MDF_ADDR0(n), addr0);
        WR4(sc, GENET_UMAC_MDF_ADDR1(n), addr1);
}

static u_int
gen_setup_multi(void *arg, struct sockaddr_dl *sdl, u_int count)
{
        struct gen_softc *sc = arg;

        /* "count + 2" to account for unicast and broadcast */
        gen_setup_rxfilter_mdf(sc, count + 2, LLADDR(sdl));
        return (1);             /* increment to count */
}

static void
gen_setup_rxfilter(struct gen_softc *sc)
{
        if_t ifp = sc->ifp;
        uint32_t cmd, mdf_ctrl;
        u_int n;

        GEN_ASSERT_LOCKED(sc);

        cmd = RD4(sc, GENET_UMAC_CMD);

        /*
         * Count the required number of hardware filters. We need one
         * for each multicast address, plus one for our own address and
         * the broadcast address.
         */
        n = if_llmaddr_count(ifp) + 2;

        if (n > GENET_MAX_MDF_FILTER)
                if_setflagbits(ifp, IFF_ALLMULTI, 0);
        else
                if_setflagbits(ifp, 0, IFF_ALLMULTI);

        if ((if_getflags(ifp) & (IFF_PROMISC|IFF_ALLMULTI)) != 0) {
                cmd |= GENET_UMAC_CMD_PROMISC;
                mdf_ctrl = 0;
        } else {
                cmd &= ~GENET_UMAC_CMD_PROMISC;
                gen_setup_rxfilter_mdf(sc, 0, ether_broadcastaddr);
                gen_setup_rxfilter_mdf(sc, 1, if_getlladdr(ifp));
                (void) if_foreach_llmaddr(ifp, gen_setup_multi, sc);
                mdf_ctrl = (__BIT(GENET_MAX_MDF_FILTER) - 1)  &~
                    (__BIT(GENET_MAX_MDF_FILTER - n) - 1);
        }

        WR4(sc, GENET_UMAC_CMD, cmd);
        WR4(sc, GENET_UMAC_MDF_CTRL, mdf_ctrl);
}

static void
gen_set_enaddr(struct gen_softc *sc)
{
        uint8_t *enaddr;
        uint32_t val;
        if_t ifp;

        GEN_ASSERT_LOCKED(sc);

        ifp = sc->ifp;

        /* Write our unicast address */
        enaddr = if_getlladdr(ifp);
        /* Write hardware address */
        val = enaddr[3] | (enaddr[2] << 8) | (enaddr[1] << 16) |
            (enaddr[0] << 24);
        WR4(sc, GENET_UMAC_MAC0, val);
        val = enaddr[5] | (enaddr[4] << 8);
        WR4(sc, GENET_UMAC_MAC1, val);
}

static void
gen_start_locked(struct gen_softc *sc)
{
        struct mbuf *m;
        if_t ifp;
        int err;

        GEN_ASSERT_LOCKED(sc);

        if (!sc->link)
                return;

        ifp = sc->ifp;

        if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
            IFF_DRV_RUNNING)
                return;

        while (true) {
                m = if_dequeue(ifp);
                if (m == NULL)
                        break;

                err = gen_encap(sc, &m);
                if (err != 0) {
                        if (err == ENOBUFS)
                                if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
                        else if (m == NULL)
                                if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
                        if (m != NULL)
                                if_sendq_prepend(ifp, m);
                        break;
                }
                bpf_mtap_if(ifp, m);
        }
}

static void
gen_start(if_t ifp)
{
        struct gen_softc *sc;

        sc = if_getsoftc(ifp);

        GEN_LOCK(sc);
        gen_start_locked(sc);
        GEN_UNLOCK(sc);
}

/* Test for any delayed checksum */
#define CSUM_DELAY_ANY  (CSUM_TCP | CSUM_UDP | CSUM_IP6_TCP | CSUM_IP6_UDP)

static int
gen_encap(struct gen_softc *sc, struct mbuf **mp)
{
        bus_dmamap_t map;
        bus_dma_segment_t segs[TX_MAX_SEGS];
        int error, nsegs, cur, first, i, index, offset;
        uint32_t csuminfo, length_status, csum_flags = 0, csumdata;
        struct mbuf *m;
        struct statusblock *sb = NULL;
        struct tx_queue *q;
        struct gen_ring_ent *ent;

        GEN_ASSERT_LOCKED(sc);

        q = &sc->tx_queue[DEF_TXQUEUE];
        if (q->queued == q->nentries) {
                /* tx_queue is full */
                return (ENOBUFS);
        }

        m = *mp;

        /*
         * Don't attempt to send packets with only an Ethernet header in
         * first mbuf; see comment above with gen_tx_hdr_min.
         */
        if (m->m_len == sizeof(struct ether_header)) {
                m = m_pullup(m, MIN(m->m_pkthdr.len, gen_tx_hdr_min));
                if (m == NULL) {
                        if (if_getflags(sc->ifp) & IFF_DEBUG)
                                device_printf(sc->dev,
                                    "header pullup fail\n");
                        *mp = NULL;
                        return (ENOMEM);
                }
        }

        if ((if_getcapenable(sc->ifp) & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6)) !=
            0) {
                csum_flags = m->m_pkthdr.csum_flags;
                csumdata = m->m_pkthdr.csum_data;
                M_PREPEND(m, sizeof(struct statusblock), M_NOWAIT);
                if (m == NULL) {
                        if (if_getflags(sc->ifp) & IFF_DEBUG)
                                device_printf(sc->dev, "prepend fail\n");
                        *mp = NULL;
                        return (ENOMEM);
                }
                offset = gen_parse_tx(m, csum_flags);
                sb = mtod(m, struct statusblock *);
                if ((csum_flags & CSUM_DELAY_ANY) != 0) {
                        csuminfo = (offset << TXCSUM_OFF_SHIFT) |
                            (offset + csumdata);
                        csuminfo |= TXCSUM_LEN_VALID;
                        if (csum_flags & (CSUM_UDP | CSUM_IP6_UDP))
                                csuminfo |= TXCSUM_UDP;
                        sb->txcsuminfo = csuminfo;
                } else
                        sb->txcsuminfo = 0;
        }

        *mp = m;

        cur = first = q->cur;
        ent = &q->entries[cur];
        map = ent->map;
        error = bus_dmamap_load_mbuf_sg(sc->tx_buf_tag, map, m, segs,
            &nsegs, BUS_DMA_NOWAIT);
        if (error == EFBIG) {
                m = m_collapse(m, M_NOWAIT, TX_MAX_SEGS);
                if (m == NULL) {
                        device_printf(sc->dev,
                            "gen_encap: m_collapse failed\n");
                        m_freem(*mp);
                        *mp = NULL;
                        return (ENOMEM);
                }
                *mp = m;
                error = bus_dmamap_load_mbuf_sg(sc->tx_buf_tag, map, m,
                    segs, &nsegs, BUS_DMA_NOWAIT);
                if (error != 0) {
                        m_freem(*mp);
                        *mp = NULL;
                }
        }
        if (error != 0) {
                device_printf(sc->dev,
                    "gen_encap: bus_dmamap_load_mbuf_sg failed\n");
                return (error);
        }
        if (nsegs == 0) {
                m_freem(*mp);
                *mp = NULL;
                return (EIO);
        }

        /* Remove statusblock after mapping, before possible requeue or bpf. */
        if (sb != NULL) {
                m->m_data += sizeof(struct statusblock);
                m->m_len -= sizeof(struct statusblock);
                m->m_pkthdr.len -= sizeof(struct statusblock);
        }
        if (q->queued + nsegs > q->nentries) {
                bus_dmamap_unload(sc->tx_buf_tag, map);
                return (ENOBUFS);
        }

        bus_dmamap_sync(sc->tx_buf_tag, map, BUS_DMASYNC_PREWRITE);

        index = q->prod_idx & (q->nentries - 1);
        for (i = 0; i < nsegs; i++) {
                ent = &q->entries[cur];
                length_status = GENET_TX_DESC_STATUS_QTAG_MASK;
                if (i == 0) {
                        length_status |= GENET_TX_DESC_STATUS_SOP |
                            GENET_TX_DESC_STATUS_CRC;
                        if ((csum_flags & CSUM_DELAY_ANY) != 0)
                                length_status |= GENET_TX_DESC_STATUS_CKSUM;
                }
                if (i == nsegs - 1)
                        length_status |= GENET_TX_DESC_STATUS_EOP;

                length_status |= segs[i].ds_len <<
                    GENET_TX_DESC_STATUS_BUFLEN_SHIFT;

                WR4(sc, GENET_TX_DESC_ADDRESS_LO(index),
                    (uint32_t)segs[i].ds_addr);
                WR4(sc, GENET_TX_DESC_ADDRESS_HI(index),
                    (uint32_t)(segs[i].ds_addr >> 32));
                WR4(sc, GENET_TX_DESC_STATUS(index), length_status);

                ++q->queued;
                cur = TX_NEXT(cur, q->nentries);
                index = TX_NEXT(index, q->nentries);
        }

        q->prod_idx += nsegs;
        q->prod_idx &= GENET_TX_DMA_PROD_CONS_MASK;
        /* We probably don't need to write the producer index on every iter */
        if (nsegs != 0)
                WR4(sc, GENET_TX_DMA_PROD_INDEX(q->hwindex), q->prod_idx);
        q->cur = cur;

        /* Store mbuf in the last segment */
        q->entries[first].mbuf = m;

        return (0);
}

/*
 * Parse a packet to find the offset of the transport header for checksum
 * offload.  Ensure that the link and network headers are contiguous with
 * the status block, or transmission fails.
 */
static int
gen_parse_tx(struct mbuf *m, int csum_flags)
{
        int offset, off_in_m;
        bool copy = false, shift = false;
        u_char *p, *copy_p = NULL;
        struct mbuf *m0 = m;
        uint16_t ether_type;

        if (m->m_len == sizeof(struct statusblock)) {
                /* M_PREPEND placed statusblock at end; move to beginning */
                m->m_data = m->m_pktdat;
                copy_p = mtodo(m, sizeof(struct statusblock));
                m = m->m_next;
                off_in_m = 0;
                p = mtod(m, u_char *);
                copy = true;
        } else {
                /*
                 * If statusblock is not at beginning of mbuf (likely),
                 * then remember to move mbuf contents down before copying
                 * after them.
                 */
                if ((m->m_flags & M_EXT) == 0 && m->m_data != m->m_pktdat)
                        shift = true;
                p = mtodo(m, sizeof(struct statusblock));
                off_in_m = sizeof(struct statusblock);
        }

/*
 * If headers need to be copied contiguous to statusblock, do so.
 * If copying to the internal mbuf data area, and the status block
 * is not at the beginning of that area, shift the status block (which
 * is empty) and following data.
 */
#define COPY(size) {                                                    \
        int hsize = size;                                               \
        if (copy) {                                                     \
                if (shift) {                                            \
                        u_char *p0;                                     \
                        shift = false;                                  \
                        p0 = mtodo(m0, sizeof(struct statusblock));     \
                        m0->m_data = m0->m_pktdat;                      \
                        bcopy(p0, mtodo(m0, sizeof(struct statusblock)),\
                            m0->m_len - sizeof(struct statusblock));    \
                        copy_p = mtodo(m0, m0->m_len);                  \
                }                                                       \
                bcopy(p, copy_p, hsize);                                \
                m0->m_len += hsize;                                     \
                m->m_len -= hsize;                                      \
                m->m_data += hsize;                                     \
        }                                                               \
        copy_p += hsize;                                                \
}

        KASSERT((sizeof(struct statusblock) + sizeof(struct ether_vlan_header) +
            sizeof(struct ip6_hdr) <= MLEN), ("%s: mbuf too small", __func__));

        if (((struct ether_header *)p)->ether_type == htons(ETHERTYPE_VLAN)) {
                offset = sizeof(struct ether_vlan_header);
                ether_type = ntohs(((struct ether_vlan_header *)p)->evl_proto);
                COPY(sizeof(struct ether_vlan_header));
                if (m->m_len == off_in_m + sizeof(struct ether_vlan_header)) {
                        m = m->m_next;
                        off_in_m = 0;
                        p = mtod(m, u_char *);
                        copy = true;
                } else {
                        off_in_m += sizeof(struct ether_vlan_header);
                        p += sizeof(struct ether_vlan_header);
                }
        } else {
                offset = sizeof(struct ether_header);
                ether_type = ntohs(((struct ether_header *)p)->ether_type);
                COPY(sizeof(struct ether_header));
                if (m->m_len == off_in_m + sizeof(struct ether_header)) {
                        m = m->m_next;
                        off_in_m = 0;
                        p = mtod(m, u_char *);
                        copy = true;
                } else {
                        off_in_m += sizeof(struct ether_header);
                        p += sizeof(struct ether_header);
                }
        }
        if (ether_type == ETHERTYPE_IP) {
                COPY(((struct ip *)p)->ip_hl << 2);
                offset += ((struct ip *)p)->ip_hl << 2;
        } else if (ether_type == ETHERTYPE_IPV6) {
                COPY(sizeof(struct ip6_hdr));
                offset += sizeof(struct ip6_hdr);
        } else {
                /*
                 * Unknown whether most other cases require moving a header;
                 * ARP works without.  However, Wake On LAN packets sent
                 * by wake(8) via BPF need something like this.
                 */
                COPY(MIN(gen_tx_hdr_min, m->m_len));
                offset += MIN(gen_tx_hdr_min, m->m_len);
        }
        return (offset);
#undef COPY
}

static void
gen_intr(void *arg)
{
        struct gen_softc *sc = arg;
        uint32_t val;

        GEN_LOCK(sc);

        val = RD4(sc, GENET_INTRL2_CPU_STAT);
        val &= ~RD4(sc, GENET_INTRL2_CPU_STAT_MASK);
        WR4(sc, GENET_INTRL2_CPU_CLEAR, val);

        if (val & GENET_IRQ_RXDMA_DONE)
                gen_rxintr(sc, &sc->rx_queue[DEF_RXQUEUE]);

        if (val & GENET_IRQ_TXDMA_DONE) {
                gen_txintr(sc, &sc->tx_queue[DEF_TXQUEUE]);
                if (!if_sendq_empty(sc->ifp))
                        gen_start_locked(sc);
        }

        GEN_UNLOCK(sc);
}

static int
gen_rxintr(struct gen_softc *sc, struct rx_queue *q)
{
        if_t ifp;
        struct mbuf *m, *mh, *mt;
        struct statusblock *sb = NULL;
        int error, index, len, cnt, npkt, n;
        uint32_t status, prod_idx, total;

        ifp = sc->ifp;
        mh = mt = NULL;
        cnt = 0;
        npkt = 0;

        prod_idx = RD4(sc, GENET_RX_DMA_PROD_INDEX(q->hwindex)) &
            GENET_RX_DMA_PROD_CONS_MASK;
        total = (prod_idx - q->cons_idx) & GENET_RX_DMA_PROD_CONS_MASK;

        index = q->cons_idx & (RX_DESC_COUNT - 1);
        for (n = 0; n < total; n++) {
                bus_dmamap_sync(sc->rx_buf_tag, q->entries[index].map,
                    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
                bus_dmamap_unload(sc->rx_buf_tag, q->entries[index].map);

                m = q->entries[index].mbuf;

                if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) {
                        sb = mtod(m, struct statusblock *);
                        status = sb->status_buflen;
                } else
                        status = RD4(sc, GENET_RX_DESC_STATUS(index));

                len = (status & GENET_RX_DESC_STATUS_BUFLEN_MASK) >>
                    GENET_RX_DESC_STATUS_BUFLEN_SHIFT;

                /* check for errors */
                if ((status &
                    (GENET_RX_DESC_STATUS_SOP | GENET_RX_DESC_STATUS_EOP |
                    GENET_RX_DESC_STATUS_RX_ERROR)) !=
                    (GENET_RX_DESC_STATUS_SOP | GENET_RX_DESC_STATUS_EOP)) {
                        if (if_getflags(ifp) & IFF_DEBUG)
                                device_printf(sc->dev,
                                    "error/frag %x csum %x\n", status,
                                    sb->rxcsum);
                        if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
                        continue;
                }

                error = gen_newbuf_rx(sc, q, index);
                if (error != 0) {
                        if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
                        if (if_getflags(ifp) & IFF_DEBUG)
                                device_printf(sc->dev, "gen_newbuf_rx %d\n",
                                    error);
                        /* reuse previous mbuf */
                        (void) gen_mapbuf_rx(sc, q, index, m);
                        continue;
                }

                if (sb != NULL) {
                        if (status & GENET_RX_DESC_STATUS_CKSUM_OK) {
                                /* L4 checksum checked; not sure about L3. */
                                m->m_pkthdr.csum_flags = CSUM_DATA_VALID |
                                    CSUM_PSEUDO_HDR;
                                m->m_pkthdr.csum_data = 0xffff;
                        }
                        m->m_data += sizeof(struct statusblock);
                        m->m_len -= sizeof(struct statusblock);
                        len -= sizeof(struct statusblock);
                }
                if (len > ETHER_ALIGN) {
                        m_adj(m, ETHER_ALIGN);
                        len -= ETHER_ALIGN;
                }

                m->m_pkthdr.rcvif = ifp;
                m->m_pkthdr.len = len;
                m->m_len = len;
                if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);

                m->m_nextpkt = NULL;
                if (mh == NULL)
                        mh = m;
                else
                        mt->m_nextpkt = m;
                mt = m;
                ++cnt;
                ++npkt;

                index = RX_NEXT(index, q->nentries);

                q->cons_idx = (q->cons_idx + 1) & GENET_RX_DMA_PROD_CONS_MASK;
                WR4(sc, GENET_RX_DMA_CONS_INDEX(q->hwindex), q->cons_idx);

                if (cnt == gen_rx_batch) {
                        GEN_UNLOCK(sc);
                        if_input(ifp, mh);
                        GEN_LOCK(sc);
                        mh = mt = NULL;
                        cnt = 0;
                }
        }

        if (mh != NULL) {
                GEN_UNLOCK(sc);
                if_input(ifp, mh);
                GEN_LOCK(sc);
        }

        return (npkt);
}

static void
gen_txintr(struct gen_softc *sc, struct tx_queue *q)
{
        uint32_t cons_idx, total;
        struct gen_ring_ent *ent;
        if_t ifp;
        int i, prog;

        GEN_ASSERT_LOCKED(sc);

        ifp = sc->ifp;

        cons_idx = RD4(sc, GENET_TX_DMA_CONS_INDEX(q->hwindex)) &
            GENET_TX_DMA_PROD_CONS_MASK;
        total = (cons_idx - q->cons_idx) & GENET_TX_DMA_PROD_CONS_MASK;

        prog = 0;
        for (i = q->next; q->queued > 0 && total > 0;
            i = TX_NEXT(i, q->nentries), total--) {
                /* XXX check for errors */

                ent = &q->entries[i];
                if (ent->mbuf != NULL) {
                        bus_dmamap_sync(sc->tx_buf_tag, ent->map,
                            BUS_DMASYNC_POSTWRITE);
                        bus_dmamap_unload(sc->tx_buf_tag, ent->map);
                        m_freem(ent->mbuf);
                        ent->mbuf = NULL;
                        if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
                }

                prog++;
                --q->queued;
        }

        if (prog > 0) {
                q->next = i;
                if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE);
        }

        q->cons_idx = cons_idx;
}

static void
gen_intr2(void *arg)
{
        struct gen_softc *sc = arg;

        device_printf(sc->dev, "gen_intr2\n");
}

static int
gen_newbuf_rx(struct gen_softc *sc, struct rx_queue *q, int index)
{
        struct mbuf *m;

        m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
        if (m == NULL)
                return (ENOBUFS);

        m->m_pkthdr.len = m->m_len = m->m_ext.ext_size;
        m_adj(m, ETHER_ALIGN);

        return (gen_mapbuf_rx(sc, q, index, m));
}

static int
gen_mapbuf_rx(struct gen_softc *sc, struct rx_queue *q, int index,
    struct mbuf *m)
{
        bus_dma_segment_t seg;
        bus_dmamap_t map;
        int nsegs;

        map = q->entries[index].map;
        if (bus_dmamap_load_mbuf_sg(sc->rx_buf_tag, map, m, &seg, &nsegs,
            BUS_DMA_NOWAIT) != 0) {
                m_freem(m);
                return (ENOBUFS);
        }

        bus_dmamap_sync(sc->rx_buf_tag, map, BUS_DMASYNC_PREREAD);

        q->entries[index].mbuf = m;
        WR4(sc, GENET_RX_DESC_ADDRESS_LO(index), (uint32_t)seg.ds_addr);
        WR4(sc, GENET_RX_DESC_ADDRESS_HI(index), (uint32_t)(seg.ds_addr >> 32));

        return (0);
}

static int
gen_ioctl(if_t ifp, u_long cmd, caddr_t data)
{
        struct gen_softc *sc;
        struct mii_data *mii;
        struct ifreq *ifr;
        int flags, enable, error;

        sc = if_getsoftc(ifp);
        mii = device_get_softc(sc->miibus);
        ifr = (struct ifreq *)data;
        error = 0;

        switch (cmd) {
        case SIOCSIFFLAGS:
                GEN_LOCK(sc);
                if (if_getflags(ifp) & IFF_UP) {
                        if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
                                flags = if_getflags(ifp) ^ sc->if_flags;
                                if ((flags & (IFF_PROMISC|IFF_ALLMULTI)) != 0)
                                        gen_setup_rxfilter(sc);
                        } else
                                gen_init_locked(sc);
                } else {
                        if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
                                gen_stop(sc);
                }
                sc->if_flags = if_getflags(ifp);
                GEN_UNLOCK(sc);
                break;

        case SIOCADDMULTI:
        case SIOCDELMULTI:
                if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
                        GEN_LOCK(sc);
                        gen_setup_rxfilter(sc);
                        GEN_UNLOCK(sc);
                }
                break;

        case SIOCSIFMEDIA:
        case SIOCGIFMEDIA:
                error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd);
                break;

        case SIOCSIFCAP:
                enable = if_getcapenable(ifp);
                flags = ifr->ifr_reqcap ^ enable;
                if (flags & IFCAP_RXCSUM)
                        enable ^= IFCAP_RXCSUM;
                if (flags & IFCAP_RXCSUM_IPV6)
                        enable ^= IFCAP_RXCSUM_IPV6;
                if (flags & IFCAP_TXCSUM)
                        enable ^= IFCAP_TXCSUM;
                if (flags & IFCAP_TXCSUM_IPV6)
                        enable ^= IFCAP_TXCSUM_IPV6;
                if (enable & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6))
                        if_sethwassist(ifp, GEN_CSUM_FEATURES);
                else
                        if_sethwassist(ifp, 0);
                if_setcapenable(ifp, enable);
                if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
                        gen_enable_offload(sc);
                break;

        default:
                error = ether_ioctl(ifp, cmd, data);
                break;
        }
        return (error);
}

static void
gen_tick(void *softc)
{
        struct gen_softc *sc;
        struct mii_data *mii;
        if_t ifp;
        int link;

        sc = softc;
        ifp = sc->ifp;
        mii = device_get_softc(sc->miibus);

        GEN_ASSERT_LOCKED(sc);

        if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
                return;

        link = sc->link;
        mii_tick(mii);
        if (sc->link && !link)
                gen_start_locked(sc);

        callout_reset(&sc->stat_ch, hz, gen_tick, sc);
}

#define MII_BUSY_RETRY          1000

static int
gen_miibus_readreg(device_t dev, int phy, int reg)
{
        struct gen_softc *sc;
        int retry, val;

        sc = device_get_softc(dev);
        val = 0;

        WR4(sc, GENET_MDIO_CMD, GENET_MDIO_READ |
            (phy << GENET_MDIO_ADDR_SHIFT) | (reg << GENET_MDIO_REG_SHIFT));
        val = RD4(sc, GENET_MDIO_CMD);
        WR4(sc, GENET_MDIO_CMD, val | GENET_MDIO_START_BUSY);
        for (retry = MII_BUSY_RETRY; retry > 0; retry--) {
                if (((val = RD4(sc, GENET_MDIO_CMD)) &
                    GENET_MDIO_START_BUSY) == 0) {
                        if (val & GENET_MDIO_READ_FAILED)
                                return (0);     /* -1? */
                        val &= GENET_MDIO_VAL_MASK;
                        break;
                }
                DELAY(10);
        }

        if (retry == 0)
                device_printf(dev, "phy read timeout, phy=%d reg=%d\n",
                    phy, reg);

        return (val);
}

static int
gen_miibus_writereg(device_t dev, int phy, int reg, int val)
{
        struct gen_softc *sc;
        int retry;

        sc = device_get_softc(dev);

        WR4(sc, GENET_MDIO_CMD, GENET_MDIO_WRITE |
            (phy << GENET_MDIO_ADDR_SHIFT) | (reg << GENET_MDIO_REG_SHIFT) |
            (val & GENET_MDIO_VAL_MASK));
        val = RD4(sc, GENET_MDIO_CMD);
        WR4(sc, GENET_MDIO_CMD, val | GENET_MDIO_START_BUSY);
        for (retry = MII_BUSY_RETRY; retry > 0; retry--) {
                val = RD4(sc, GENET_MDIO_CMD);
                if ((val & GENET_MDIO_START_BUSY) == 0)
                        break;
                DELAY(10);
        }
        if (retry == 0)
                device_printf(dev, "phy write timeout, phy=%d reg=%d\n",
                    phy, reg);

        return (0);
}

static void
gen_update_link_locked(struct gen_softc *sc)
{
        struct mii_data *mii;
        uint32_t val;
        u_int speed;

        GEN_ASSERT_LOCKED(sc);

        if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) == 0)
                return;
        mii = device_get_softc(sc->miibus);

        if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
            (IFM_ACTIVE | IFM_AVALID)) {
                switch (IFM_SUBTYPE(mii->mii_media_active)) {
                case IFM_1000_T:
                case IFM_1000_SX:
                        speed = GENET_UMAC_CMD_SPEED_1000;
                        sc->link = 1;
                        break;
                case IFM_100_TX:
                        speed = GENET_UMAC_CMD_SPEED_100;
                        sc->link = 1;
                        break;
                case IFM_10_T:
                        speed = GENET_UMAC_CMD_SPEED_10;
                        sc->link = 1;
                        break;
                default:
                        sc->link = 0;
                        break;
                }
        } else
                sc->link = 0;

        if (sc->link == 0)
                return;

        val = RD4(sc, GENET_EXT_RGMII_OOB_CTRL);
        val &= ~GENET_EXT_RGMII_OOB_OOB_DISABLE;
        val |= GENET_EXT_RGMII_OOB_RGMII_LINK;
        val |= GENET_EXT_RGMII_OOB_RGMII_MODE_EN;
        if (sc->phy_mode == MII_CONTYPE_RGMII)
                val |= GENET_EXT_RGMII_OOB_ID_MODE_DISABLE;
        else
                val &= ~GENET_EXT_RGMII_OOB_ID_MODE_DISABLE;
        WR4(sc, GENET_EXT_RGMII_OOB_CTRL, val);

        val = RD4(sc, GENET_UMAC_CMD);
        val &= ~GENET_UMAC_CMD_SPEED;
        val |= speed;
        WR4(sc, GENET_UMAC_CMD, val);
}

static void
gen_link_task(void *arg, int pending)
{
        struct gen_softc *sc;

        sc = arg;

        GEN_LOCK(sc);
        gen_update_link_locked(sc);
        GEN_UNLOCK(sc);
}

static void
gen_miibus_statchg(device_t dev)
{
        struct gen_softc *sc;

        sc = device_get_softc(dev);

        taskqueue_enqueue(taskqueue_swi, &sc->link_task);
}

static void
gen_media_status(if_t ifp, struct ifmediareq *ifmr)
{
        struct gen_softc *sc;
        struct mii_data *mii;

        sc = if_getsoftc(ifp);
        mii = device_get_softc(sc->miibus);

        GEN_LOCK(sc);
        mii_pollstat(mii);
        ifmr->ifm_active = mii->mii_media_active;
        ifmr->ifm_status = mii->mii_media_status;
        GEN_UNLOCK(sc);
}

static int
gen_media_change(if_t ifp)
{
        struct gen_softc *sc;
        struct mii_data *mii;
        int error;

        sc = if_getsoftc(ifp);
        mii = device_get_softc(sc->miibus);

        GEN_LOCK(sc);
        error = mii_mediachg(mii);
        GEN_UNLOCK(sc);

        return (error);
}

static device_method_t gen_methods[] = {
        /* Device interface */
        DEVMETHOD(device_probe,         gen_probe),
        DEVMETHOD(device_attach,        gen_attach),
        DEVMETHOD(device_detach,        gen_detach),

        /* MII interface */
        DEVMETHOD(miibus_readreg,       gen_miibus_readreg),
        DEVMETHOD(miibus_writereg,      gen_miibus_writereg),
        DEVMETHOD(miibus_statchg,       gen_miibus_statchg),

        DEVMETHOD_END
};

static driver_t gen_driver = {
        "genet",
        gen_methods,
        sizeof(struct gen_softc),
};

DRIVER_MODULE(genet, simplebus, gen_driver, 0, 0);
DRIVER_MODULE(miibus, genet, miibus_driver, 0, 0);
MODULE_DEPEND(genet, ether, 1, 1, 1);
MODULE_DEPEND(genet, miibus, 1, 1, 1);