/*      $NetBSD: if_dge.c,v 1.65 2024/07/05 04:31:51 rin Exp $ */

/*
 * Copyright (c) 2004, SUNET, Swedish University Computer Network.
 * All rights reserved.
 *
 * Written by Anders Magnusson for SUNET, Swedish University Computer Network.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed for the NetBSD Project by
 *      SUNET, Swedish University Computer Network.
 * 4. The name of SUNET may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY SUNET ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Copyright (c) 2001, 2002, 2003 Wasabi Systems, Inc.
 * All rights reserved.
 *
 * Written by Jason R. Thorpe for Wasabi Systems, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed for the NetBSD Project by
 *      Wasabi Systems, Inc.
 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
 *    or promote products derived from this software without specific prior
 *    written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Device driver for the Intel 82597EX Ten Gigabit Ethernet controller.
 *
 * TODO (in no specific order):
 *      HW VLAN support.
 *      TSE offloading (needs kernel changes...)
 *      RAIDC (receive interrupt delay adaptation)
 *      Use memory > 4GB.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: if_dge.c,v 1.65 2024/07/05 04:31:51 rin Exp $");



#include <sys/param.h>
#include <sys/systm.h>
#include <sys/callout.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <sys/errno.h>
#include <sys/device.h>
#include <sys/queue.h>
#include <sys/rndsource.h>

#include <net/if.h>
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/if_ether.h>
#include <net/bpf.h>

#include <netinet/in.h>                 /* XXX for struct ip */
#include <netinet/in_systm.h>           /* XXX for struct ip */
#include <netinet/ip.h>                 /* XXX for struct ip */
#include <netinet/tcp.h>                /* XXX for struct tcphdr */

#include <sys/bus.h>
#include <sys/intr.h>
#include <machine/endian.h>

#include <dev/mii/mii.h>
#include <dev/mii/miivar.h>
#include <dev/mii/mii_bitbang.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <dev/pci/pcidevs.h>

#include <dev/pci/if_dgereg.h>

/*
 * The receive engine may sometimes become off-by-one when writing back
 * chained descriptors.  Avoid this by allocating a large chunk of
 * memory and use if instead (to avoid chained descriptors).
 * This only happens with chained descriptors under heavy load.
 */
#define DGE_OFFBYONE_RXBUG

#define DGE_EVENT_COUNTERS
#define DGE_DEBUG

#ifdef DGE_DEBUG
#define DGE_DEBUG_LINK          0x01
#define DGE_DEBUG_TX            0x02
#define DGE_DEBUG_RX            0x04
#define DGE_DEBUG_CKSUM         0x08
int     dge_debug = 0;

#define DPRINTF(x, y)   if (dge_debug & (x)) printf y
#else
#define DPRINTF(x, y)   /* nothing */
#endif /* DGE_DEBUG */

/*
 * Transmit descriptor list size. We allow up to 100 DMA segments per
 * packet (Intel reports of jumbo frame packets with as
 * many as 80 DMA segments when using 16k buffers).
 */
#define DGE_NTXSEGS             100
#define DGE_IFQUEUELEN          20000
#define DGE_TXQUEUELEN          2048
#define DGE_TXQUEUELEN_MASK     (DGE_TXQUEUELEN - 1)
#define DGE_TXQUEUE_GC          (DGE_TXQUEUELEN / 8)
#define DGE_NTXDESC             1024
#define DGE_NTXDESC_MASK                (DGE_NTXDESC - 1)
#define DGE_NEXTTX(x)           (((x) + 1) & DGE_NTXDESC_MASK)
#define DGE_NEXTTXS(x)          (((x) + 1) & DGE_TXQUEUELEN_MASK)

/*
 * Receive descriptor list size.
 * Packet is of size MCLBYTES, and for jumbo packets buffers may
 * be chained.  Due to the nature of the card (high-speed), keep this
 * ring large. With 2k buffers the ring can store 400 jumbo packets,
 * which at full speed will be received in just under 3ms.
 */
#define DGE_NRXDESC             2048
#define DGE_NRXDESC_MASK        (DGE_NRXDESC - 1)
#define DGE_NEXTRX(x)           (((x) + 1) & DGE_NRXDESC_MASK)
/*
 * # of descriptors between head and written descriptors.
 * This is to work-around two erratas.
 */
#define DGE_RXSPACE             10
#define DGE_PREVRX(x)           (((x) - DGE_RXSPACE) & DGE_NRXDESC_MASK)
/*
 * Receive descriptor fetch thresholds. These are values recommended
 * by Intel, do not touch them unless you know what you are doing.
 */
#define RXDCTL_PTHRESH_VAL      128
#define RXDCTL_HTHRESH_VAL      16
#define RXDCTL_WTHRESH_VAL      16


/*
 * Tweakable parameters; default values.
 */
#define FCRTH   0x30000 /* Send XOFF water mark */
#define FCRTL   0x28000 /* Send XON water mark */
#define RDTR    0x20    /* Interrupt delay after receive, .8192us units */
#define TIDV    0x20    /* Interrupt delay after send, .8192us units */

/*
 * Control structures are DMA'd to the i82597 chip.  We allocate them in
 * a single clump that maps to a single DMA segment to make serveral things
 * easier.
 */
struct dge_control_data {
        /*
         * The transmit descriptors.
         */
        struct dge_tdes wcd_txdescs[DGE_NTXDESC];

        /*
         * The receive descriptors.
         */
        struct dge_rdes wcd_rxdescs[DGE_NRXDESC];
};

#define DGE_CDOFF(x)    offsetof(struct dge_control_data, x)
#define DGE_CDTXOFF(x)  DGE_CDOFF(wcd_txdescs[(x)])
#define DGE_CDRXOFF(x)  DGE_CDOFF(wcd_rxdescs[(x)])

/*
 * The DGE interface have a higher max MTU size than normal jumbo frames.
 */
#define DGE_MAX_MTU     16288   /* Max MTU size for this interface */

/*
 * Software state for transmit jobs.
 */
struct dge_txsoft {
        struct mbuf *txs_mbuf;          /* head of our mbuf chain */
        bus_dmamap_t txs_dmamap;        /* our DMA map */
        int txs_firstdesc;              /* first descriptor in packet */
        int txs_lastdesc;               /* last descriptor in packet */
        int txs_ndesc;                  /* # of descriptors used */
};

/*
 * Software state for receive buffers.  Each descriptor gets a
 * 2k (MCLBYTES) buffer and a DMA map.  For packets which fill
 * more than one buffer, we chain them together.
 */
struct dge_rxsoft {
        struct mbuf *rxs_mbuf;          /* head of our mbuf chain */
        bus_dmamap_t rxs_dmamap;        /* our DMA map */
};

/*
 * Software state per device.
 */
struct dge_softc {
        device_t sc_dev;                /* generic device information */
        bus_space_tag_t sc_st;          /* bus space tag */
        bus_space_handle_t sc_sh;       /* bus space handle */
        bus_dma_tag_t sc_dmat;          /* bus DMA tag */
        struct ethercom sc_ethercom;    /* ethernet common data */

        int sc_flags;                   /* flags; see below */
        int sc_bus_speed;               /* PCI/PCIX bus speed */
        int sc_pcix_offset;             /* PCIX capability register offset */

        const struct dge_product *sc_dgep; /* Pointer to the dge_product entry */
        pci_chipset_tag_t sc_pc;
        pcitag_t sc_pt;
        int sc_mmrbc;                   /* Max PCIX memory read byte count */

        void *sc_ih;                    /* interrupt cookie */

        struct ifmedia sc_media;

        bus_dmamap_t sc_cddmamap;       /* control data DMA map */
#define sc_cddma        sc_cddmamap->dm_segs[0].ds_addr

        int             sc_align_tweak;

        /*
         * Software state for the transmit and receive descriptors.
         */
        struct dge_txsoft sc_txsoft[DGE_TXQUEUELEN];
        struct dge_rxsoft sc_rxsoft[DGE_NRXDESC];

        /*
         * Control data structures.
         */
        struct dge_control_data *sc_control_data;
#define sc_txdescs      sc_control_data->wcd_txdescs
#define sc_rxdescs      sc_control_data->wcd_rxdescs

#ifdef DGE_EVENT_COUNTERS
        /* Event counters. */
        struct evcnt sc_ev_txsstall;    /* Tx stalled due to no txs */
        struct evcnt sc_ev_txdstall;    /* Tx stalled due to no txd */
        struct evcnt sc_ev_txforceintr; /* Tx interrupts forced */
        struct evcnt sc_ev_txdw;        /* Tx descriptor interrupts */
        struct evcnt sc_ev_txqe;        /* Tx queue empty interrupts */
        struct evcnt sc_ev_rxintr;      /* Rx interrupts */
        struct evcnt sc_ev_linkintr;    /* Link interrupts */

        struct evcnt sc_ev_rxipsum;     /* IP checksums checked in-bound */
        struct evcnt sc_ev_rxtusum;     /* TCP/UDP cksums checked in-bound */
        struct evcnt sc_ev_txipsum;     /* IP checksums comp. out-bound */
        struct evcnt sc_ev_txtusum;     /* TCP/UDP cksums comp. out-bound */

        struct evcnt sc_ev_txctx_init;  /* Tx cksum context cache initialized */
        struct evcnt sc_ev_txctx_hit;   /* Tx cksum context cache hit */
        struct evcnt sc_ev_txctx_miss;  /* Tx cksum context cache miss */

        struct evcnt sc_ev_txseg[DGE_NTXSEGS]; /* Tx packets w/ N segments */
        struct evcnt sc_ev_txdrop;      /* Tx packets dropped (too many segs) */
#endif /* DGE_EVENT_COUNTERS */

        int     sc_txfree;              /* number of free Tx descriptors */
        int     sc_txnext;              /* next ready Tx descriptor */

        int     sc_txsfree;             /* number of free Tx jobs */
        int     sc_txsnext;             /* next free Tx job */
        int     sc_txsdirty;            /* dirty Tx jobs */

        uint32_t sc_txctx_ipcs;         /* cached Tx IP cksum ctx */
        uint32_t sc_txctx_tucs;         /* cached Tx TCP/UDP cksum ctx */

        int     sc_rxptr;               /* next ready Rx descriptor/queue ent */
        int     sc_rxdiscard;
        int     sc_rxlen;
        struct mbuf *sc_rxhead;
        struct mbuf *sc_rxtail;
        struct mbuf **sc_rxtailp;

        uint32_t sc_ctrl0;              /* prototype CTRL0 register */
        uint32_t sc_icr;                /* prototype interrupt bits */
        uint32_t sc_tctl;               /* prototype TCTL register */
        uint32_t sc_rctl;               /* prototype RCTL register */

        int sc_mchash_type;             /* multicast filter offset */

        uint16_t sc_eeprom[EEPROM_SIZE];

        krndsource_t rnd_source; /* random source */
#ifdef DGE_OFFBYONE_RXBUG
        void *sc_bugbuf;
        SLIST_HEAD(, rxbugentry) sc_buglist;
        bus_dmamap_t sc_bugmap;
        struct rxbugentry *sc_entry;
#endif
};

#define DGE_RXCHAIN_RESET(sc)                                           \
do {                                                                    \
        (sc)->sc_rxtailp = &(sc)->sc_rxhead;                            \
        *(sc)->sc_rxtailp = NULL;                                       \
        (sc)->sc_rxlen = 0;                                             \
} while (/*CONSTCOND*/0)

#define DGE_RXCHAIN_LINK(sc, m)                                         \
do {                                                                    \
        *(sc)->sc_rxtailp = (sc)->sc_rxtail = (m);                      \
        (sc)->sc_rxtailp = &(m)->m_next;                                \
} while (/*CONSTCOND*/0)

/* sc_flags */
#define DGE_F_BUS64             0x20    /* bus is 64-bit */
#define DGE_F_PCIX              0x40    /* bus is PCI-X */

#ifdef DGE_EVENT_COUNTERS
#define DGE_EVCNT_INCR(ev)      (ev)->ev_count++
#else
#define DGE_EVCNT_INCR(ev)      /* nothing */
#endif

#define CSR_READ(sc, reg)                                               \
        bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (reg))
#define CSR_WRITE(sc, reg, val)                                         \
        bus_space_write_4((sc)->sc_st, (sc)->sc_sh, (reg), (val))

#define DGE_CDTXADDR(sc, x)     ((sc)->sc_cddma + DGE_CDTXOFF((x)))
#define DGE_CDRXADDR(sc, x)     ((sc)->sc_cddma + DGE_CDRXOFF((x)))

#define DGE_CDTXSYNC(sc, x, n, ops)                                     \
do {                                                                    \
        int __x, __n;                                                   \
                                                                        \
        __x = (x);                                                      \
        __n = (n);                                                      \
                                                                        \
        /* If it will wrap around, sync to the end of the ring. */      \
        if ((__x + __n) > DGE_NTXDESC) {                                \
                bus_dmamap_sync((sc)->sc_dmat, (sc)->sc_cddmamap,       \
                    DGE_CDTXOFF(__x), sizeof(struct dge_tdes) *         \
                    (DGE_NTXDESC - __x), (ops));                        \
                __n -= (DGE_NTXDESC - __x);                             \
                __x = 0;                                                \
        }                                                               \
                                                                        \
        /* Now sync whatever is left. */                                \
        bus_dmamap_sync((sc)->sc_dmat, (sc)->sc_cddmamap,               \
            DGE_CDTXOFF(__x), sizeof(struct dge_tdes) * __n, (ops));    \
} while (/*CONSTCOND*/0)

#define DGE_CDRXSYNC(sc, x, ops)                                                \
do {                                                                    \
        bus_dmamap_sync((sc)->sc_dmat, (sc)->sc_cddmamap,               \
           DGE_CDRXOFF((x)), sizeof(struct dge_rdes), (ops));           \
} while (/*CONSTCOND*/0)

#ifdef DGE_OFFBYONE_RXBUG
#define DGE_INIT_RXDESC(sc, x)                                          \
do {                                                                    \
        struct dge_rxsoft *__rxs = &(sc)->sc_rxsoft[(x)];               \
        struct dge_rdes *__rxd = &(sc)->sc_rxdescs[(x)];                \
        struct mbuf *__m = __rxs->rxs_mbuf;                             \
        const bus_addr_t __rxaddr = sc->sc_bugmap->dm_segs[0].ds_addr + \
            (mtod((__m), char *) - (char *)sc->sc_bugbuf);              \
                                                                        \
        __rxd->dr_baddrl = htole32(__rxaddr);                           \
        __rxd->dr_baddrh = htole32(((uint64_t)__rxaddr) >> 32);         \
        __rxd->dr_len = 0;                                              \
        __rxd->dr_cksum = 0;                                            \
        __rxd->dr_status = 0;                                           \
        __rxd->dr_errors = 0;                                           \
        __rxd->dr_special = 0;                                          \
        DGE_CDRXSYNC((sc), (x), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); \
                                                                        \
        CSR_WRITE((sc), DGE_RDT, (x));                                  \
} while (/*CONSTCOND*/0)
#else
#define DGE_INIT_RXDESC(sc, x)                                          \
do {                                                                    \
        struct dge_rxsoft *__rxs = &(sc)->sc_rxsoft[(x)];               \
        struct dge_rdes *__rxd = &(sc)->sc_rxdescs[(x)];                \
        struct mbuf *__m = __rxs->rxs_mbuf;                             \
                                                                        \
        /*                                                              \
         * Note: We scoot the packet forward 2 bytes in the buffer      \
         * so that the payload after the Ethernet header is aligned     \
         * to a 4-byte boundary.                                        \
         *                                                              \
         * XXX BRAINDAMAGE ALERT!                                       \
         * The stupid chip uses the same size for every buffer, which   \
         * is set in the Receive Control register.  We are using the 2K \
         * size option, but what we REALLY want is (2K - 2)!  For this  \
         * reason, we can't "scoot" packets longer than the standard    \
         * Ethernet MTU.  On strict-alignment platforms, if the total   \
         * size exceeds (2K - 2) we set align_tweak to 0 and let        \
         * the upper layer copy the headers.                            \
         */                                                             \
        __m->m_data = __m->m_ext.ext_buf + (sc)->sc_align_tweak;        \
                                                                        \
        const bus_addr_t __rxaddr =                                     \
            __rxs->rxs_dmamap->dm_segs[0].ds_addr +                     \
            (sc)->sc_align_tweak;                                       \
                                                                        \
        __rxd->dr_baddrl = htole32(__rxaddr);                           \
        __rxd->dr_baddrh = htole32(((uint64_t)__rxaddr) >> 32);         \
        __rxd->dr_len = 0;                                              \
        __rxd->dr_cksum = 0;                                            \
        __rxd->dr_status = 0;                                           \
        __rxd->dr_errors = 0;                                           \
        __rxd->dr_special = 0;                                          \
        DGE_CDRXSYNC((sc), (x), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); \
                                                                        \
        CSR_WRITE((sc), DGE_RDT, (x));                                  \
} while (/*CONSTCOND*/0)
#endif

#ifdef DGE_OFFBYONE_RXBUG
/*
 * Allocation constants.  Much memory may be used for this.
 */
#ifndef DGE_BUFFER_SIZE
#define DGE_BUFFER_SIZE DGE_MAX_MTU
#endif
#define DGE_NBUFFERS    (4*DGE_NRXDESC)
#define DGE_RXMEM       (DGE_NBUFFERS*DGE_BUFFER_SIZE)

struct rxbugentry {
        SLIST_ENTRY(rxbugentry) rb_entry;
        int rb_slot;
};

static int
dge_alloc_rcvmem(struct dge_softc *sc)
{
        char *kva;
        bus_dma_segment_t seg;
        int i, rseg, state, error;
        struct rxbugentry *entry;

        state = error = 0;

        if (bus_dmamem_alloc(sc->sc_dmat, DGE_RXMEM, PAGE_SIZE, 0,
             &seg, 1, &rseg, BUS_DMA_NOWAIT)) {
                aprint_error_dev(sc->sc_dev, "can't alloc rx buffers\n");
                return ENOBUFS;
        }

        state = 1;
        if (bus_dmamem_map(sc->sc_dmat, &seg, rseg, DGE_RXMEM, (void **)&kva,
            BUS_DMA_NOWAIT)) {
                aprint_error_dev(sc->sc_dev,
                    "can't map DMA buffers (%d bytes)\n", (int)DGE_RXMEM);
                error = ENOBUFS;
                goto out;
        }

        state = 2;
        if (bus_dmamap_create(sc->sc_dmat, DGE_RXMEM, 1, DGE_RXMEM, 0,
            BUS_DMA_NOWAIT, &sc->sc_bugmap)) {
                aprint_error_dev(sc->sc_dev, "can't create DMA map\n");
                error = ENOBUFS;
                goto out;
        }

        state = 3;
        if (bus_dmamap_load(sc->sc_dmat, sc->sc_bugmap,
            kva, DGE_RXMEM, NULL, BUS_DMA_NOWAIT)) {
                aprint_error_dev(sc->sc_dev, "can't load DMA map\n");
                error = ENOBUFS;
                goto out;
        }

        state = 4;
        sc->sc_bugbuf = (void *)kva;
        SLIST_INIT(&sc->sc_buglist);

        /*
         * Now divide it up into DGE_BUFFER_SIZE pieces and save the addresses
         * in an array.
         */
        entry = malloc(sizeof(*entry) * DGE_NBUFFERS, M_DEVBUF, M_WAITOK);
        sc->sc_entry = entry;
        for (i = 0; i < DGE_NBUFFERS; i++) {
                entry[i].rb_slot = i;
                SLIST_INSERT_HEAD(&sc->sc_buglist, &entry[i], rb_entry);
        }
out:
        if (error != 0) {
                switch (state) {
                case 4:
                        bus_dmamap_unload(sc->sc_dmat, sc->sc_bugmap);
                        /* FALLTHROUGH */
                case 3:
                        bus_dmamap_destroy(sc->sc_dmat, sc->sc_bugmap);
                        /* FALLTHROUGH */
                case 2:
                        bus_dmamem_unmap(sc->sc_dmat, kva, DGE_RXMEM);
                        /* FALLTHROUGH */
                case 1:
                        bus_dmamem_free(sc->sc_dmat, &seg, rseg);
                        break;
                default:
                        break;
                }
        }

        return error;
}

/*
 * Allocate a jumbo buffer.
 */
static void *
dge_getbuf(struct dge_softc *sc)
{
        struct rxbugentry *entry;

        entry = SLIST_FIRST(&sc->sc_buglist);

        if (entry == NULL) {
                printf("%s: no free RX buffers\n", device_xname(sc->sc_dev));
                return NULL;
        }

        SLIST_REMOVE_HEAD(&sc->sc_buglist, rb_entry);
        return (char *)sc->sc_bugbuf + entry->rb_slot * DGE_BUFFER_SIZE;
}

/*
 * Release a jumbo buffer.
 */
static void
dge_freebuf(struct mbuf *m, void *buf, size_t size, void *arg)
{
        struct rxbugentry *entry;
        struct dge_softc *sc;
        int i, s;

        /* Extract the softc struct pointer. */
        sc = (struct dge_softc *)arg;

        if (sc == NULL)
                panic("dge_freebuf: can't find softc pointer!");

        /* calculate the slot this buffer belongs to */

        i = ((char *)buf - (char *)sc->sc_bugbuf) / DGE_BUFFER_SIZE;

        if ((i < 0) || (i >= DGE_NBUFFERS))
                panic("dge_freebuf: asked to free buffer %d!", i);

        s = splvm();
        entry = sc->sc_entry + i;
        SLIST_INSERT_HEAD(&sc->sc_buglist, entry, rb_entry);

        if (__predict_true(m != NULL))
                pool_cache_put(mb_cache, m);
        splx(s);
}
#endif

static void     dge_start(struct ifnet *);
static void     dge_watchdog(struct ifnet *);
static int      dge_ioctl(struct ifnet *, u_long, void *);
static int      dge_init(struct ifnet *);
static void     dge_stop(struct ifnet *, int);

static bool     dge_shutdown(device_t, int);

static void     dge_reset(struct dge_softc *);
static void     dge_rxdrain(struct dge_softc *);
static int      dge_add_rxbuf(struct dge_softc *, int);

static void     dge_set_filter(struct dge_softc *);

static int      dge_intr(void *);
static void     dge_txintr(struct dge_softc *);
static void     dge_rxintr(struct dge_softc *);
static void     dge_linkintr(struct dge_softc *, uint32_t);

static int      dge_match(device_t, cfdata_t, void *);
static void     dge_attach(device_t, device_t, void *);

static int      dge_read_eeprom(struct dge_softc *sc);
static int      dge_eeprom_clockin(struct dge_softc *sc);
static void     dge_eeprom_clockout(struct dge_softc *sc, int bit);
static uint16_t dge_eeprom_word(struct dge_softc *sc, int addr);
static int      dge_xgmii_mediachange(struct ifnet *);
static void     dge_xgmii_mediastatus(struct ifnet *, struct ifmediareq *);
static void     dge_xgmii_reset(struct dge_softc *);
static void     dge_xgmii_writereg(struct dge_softc *, int, int, int);


CFATTACH_DECL_NEW(dge, sizeof(struct dge_softc),
    dge_match, dge_attach, NULL, NULL);

#ifdef DGE_EVENT_COUNTERS
#if DGE_NTXSEGS > 100
#error Update dge_txseg_evcnt_names
#endif
static char (*dge_txseg_evcnt_names)[DGE_NTXSEGS][8 /* "txseg00" + \0 */];
#endif /* DGE_EVENT_COUNTERS */

/*
 * Devices supported by this driver.
 */
struct dge_product {
        const char *name;
        int flags;
#define DGEP_F_10G_LR     0x01
#define DGEP_F_10G_SR     0x02
};

static const struct dge_product i82597EX_lr = {
        .name = "Intel i82597EX 10GbE-LR Ethernet",
        .flags = DGEP_F_10G_LR
};

static const struct dge_product i82597EX_sr = {
        .name = "Intel i82597EX 10GbE-SR Ethernet",
        .flags = DGEP_F_10G_SR
};

static const struct device_compatible_entry compat_data[] = {
        { .id = PCI_ID_CODE(PCI_VENDOR_INTEL,
                PCI_PRODUCT_INTEL_82597EX),
          .data = &i82597EX_lr },

        { .id = PCI_ID_CODE(PCI_VENDOR_INTEL,
                PCI_PRODUCT_INTEL_82597EX_SR),
          .data = &i82597EX_sr },

        PCI_COMPAT_EOL
};

static int
dge_match(device_t parent, cfdata_t cf, void *aux)
{
        struct pci_attach_args *pa = aux;

        return pci_compatible_match(pa, compat_data);
}

static void
dge_attach(device_t parent, device_t self, void *aux)
{
        struct dge_softc *sc = device_private(self);
        struct pci_attach_args *pa = aux;
        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
        pci_chipset_tag_t pc = pa->pa_pc;
        pci_intr_handle_t ih;
        const char *intrstr = NULL;
        bus_dma_segment_t seg;
        int i, rseg, error;
        uint8_t enaddr[ETHER_ADDR_LEN];
        pcireg_t preg, memtype;
        uint32_t reg;
        char intrbuf[PCI_INTRSTR_LEN];
        const struct device_compatible_entry *dce;
        const struct dge_product *dgep;

        dce = pci_compatible_lookup(pa, compat_data);
        KASSERT(dce != NULL);
        sc->sc_dgep = dgep = dce->data;

        sc->sc_dev = self;
        sc->sc_pc = pa->pa_pc;
        sc->sc_pt = pa->pa_tag;

        if (pci_dma64_available(pa))
                sc->sc_dmat = pa->pa_dmat64;
        else
                sc->sc_dmat = pa->pa_dmat;

        pci_aprint_devinfo_fancy(pa, "Ethernet controller",
                dgep->name, 1);

        memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, DGE_PCI_BAR);
        if (pci_mapreg_map(pa, DGE_PCI_BAR, memtype, 0,
            &sc->sc_st, &sc->sc_sh, NULL, NULL)) {
                aprint_error_dev(sc->sc_dev,
                    "unable to map device registers\n");
                return;
        }

        /* Enable bus mastering */
        preg = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
        preg |= PCI_COMMAND_MASTER_ENABLE;
        pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, preg);

        /*
         * Map and establish our interrupt.
         */
        if (pci_intr_map(pa, &ih)) {
                aprint_error_dev(sc->sc_dev, "unable to map interrupt\n");
                return;
        }
        intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf));
        sc->sc_ih = pci_intr_establish_xname(pc, ih, IPL_NET, dge_intr, sc,
            device_xname(self));
        if (sc->sc_ih == NULL) {
                aprint_error_dev(sc->sc_dev, "unable to establish interrupt");
                if (intrstr != NULL)
                        aprint_error(" at %s", intrstr);
                aprint_error("\n");
                return;
        }
        aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr);

        /*
         * Determine a few things about the bus we're connected to.
         */
        reg = CSR_READ(sc, DGE_STATUS);
        if (reg & STATUS_BUS64)
                sc->sc_flags |= DGE_F_BUS64;

        sc->sc_flags |= DGE_F_PCIX;
        if (pci_get_capability(pa->pa_pc, pa->pa_tag,
                               PCI_CAP_PCIX,
                               &sc->sc_pcix_offset, NULL) == 0)
                aprint_error_dev(sc->sc_dev, "unable to find PCIX "
                    "capability\n");

        if (sc->sc_flags & DGE_F_PCIX) {
                switch (reg & STATUS_PCIX_MSK) {
                case STATUS_PCIX_66:
                        sc->sc_bus_speed = 66;
                        break;
                case STATUS_PCIX_100:
                        sc->sc_bus_speed = 100;
                        break;
                case STATUS_PCIX_133:
                        sc->sc_bus_speed = 133;
                        break;
                default:
                        aprint_error_dev(sc->sc_dev,
                            "unknown PCIXSPD %d; assuming 66MHz\n",
                            reg & STATUS_PCIX_MSK);
                        sc->sc_bus_speed = 66;
                }
        } else
                sc->sc_bus_speed = (reg & STATUS_BUS64) ? 66 : 33;
        aprint_verbose_dev(sc->sc_dev, "%d-bit %dMHz %s bus\n",
            (sc->sc_flags & DGE_F_BUS64) ? 64 : 32, sc->sc_bus_speed,
            (sc->sc_flags & DGE_F_PCIX) ? "PCIX" : "PCI");

        /*
         * Allocate the control data structures, and create and load the
         * DMA map for it.
         */
        if ((error = bus_dmamem_alloc(sc->sc_dmat,
            sizeof(struct dge_control_data), PAGE_SIZE, 0, &seg, 1, &rseg,
            0)) != 0) {
                aprint_error_dev(sc->sc_dev,
                    "unable to allocate control data, error = %d\n",
                    error);
                goto fail_0;
        }

        if ((error = bus_dmamem_map(sc->sc_dmat, &seg, rseg,
            sizeof(struct dge_control_data), (void **)&sc->sc_control_data,
            0)) != 0) {
                aprint_error_dev(sc->sc_dev,
                    "unable to map control data, error = %d\n", error);
                goto fail_1;
        }

        if ((error = bus_dmamap_create(sc->sc_dmat,
            sizeof(struct dge_control_data), 1,
            sizeof(struct dge_control_data), 0, 0, &sc->sc_cddmamap)) != 0) {
                aprint_error_dev(sc->sc_dev, "unable to create control data "
                    "DMA map, error = %d\n", error);
                goto fail_2;
        }

        if ((error = bus_dmamap_load(sc->sc_dmat, sc->sc_cddmamap,
            sc->sc_control_data, sizeof(struct dge_control_data), NULL,
            0)) != 0) {
                aprint_error_dev(sc->sc_dev,
                    "unable to load control data DMA map, error = %d\n",
                    error);
                goto fail_3;
        }

#ifdef DGE_OFFBYONE_RXBUG
        if (dge_alloc_rcvmem(sc) != 0)
                return; /* Already complained */
#endif
        /*
         * Create the transmit buffer DMA maps.
         */
        for (i = 0; i < DGE_TXQUEUELEN; i++) {
                if ((error = bus_dmamap_create(sc->sc_dmat, DGE_MAX_MTU,
                    DGE_NTXSEGS, MCLBYTES, 0, 0,
                    &sc->sc_txsoft[i].txs_dmamap)) != 0) {
                        aprint_error_dev(sc->sc_dev, "unable to create Tx DMA map %d, "
                            "error = %d\n", i, error);
                        goto fail_4;
                }
        }

        /*
         * Create the receive buffer DMA maps.
         */
        for (i = 0; i < DGE_NRXDESC; i++) {
#ifdef DGE_OFFBYONE_RXBUG
                if ((error = bus_dmamap_create(sc->sc_dmat, DGE_BUFFER_SIZE, 1,
                    DGE_BUFFER_SIZE, 0, 0, &sc->sc_rxsoft[i].rxs_dmamap)) != 0) {
#else
                if ((error = bus_dmamap_create(sc->sc_dmat, MCLBYTES, 1,
                    MCLBYTES, 0, 0, &sc->sc_rxsoft[i].rxs_dmamap)) != 0) {
#endif
                        aprint_error_dev(sc->sc_dev, "unable to create Rx DMA "
                            "map %d, error = %d\n", i, error);
                        goto fail_5;
                }
                sc->sc_rxsoft[i].rxs_mbuf = NULL;
        }

        /*
         * Set bits in ctrl0 register.
         * Should get the software defined pins out of EEPROM?
         */
        sc->sc_ctrl0 |= CTRL0_RPE | CTRL0_TPE; /* XON/XOFF */
        sc->sc_ctrl0 |= CTRL0_SDP3_DIR | CTRL0_SDP2_DIR | CTRL0_SDP1_DIR |
            CTRL0_SDP0_DIR | CTRL0_SDP3 | CTRL0_SDP2 | CTRL0_SDP0;

        /*
         * Reset the chip to a known state.
         */
        dge_reset(sc);

        /*
         * Reset the PHY.
         */
        dge_xgmii_reset(sc);

        /*
         * Read in EEPROM data.
         */
        if (dge_read_eeprom(sc)) {
                aprint_error_dev(sc->sc_dev, "couldn't read EEPROM\n");
                return;
        }

        /*
         * Get the ethernet address.
         */
        enaddr[0] = sc->sc_eeprom[EE_ADDR01] & 0377;
        enaddr[1] = sc->sc_eeprom[EE_ADDR01] >> 8;
        enaddr[2] = sc->sc_eeprom[EE_ADDR23] & 0377;
        enaddr[3] = sc->sc_eeprom[EE_ADDR23] >> 8;
        enaddr[4] = sc->sc_eeprom[EE_ADDR45] & 0377;
        enaddr[5] = sc->sc_eeprom[EE_ADDR45] >> 8;

        aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n",
            ether_sprintf(enaddr));

        /*
         * Setup media stuff.
         */
        sc->sc_ethercom.ec_ifmedia = &sc->sc_media;
        ifmedia_init(&sc->sc_media, IFM_IMASK, dge_xgmii_mediachange,
            dge_xgmii_mediastatus);
        if (dgep->flags & DGEP_F_10G_SR) {
                ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_10G_SR, 0, NULL);
                ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_10G_SR);
        } else { /* XXX default is LR */
                ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_10G_LR, 0, NULL);
                ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_10G_LR);
        }

        ifp = &sc->sc_ethercom.ec_if;
        strlcpy(ifp->if_xname, device_xname(sc->sc_dev), IFNAMSIZ);
        ifp->if_softc = sc;
        ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
        ifp->if_ioctl = dge_ioctl;
        ifp->if_start = dge_start;
        ifp->if_watchdog = dge_watchdog;
        ifp->if_init = dge_init;
        ifp->if_stop = dge_stop;
        IFQ_SET_MAXLEN(&ifp->if_snd, uimax(DGE_IFQUEUELEN, IFQ_MAXLEN));
        IFQ_SET_READY(&ifp->if_snd);

        sc->sc_ethercom.ec_capabilities |=
            ETHERCAP_JUMBO_MTU | ETHERCAP_VLAN_MTU;

        /*
         * We can perform TCPv4 and UDPv4 checksums in-bound.
         */
        ifp->if_capabilities |=
            IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx |
            IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx |
            IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx;

        /*
         * Attach the interface.
         */
        if_attach(ifp);
        if_deferred_start_init(ifp, NULL);
        ether_ifattach(ifp, enaddr);
        rnd_attach_source(&sc->rnd_source, device_xname(sc->sc_dev),
            RND_TYPE_NET, RND_FLAG_DEFAULT);

#ifdef DGE_EVENT_COUNTERS
        /* Fix segment event naming */
        if (dge_txseg_evcnt_names == NULL) {
                dge_txseg_evcnt_names =
                    malloc(sizeof(*dge_txseg_evcnt_names), M_DEVBUF, M_WAITOK);
                for (i = 0; i < DGE_NTXSEGS; i++)
                        snprintf((*dge_txseg_evcnt_names)[i],
                            sizeof((*dge_txseg_evcnt_names)[i]), "txseg%d", i);
        }

        /* Attach event counters. */
        evcnt_attach_dynamic(&sc->sc_ev_txsstall, EVCNT_TYPE_MISC,
            NULL, device_xname(sc->sc_dev), "txsstall");
        evcnt_attach_dynamic(&sc->sc_ev_txdstall, EVCNT_TYPE_MISC,
            NULL, device_xname(sc->sc_dev), "txdstall");
        evcnt_attach_dynamic(&sc->sc_ev_txforceintr, EVCNT_TYPE_MISC,
            NULL, device_xname(sc->sc_dev), "txforceintr");
        evcnt_attach_dynamic(&sc->sc_ev_txdw, EVCNT_TYPE_INTR,
            NULL, device_xname(sc->sc_dev), "txdw");
        evcnt_attach_dynamic(&sc->sc_ev_txqe, EVCNT_TYPE_INTR,
            NULL, device_xname(sc->sc_dev), "txqe");
        evcnt_attach_dynamic(&sc->sc_ev_rxintr, EVCNT_TYPE_INTR,
            NULL, device_xname(sc->sc_dev), "rxintr");
        evcnt_attach_dynamic(&sc->sc_ev_linkintr, EVCNT_TYPE_INTR,
            NULL, device_xname(sc->sc_dev), "linkintr");

        evcnt_attach_dynamic(&sc->sc_ev_rxipsum, EVCNT_TYPE_MISC,
            NULL, device_xname(sc->sc_dev), "rxipsum");
        evcnt_attach_dynamic(&sc->sc_ev_rxtusum, EVCNT_TYPE_MISC,
            NULL, device_xname(sc->sc_dev), "rxtusum");
        evcnt_attach_dynamic(&sc->sc_ev_txipsum, EVCNT_TYPE_MISC,
            NULL, device_xname(sc->sc_dev), "txipsum");
        evcnt_attach_dynamic(&sc->sc_ev_txtusum, EVCNT_TYPE_MISC,
            NULL, device_xname(sc->sc_dev), "txtusum");

        evcnt_attach_dynamic(&sc->sc_ev_txctx_init, EVCNT_TYPE_MISC,
            NULL, device_xname(sc->sc_dev), "txctx init");
        evcnt_attach_dynamic(&sc->sc_ev_txctx_hit, EVCNT_TYPE_MISC,
            NULL, device_xname(sc->sc_dev), "txctx hit");
        evcnt_attach_dynamic(&sc->sc_ev_txctx_miss, EVCNT_TYPE_MISC,
            NULL, device_xname(sc->sc_dev), "txctx miss");

        for (i = 0; i < DGE_NTXSEGS; i++)
                evcnt_attach_dynamic(&sc->sc_ev_txseg[i], EVCNT_TYPE_MISC,
                    NULL, device_xname(sc->sc_dev), (*dge_txseg_evcnt_names)[i]);

        evcnt_attach_dynamic(&sc->sc_ev_txdrop, EVCNT_TYPE_MISC,
            NULL, device_xname(sc->sc_dev), "txdrop");

#endif /* DGE_EVENT_COUNTERS */

        /*
         * Make sure the interface is shutdown during reboot.
         */
        if (pmf_device_register1(self, NULL, NULL, dge_shutdown))
                pmf_class_network_register(self, ifp);
        else
                aprint_error_dev(self, "couldn't establish power handler\n");

        return;

        /*
         * Free any resources we've allocated during the failed attach
         * attempt.  Do this in reverse order and fall through.
         */
 fail_5:
        for (i = 0; i < DGE_NRXDESC; i++) {
                if (sc->sc_rxsoft[i].rxs_dmamap != NULL)
                        bus_dmamap_destroy(sc->sc_dmat,
                            sc->sc_rxsoft[i].rxs_dmamap);
        }
 fail_4:
        for (i = 0; i < DGE_TXQUEUELEN; i++) {
                if (sc->sc_txsoft[i].txs_dmamap != NULL)
                        bus_dmamap_destroy(sc->sc_dmat,
                            sc->sc_txsoft[i].txs_dmamap);
        }
        bus_dmamap_unload(sc->sc_dmat, sc->sc_cddmamap);
 fail_3:
        bus_dmamap_destroy(sc->sc_dmat, sc->sc_cddmamap);
 fail_2:
        bus_dmamem_unmap(sc->sc_dmat, (void *)sc->sc_control_data,
            sizeof(struct dge_control_data));
 fail_1:
        bus_dmamem_free(sc->sc_dmat, &seg, rseg);
 fail_0:
        return;
}

/*
 * dge_shutdown:
 *
 *      Make sure the interface is stopped at reboot time.
 */
static bool
dge_shutdown(device_t self, int howto)
{
        struct dge_softc *sc;

        sc = device_private(self);
        dge_stop(&sc->sc_ethercom.ec_if, 1);

        return true;
}

/*
 * dge_tx_cksum:
 *
 *      Set up TCP/IP checksumming parameters for the
 *      specified packet.
 */
static int
dge_tx_cksum(struct dge_softc *sc, struct dge_txsoft *txs, uint8_t *fieldsp)
{
        struct mbuf *m0 = txs->txs_mbuf;
        struct dge_ctdes *t;
        uint32_t ipcs, tucs;
        struct ether_header *eh;
        int offset, iphl;
        uint8_t fields = 0;

        /*
         * XXX It would be nice if the mbuf pkthdr had offset
         * fields for the protocol headers.
         */

        eh = mtod(m0, struct ether_header *);
        switch (htons(eh->ether_type)) {
        case ETHERTYPE_IP:
                offset = ETHER_HDR_LEN;
                break;

        case ETHERTYPE_VLAN:
                offset = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
                break;

        default:
                /*
                 * Don't support this protocol or encapsulation.
                 */
                *fieldsp = 0;
                return 0;
        }

        iphl = M_CSUM_DATA_IPv4_IPHL(m0->m_pkthdr.csum_data);

        /*
         * NOTE: Even if we're not using the IP or TCP/UDP checksum
         * offload feature, if we load the context descriptor, we
         * MUST provide valid values for IPCSS and TUCSS fields.
         */

        if (m0->m_pkthdr.csum_flags & M_CSUM_IPv4) {
                DGE_EVCNT_INCR(&sc->sc_ev_txipsum);
                fields |= TDESC_POPTS_IXSM;
                ipcs = DGE_TCPIP_IPCSS(offset) |
                    DGE_TCPIP_IPCSO(offset + offsetof(struct ip, ip_sum)) |
                    DGE_TCPIP_IPCSE(offset + iphl - 1);
        } else if (__predict_true(sc->sc_txctx_ipcs != 0xffffffff)) {
                /* Use the cached value. */
                ipcs = sc->sc_txctx_ipcs;
        } else {
                /* Just initialize it to the likely value anyway. */
                ipcs = DGE_TCPIP_IPCSS(offset) |
                    DGE_TCPIP_IPCSO(offset + offsetof(struct ip, ip_sum)) |
                    DGE_TCPIP_IPCSE(offset + iphl - 1);
        }
        DPRINTF(DGE_DEBUG_CKSUM,
            ("%s: CKSUM: offset %d ipcs 0x%x\n",
            device_xname(sc->sc_dev), offset, ipcs));

        offset += iphl;

        if (m0->m_pkthdr.csum_flags & (M_CSUM_TCPv4 | M_CSUM_UDPv4)) {
                DGE_EVCNT_INCR(&sc->sc_ev_txtusum);
                fields |= TDESC_POPTS_TXSM;
                tucs = DGE_TCPIP_TUCSS(offset) |
                   DGE_TCPIP_TUCSO(offset + M_CSUM_DATA_IPv4_OFFSET(m0->m_pkthdr.csum_data)) |
                   DGE_TCPIP_TUCSE(0) /* rest of packet */;
        } else if (__predict_true(sc->sc_txctx_tucs != 0xffffffff)) {
                /* Use the cached value. */
                tucs = sc->sc_txctx_tucs;
        } else {
                /* Just initialize it to a valid TCP context. */
                tucs = DGE_TCPIP_TUCSS(offset) |
                    DGE_TCPIP_TUCSO(offset + offsetof(struct tcphdr, th_sum)) |
                    DGE_TCPIP_TUCSE(0) /* rest of packet */;
        }

        DPRINTF(DGE_DEBUG_CKSUM,
            ("%s: CKSUM: offset %d tucs 0x%x\n",
            device_xname(sc->sc_dev), offset, tucs));

        if (sc->sc_txctx_ipcs == ipcs &&
            sc->sc_txctx_tucs == tucs) {
                /* Cached context is fine. */
                DGE_EVCNT_INCR(&sc->sc_ev_txctx_hit);
        } else {
                /* Fill in the context descriptor. */
#ifdef DGE_EVENT_COUNTERS
                if (sc->sc_txctx_ipcs == 0xffffffff &&
                    sc->sc_txctx_tucs == 0xffffffff)
                        DGE_EVCNT_INCR(&sc->sc_ev_txctx_init);
                else
                        DGE_EVCNT_INCR(&sc->sc_ev_txctx_miss);
#endif
                t = (struct dge_ctdes *)&sc->sc_txdescs[sc->sc_txnext];
                t->dc_tcpip_ipcs = htole32(ipcs);
                t->dc_tcpip_tucs = htole32(tucs);
                t->dc_tcpip_cmdlen = htole32(TDESC_DTYP_CTD);
                t->dc_tcpip_seg = 0;
                DGE_CDTXSYNC(sc, sc->sc_txnext, 1, BUS_DMASYNC_PREWRITE);

                sc->sc_txctx_ipcs = ipcs;
                sc->sc_txctx_tucs = tucs;

                sc->sc_txnext = DGE_NEXTTX(sc->sc_txnext);
                txs->txs_ndesc++;
        }

        *fieldsp = fields;

        return 0;
}

/*
 * dge_start:           [ifnet interface function]
 *
 *      Start packet transmission on the interface.
 */
static void
dge_start(struct ifnet *ifp)
{
        struct dge_softc *sc = ifp->if_softc;
        struct mbuf *m0;
        struct dge_txsoft *txs;
        bus_dmamap_t dmamap;
        int error, nexttx, lasttx = -1, ofree, seg;
        uint32_t cksumcmd;
        uint8_t cksumfields;

        if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
                return;

        /*
         * Remember the previous number of free descriptors.
         */
        ofree = sc->sc_txfree;

        /*
         * Loop through the send queue, setting up transmit descriptors
         * until we drain the queue, or use up all available transmit
         * descriptors.
         */
        for (;;) {
                /* Grab a packet off the queue. */
                IFQ_POLL(&ifp->if_snd, m0);
                if (m0 == NULL)
                        break;

                DPRINTF(DGE_DEBUG_TX,
                    ("%s: TX: have packet to transmit: %p\n",
                    device_xname(sc->sc_dev), m0));

                /* Get a work queue entry. */
                if (sc->sc_txsfree < DGE_TXQUEUE_GC) {
                        dge_txintr(sc);
                        if (sc->sc_txsfree == 0) {
                                DPRINTF(DGE_DEBUG_TX,
                                    ("%s: TX: no free job descriptors\n",
                                        device_xname(sc->sc_dev)));
                                DGE_EVCNT_INCR(&sc->sc_ev_txsstall);
                                break;
                        }
                }

                txs = &sc->sc_txsoft[sc->sc_txsnext];
                dmamap = txs->txs_dmamap;

                /*
                 * Load the DMA map.  If this fails, the packet either
                 * didn't fit in the allotted number of segments, or we
                 * were short on resources.  For the too-many-segments
                 * case, we simply report an error and drop the packet,
                 * since we can't sanely copy a jumbo packet to a single
                 * buffer.
                 */
                error = bus_dmamap_load_mbuf(sc->sc_dmat, dmamap, m0,
                    BUS_DMA_WRITE | BUS_DMA_NOWAIT);
                if (error) {
                        if (error == EFBIG) {
                                DGE_EVCNT_INCR(&sc->sc_ev_txdrop);
                                printf("%s: Tx packet consumes too many "
                                    "DMA segments, dropping...\n",
                                    device_xname(sc->sc_dev));
                                IFQ_DEQUEUE(&ifp->if_snd, m0);
                                m_freem(m0);
                                continue;
                        }
                        /*
                         * Short on resources, just stop for now.
                         */
                        DPRINTF(DGE_DEBUG_TX,
                            ("%s: TX: dmamap load failed: %d\n",
                            device_xname(sc->sc_dev), error));
                        break;
                }

                /*
                 * Ensure we have enough descriptors free to describe
                 * the packet.  Note, we always reserve one descriptor
                 * at the end of the ring due to the semantics of the
                 * TDT register, plus one more in the event we need
                 * to re-load checksum offload context.
                 */
                if (dmamap->dm_nsegs > (sc->sc_txfree - 2)) {
                        /*
                         * Not enough free descriptors to transmit this
                         * packet.  We haven't committed anything yet,
                         * so just unload the DMA map, put the packet
                         * pack on the queue, and punt.  Notify the upper
                         * layer that there are no more slots left.
                         */
                        DPRINTF(DGE_DEBUG_TX,
                            ("%s: TX: need %d descriptors, have %d\n",
                            device_xname(sc->sc_dev), dmamap->dm_nsegs,
                            sc->sc_txfree - 1));
                        ifp->if_flags |= IFF_OACTIVE;
                        bus_dmamap_unload(sc->sc_dmat, dmamap);
                        DGE_EVCNT_INCR(&sc->sc_ev_txdstall);
                        break;
                }

                IFQ_DEQUEUE(&ifp->if_snd, m0);

                /*
                 * WE ARE NOW COMMITTED TO TRANSMITTING THE PACKET.
                 */

                /* Sync the DMA map. */
                bus_dmamap_sync(sc->sc_dmat, dmamap, 0, dmamap->dm_mapsize,
                    BUS_DMASYNC_PREWRITE);

                DPRINTF(DGE_DEBUG_TX,
                    ("%s: TX: packet has %d DMA segments\n",
                    device_xname(sc->sc_dev), dmamap->dm_nsegs));

                DGE_EVCNT_INCR(&sc->sc_ev_txseg[dmamap->dm_nsegs - 1]);

                /*
                 * Store a pointer to the packet so that we can free it
                 * later.
                 *
                 * Initially, we consider the number of descriptors the
                 * packet uses the number of DMA segments.  This may be
                 * incremented by 1 if we do checksum offload (a descriptor
                 * is used to set the checksum context).
                 */
                txs->txs_mbuf = m0;
                txs->txs_firstdesc = sc->sc_txnext;
                txs->txs_ndesc = dmamap->dm_nsegs;

                /*
                 * Set up checksum offload parameters for
                 * this packet.
                 */
                if (m0->m_pkthdr.csum_flags &
                    (M_CSUM_IPv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) {
                        if (dge_tx_cksum(sc, txs, &cksumfields) != 0) {
                                /* Error message already displayed. */
                                bus_dmamap_unload(sc->sc_dmat, dmamap);
                                continue;
                        }
                } else {
                        cksumfields = 0;
                }

                cksumcmd = TDESC_DCMD_IDE | TDESC_DTYP_DATA;

                /*
                 * Initialize the transmit descriptor.
                 */
                for (nexttx = sc->sc_txnext, seg = 0;
                     seg < dmamap->dm_nsegs;
                     seg++, nexttx = DGE_NEXTTX(nexttx)) {
                        sc->sc_txdescs[nexttx].dt_baddrh =
                            htole32(((uint64_t)dmamap->dm_segs[seg].ds_addr) >> 32);
                        sc->sc_txdescs[nexttx].dt_baddrl =
                            htole32(dmamap->dm_segs[seg].ds_addr);
                        sc->sc_txdescs[nexttx].dt_ctl =
                            htole32(cksumcmd | dmamap->dm_segs[seg].ds_len);
                        sc->sc_txdescs[nexttx].dt_status = 0;
                        sc->sc_txdescs[nexttx].dt_popts = cksumfields;
                        sc->sc_txdescs[nexttx].dt_vlan = 0;
                        lasttx = nexttx;

                        DPRINTF(DGE_DEBUG_TX,
                            ("%s: TX: desc %d: high 0x%08lx, low 0x%08lx, len 0x%04lx\n",
                            device_xname(sc->sc_dev), nexttx,
                            (unsigned long)(((uint64_t)dmamap->dm_segs[seg].ds_addr) >> 32),
                            (unsigned long)((uint32_t)dmamap->dm_segs[seg].ds_addr),
                            (unsigned long)dmamap->dm_segs[seg].ds_len));
                }

                KASSERT(lasttx != -1);

                /*
                 * Set up the command byte on the last descriptor of
                 * the packet.  If we're in the interrupt delay window,
                 * delay the interrupt.
                 */
                sc->sc_txdescs[lasttx].dt_ctl |=
                    htole32(TDESC_DCMD_EOP | TDESC_DCMD_RS);

                txs->txs_lastdesc = lasttx;

                DPRINTF(DGE_DEBUG_TX,
                    ("%s: TX: desc %d: cmdlen 0x%08x\n", device_xname(sc->sc_dev),
                    lasttx, le32toh(sc->sc_txdescs[lasttx].dt_ctl)));

                /* Sync the descriptors we're using. */
                DGE_CDTXSYNC(sc, sc->sc_txnext, dmamap->dm_nsegs,
                    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

                /* Give the packet to the chip. */
                CSR_WRITE(sc, DGE_TDT, nexttx);

                DPRINTF(DGE_DEBUG_TX,
                    ("%s: TX: TDT -> %d\n", device_xname(sc->sc_dev), nexttx));

                DPRINTF(DGE_DEBUG_TX,
                    ("%s: TX: finished transmitting packet, job %d\n",
                    device_xname(sc->sc_dev), sc->sc_txsnext));

                /* Advance the tx pointer. */
                sc->sc_txfree -= txs->txs_ndesc;
                sc->sc_txnext = nexttx;

                sc->sc_txsfree--;
                sc->sc_txsnext = DGE_NEXTTXS(sc->sc_txsnext);

                /* Pass the packet to any BPF listeners. */
                bpf_mtap(ifp, m0, BPF_D_OUT);
        }

        if (sc->sc_txsfree == 0 || sc->sc_txfree <= 2) {
                /* No more slots; notify upper layer. */
                ifp->if_flags |= IFF_OACTIVE;
        }

        if (sc->sc_txfree != ofree) {
                /* Set a watchdog timer in case the chip flakes out. */
                ifp->if_timer = 5;
        }
}

/*
 * dge_watchdog:                [ifnet interface function]
 *
 *      Watchdog timer handler.
 */
static void
dge_watchdog(struct ifnet *ifp)
{
        struct dge_softc *sc = ifp->if_softc;

        /*
         * Since we're using delayed interrupts, sweep up
         * before we report an error.
         */
        dge_txintr(sc);

        if (sc->sc_txfree != DGE_NTXDESC) {
                printf("%s: device timeout (txfree %d txsfree %d txnext %d)\n",
                    device_xname(sc->sc_dev), sc->sc_txfree, sc->sc_txsfree,
                    sc->sc_txnext);
                if_statinc(ifp, if_oerrors);

                /* Reset the interface. */
                (void) dge_init(ifp);
        }

        /* Try to get more packets going. */
        dge_start(ifp);
}

/*
 * dge_ioctl:           [ifnet interface function]
 *
 *      Handle control requests from the operator.
 */
static int
dge_ioctl(struct ifnet *ifp, u_long cmd, void *data)
{
        struct dge_softc *sc = ifp->if_softc;
        struct ifreq *ifr = (struct ifreq *) data;
        pcireg_t preg;
        int s, error, mmrbc;

        s = splnet();

        switch (cmd) {
        case SIOCSIFMTU:
                if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > DGE_MAX_MTU)
                        error = EINVAL;
                else if ((error = ifioctl_common(ifp, cmd, data)) != ENETRESET)
                        break;
                else if (ifp->if_flags & IFF_UP)
                        error = if_init(ifp);
                else
                        error = 0;
                break;

        case SIOCSIFFLAGS:
                if ((error = ifioctl_common(ifp, cmd, data)) != 0)
                        break;
                /* extract link flags */
                if ((ifp->if_flags & IFF_LINK0) == 0 &&
                    (ifp->if_flags & IFF_LINK1) == 0)
                        mmrbc = PCIX_MMRBC_512;
                else if ((ifp->if_flags & IFF_LINK0) == 0 &&
                    (ifp->if_flags & IFF_LINK1) != 0)
                        mmrbc = PCIX_MMRBC_1024;
                else if ((ifp->if_flags & IFF_LINK0) != 0 &&
                    (ifp->if_flags & IFF_LINK1) == 0)
                        mmrbc = PCIX_MMRBC_2048;
                else
                        mmrbc = PCIX_MMRBC_4096;
                if (mmrbc != sc->sc_mmrbc) {
                        preg = pci_conf_read(sc->sc_pc, sc->sc_pt,DGE_PCIX_CMD);
                        preg &= ~PCIX_MMRBC_MSK;
                        preg |= mmrbc;
                        pci_conf_write(sc->sc_pc, sc->sc_pt,DGE_PCIX_CMD, preg);
                        sc->sc_mmrbc = mmrbc;
                }
                /* FALLTHROUGH */
        default:
                if ((error = ether_ioctl(ifp, cmd, data)) != ENETRESET)
                        break;

                error = 0;

                if (cmd == SIOCSIFCAP)
                        error = if_init(ifp);
                else if (cmd != SIOCADDMULTI && cmd != SIOCDELMULTI)
                        ;
                else if (ifp->if_flags & IFF_RUNNING) {
                        /*
                         * Multicast list has changed; set the hardware filter
                         * accordingly.
                         */
                        dge_set_filter(sc);
                }
                break;
        }

        /* Try to get more packets going. */
        dge_start(ifp);

        splx(s);
        return error;
}

/*
 * dge_intr:
 *
 *      Interrupt service routine.
 */
static int
dge_intr(void *arg)
{
        struct dge_softc *sc = arg;
        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
        uint32_t icr;
        int wantinit, handled = 0;

        for (wantinit = 0; wantinit == 0;) {
                icr = CSR_READ(sc, DGE_ICR);
                if ((icr & sc->sc_icr) == 0)
                        break;

                rnd_add_uint32(&sc->rnd_source, icr);

                handled = 1;

#if defined(DGE_DEBUG) || defined(DGE_EVENT_COUNTERS)
                if (icr & (ICR_RXDMT0 | ICR_RXT0)) {
                        DPRINTF(DGE_DEBUG_RX,
                            ("%s: RX: got Rx intr 0x%08x\n",
                            device_xname(sc->sc_dev),
                            icr & (ICR_RXDMT0 | ICR_RXT0)));
                        DGE_EVCNT_INCR(&sc->sc_ev_rxintr);
                }
#endif
                dge_rxintr(sc);

#if defined(DGE_DEBUG) || defined(DGE_EVENT_COUNTERS)
                if (icr & ICR_TXDW) {
                        DPRINTF(DGE_DEBUG_TX,
                            ("%s: TX: got TXDW interrupt\n",
                            device_xname(sc->sc_dev)));
                        DGE_EVCNT_INCR(&sc->sc_ev_txdw);
                }
                if (icr & ICR_TXQE)
                        DGE_EVCNT_INCR(&sc->sc_ev_txqe);
#endif
                dge_txintr(sc);

                if (icr & (ICR_LSC | ICR_RXSEQ)) {
                        DGE_EVCNT_INCR(&sc->sc_ev_linkintr);
                        dge_linkintr(sc, icr);
                }

                if (icr & ICR_RXO) {
                        printf("%s: Receive overrun\n",
                            device_xname(sc->sc_dev));
                        wantinit = 1;
                }
        }

        if (handled) {
                if (wantinit)
                        dge_init(ifp);

                /* Try to get more packets going. */
                if_schedule_deferred_start(ifp);
        }

        return handled;
}

/*
 * dge_txintr:
 *
 *      Helper; handle transmit interrupts.
 */
static void
dge_txintr(struct dge_softc *sc)
{
        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
        struct dge_txsoft *txs;
        uint8_t status;
        int i;

        ifp->if_flags &= ~IFF_OACTIVE;

        /*
         * Go through the Tx list and free mbufs for those
         * frames which have been transmitted.
         */
        for (i = sc->sc_txsdirty; sc->sc_txsfree != DGE_TXQUEUELEN;
             i = DGE_NEXTTXS(i), sc->sc_txsfree++) {
                txs = &sc->sc_txsoft[i];

                DPRINTF(DGE_DEBUG_TX,
                    ("%s: TX: checking job %d\n", device_xname(sc->sc_dev), i));

                DGE_CDTXSYNC(sc, txs->txs_firstdesc, txs->txs_dmamap->dm_nsegs,
                    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);

                status =
                    sc->sc_txdescs[txs->txs_lastdesc].dt_status;
                if ((status & TDESC_STA_DD) == 0) {
                        DGE_CDTXSYNC(sc, txs->txs_lastdesc, 1,
                            BUS_DMASYNC_PREREAD);
                        break;
                }

                DPRINTF(DGE_DEBUG_TX,
                    ("%s: TX: job %d done: descs %d..%d\n",
                    device_xname(sc->sc_dev), i, txs->txs_firstdesc,
                    txs->txs_lastdesc));

                if_statinc(ifp, if_opackets);
                sc->sc_txfree += txs->txs_ndesc;
                bus_dmamap_sync(sc->sc_dmat, txs->txs_dmamap,
                    0, txs->txs_dmamap->dm_mapsize, BUS_DMASYNC_POSTWRITE);
                bus_dmamap_unload(sc->sc_dmat, txs->txs_dmamap);
                m_freem(txs->txs_mbuf);
                txs->txs_mbuf = NULL;
        }

        /* Update the dirty transmit buffer pointer. */
        sc->sc_txsdirty = i;
        DPRINTF(DGE_DEBUG_TX,
            ("%s: TX: txsdirty -> %d\n", device_xname(sc->sc_dev), i));

        /*
         * If there are no more pending transmissions, cancel the watchdog
         * timer.
         */
        if (sc->sc_txsfree == DGE_TXQUEUELEN)
                ifp->if_timer = 0;
}

/*
 * dge_rxintr:
 *
 *      Helper; handle receive interrupts.
 */
static void
dge_rxintr(struct dge_softc *sc)
{
        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
        struct dge_rxsoft *rxs;
        struct mbuf *m;
        int i, len;
        uint8_t status, errors;

        for (i = sc->sc_rxptr;; i = DGE_NEXTRX(i)) {
                rxs = &sc->sc_rxsoft[i];

                DPRINTF(DGE_DEBUG_RX,
                    ("%s: RX: checking descriptor %d\n",
                    device_xname(sc->sc_dev), i));

                DGE_CDRXSYNC(sc, i,
                    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);

                status = sc->sc_rxdescs[i].dr_status;
                errors = sc->sc_rxdescs[i].dr_errors;
                len = le16toh(sc->sc_rxdescs[i].dr_len);

                if ((status & RDESC_STS_DD) == 0) {
                        /* We have processed all of the receive descriptors. */
                        DGE_CDRXSYNC(sc, i, BUS_DMASYNC_PREREAD);
                        break;
                }

                if (__predict_false(sc->sc_rxdiscard)) {
                        DPRINTF(DGE_DEBUG_RX,
                            ("%s: RX: discarding contents of descriptor %d\n",
                            device_xname(sc->sc_dev), i));
                        DGE_INIT_RXDESC(sc, i);
                        if (status & RDESC_STS_EOP) {
                                /* Reset our state. */
                                DPRINTF(DGE_DEBUG_RX,
                                    ("%s: RX: resetting rxdiscard -> 0\n",
                                    device_xname(sc->sc_dev)));
                                sc->sc_rxdiscard = 0;
                        }
                        continue;
                }

                bus_dmamap_sync(sc->sc_dmat, rxs->rxs_dmamap, 0,
                    rxs->rxs_dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD);

                m = rxs->rxs_mbuf;

                /*
                 * Add a new receive buffer to the ring.
                 */
                if (dge_add_rxbuf(sc, i) != 0) {
                        /*
                         * Failed, throw away what we've done so
                         * far, and discard the rest of the packet.
                         */
                        if_statinc(ifp, if_ierrors);
                        bus_dmamap_sync(sc->sc_dmat, rxs->rxs_dmamap, 0,
                            rxs->rxs_dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
                        DGE_INIT_RXDESC(sc, i);
                        if ((status & RDESC_STS_EOP) == 0)
                                sc->sc_rxdiscard = 1;
                        m_freem(sc->sc_rxhead);
                        DGE_RXCHAIN_RESET(sc);
                        DPRINTF(DGE_DEBUG_RX,
                            ("%s: RX: Rx buffer allocation failed, "
                            "dropping packet%s\n", device_xname(sc->sc_dev),
                            sc->sc_rxdiscard ? " (discard)" : ""));
                        continue;
                }
                DGE_INIT_RXDESC(sc, DGE_PREVRX(i)); /* Write the descriptor */

                DGE_RXCHAIN_LINK(sc, m);

                m->m_len = len;

                DPRINTF(DGE_DEBUG_RX,
                    ("%s: RX: buffer at %p len %d\n",
                    device_xname(sc->sc_dev), m->m_data, len));

                /*
                 * If this is not the end of the packet, keep
                 * looking.
                 */
                if ((status & RDESC_STS_EOP) == 0) {
                        sc->sc_rxlen += len;
                        DPRINTF(DGE_DEBUG_RX,
                            ("%s: RX: not yet EOP, rxlen -> %d\n",
                            device_xname(sc->sc_dev), sc->sc_rxlen));
                        continue;
                }

                /*
                 * Okay, we have the entire packet now...
                 */
                *sc->sc_rxtailp = NULL;
                m = sc->sc_rxhead;
                len += sc->sc_rxlen;

                DGE_RXCHAIN_RESET(sc);

                DPRINTF(DGE_DEBUG_RX,
                    ("%s: RX: have entire packet, len -> %d\n",
                    device_xname(sc->sc_dev), len));

                /*
                 * If an error occurred, update stats and drop the packet.
                 */
                if (errors & (RDESC_ERR_CE | RDESC_ERR_SE | RDESC_ERR_P |
                    RDESC_ERR_RXE)) {
                        if_statinc(ifp, if_ierrors);
                        if (errors & RDESC_ERR_SE)
                                printf("%s: symbol error\n",
                                    device_xname(sc->sc_dev));
                        else if (errors & RDESC_ERR_P)
                                printf("%s: parity error\n",
                                    device_xname(sc->sc_dev));
                        else if (errors & RDESC_ERR_CE)
                                printf("%s: CRC error\n",
                                    device_xname(sc->sc_dev));
                        m_freem(m);
                        continue;
                }

                /*
                 * No errors.  Receive the packet.
                 */
                m_set_rcvif(m, ifp);
                m->m_pkthdr.len = len;

                /*
                 * Set up checksum info for this packet.
                 */
                if (status & RDESC_STS_IPCS) {
                        DGE_EVCNT_INCR(&sc->sc_ev_rxipsum);
                        m->m_pkthdr.csum_flags |= M_CSUM_IPv4;
                        if (errors & RDESC_ERR_IPE)
                                m->m_pkthdr.csum_flags |= M_CSUM_IPv4_BAD;
                }
                if (status & RDESC_STS_TCPCS) {
                        /*
                         * Note: we don't know if this was TCP or UDP,
                         * so we just set both bits, and expect the
                         * upper layers to deal.
                         */
                        DGE_EVCNT_INCR(&sc->sc_ev_rxtusum);
                        m->m_pkthdr.csum_flags |= M_CSUM_TCPv4 | M_CSUM_UDPv4;
                        if (errors & RDESC_ERR_TCPE)
                                m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD;
                }

                /* Pass it on. */
                if_percpuq_enqueue(ifp->if_percpuq, m);
        }

        /* Update the receive pointer. */
        sc->sc_rxptr = i;

        DPRINTF(DGE_DEBUG_RX,
            ("%s: RX: rxptr -> %d\n", device_xname(sc->sc_dev), i));
}

/*
 * dge_linkintr:
 *
 *      Helper; handle link interrupts.
 */
static void
dge_linkintr(struct dge_softc *sc, uint32_t icr)
{
        uint32_t status;

        if (icr & ICR_LSC) {
                status = CSR_READ(sc, DGE_STATUS);
                if (status & STATUS_LINKUP) {
                        DPRINTF(DGE_DEBUG_LINK, ("%s: LINK: LSC -> up\n",
                            device_xname(sc->sc_dev)));
                } else {
                        DPRINTF(DGE_DEBUG_LINK, ("%s: LINK: LSC -> down\n",
                            device_xname(sc->sc_dev)));
                }
        } else if (icr & ICR_RXSEQ) {
                DPRINTF(DGE_DEBUG_LINK,
                    ("%s: LINK: Receive sequence error\n",
                    device_xname(sc->sc_dev)));
        }
        /* XXX - fix errata */
}

/*
 * dge_reset:
 *
 *      Reset the i82597 chip.
 */
static void
dge_reset(struct dge_softc *sc)
{
        int i;

        /*
         * Do a chip reset.
         */
        CSR_WRITE(sc, DGE_CTRL0, CTRL0_RST | sc->sc_ctrl0);

        delay(10000);

        for (i = 0; i < 1000; i++) {
                if ((CSR_READ(sc, DGE_CTRL0) & CTRL0_RST) == 0)
                        break;
                delay(20);
        }

        if (CSR_READ(sc, DGE_CTRL0) & CTRL0_RST)
                printf("%s: WARNING: reset failed to complete\n",
                    device_xname(sc->sc_dev));
        /*
         * Reset the EEPROM logic.
         * This will cause the chip to reread its default values,
         * which doesn't happen otherwise (errata).
         */
        CSR_WRITE(sc, DGE_CTRL1, CTRL1_EE_RST);
        delay(10000);
}

/*
 * dge_init:            [ifnet interface function]
 *
 *      Initialize the interface.  Must be called at splnet().
 */
static int
dge_init(struct ifnet *ifp)
{
        struct dge_softc *sc = ifp->if_softc;
        struct dge_rxsoft *rxs;
        int i, error = 0;
        uint32_t reg;

        /*
         * *_HDR_ALIGNED_P is constant 1 if __NO_STRICT_ALIGNMENT is set.
         * There is a small but measurable benefit to avoiding the adjusment
         * of the descriptor so that the headers are aligned, for normal mtu,
         * on such platforms.  One possibility is that the DMA itself is
         * slightly more efficient if the front of the entire packet (instead
         * of the front of the headers) is aligned.
         *
         * Note we must always set align_tweak to 0 if we are using
         * jumbo frames.
         */
#ifdef __NO_STRICT_ALIGNMENT
        sc->sc_align_tweak = 0;
#else
        if ((ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN) > (MCLBYTES - 2))
                sc->sc_align_tweak = 0;
        else
                sc->sc_align_tweak = 2;
#endif /* __NO_STRICT_ALIGNMENT */

        /* Cancel any pending I/O. */
        dge_stop(ifp, 0);

        /* Reset the chip to a known state. */
        dge_reset(sc);

        /* Initialize the transmit descriptor ring. */
        memset(sc->sc_txdescs, 0, sizeof(sc->sc_txdescs));
        DGE_CDTXSYNC(sc, 0, DGE_NTXDESC,
            BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
        sc->sc_txfree = DGE_NTXDESC;
        sc->sc_txnext = 0;

        sc->sc_txctx_ipcs = 0xffffffff;
        sc->sc_txctx_tucs = 0xffffffff;

        CSR_WRITE(sc, DGE_TDBAH, ((uint64_t)DGE_CDTXADDR(sc, 0)) >> 32);
        CSR_WRITE(sc, DGE_TDBAL, DGE_CDTXADDR(sc, 0));
        CSR_WRITE(sc, DGE_TDLEN, sizeof(sc->sc_txdescs));
        CSR_WRITE(sc, DGE_TDH, 0);
        CSR_WRITE(sc, DGE_TDT, 0);
        CSR_WRITE(sc, DGE_TIDV, TIDV);

#if 0
        CSR_WRITE(sc, DGE_TXDCTL, TXDCTL_PTHRESH(0) |
            TXDCTL_HTHRESH(0) | TXDCTL_WTHRESH(0));
#endif
        CSR_WRITE(sc, DGE_RXDCTL,
            RXDCTL_PTHRESH(RXDCTL_PTHRESH_VAL) |
            RXDCTL_HTHRESH(RXDCTL_HTHRESH_VAL) |
            RXDCTL_WTHRESH(RXDCTL_WTHRESH_VAL));

        /* Initialize the transmit job descriptors. */
        for (i = 0; i < DGE_TXQUEUELEN; i++)
                sc->sc_txsoft[i].txs_mbuf = NULL;
        sc->sc_txsfree = DGE_TXQUEUELEN;
        sc->sc_txsnext = 0;
        sc->sc_txsdirty = 0;

        /*
         * Initialize the receive descriptor and receive job
         * descriptor rings.
         */
        CSR_WRITE(sc, DGE_RDBAH, ((uint64_t)DGE_CDRXADDR(sc, 0)) >> 32);
        CSR_WRITE(sc, DGE_RDBAL, DGE_CDRXADDR(sc, 0));
        CSR_WRITE(sc, DGE_RDLEN, sizeof(sc->sc_rxdescs));
        CSR_WRITE(sc, DGE_RDH, DGE_RXSPACE);
        CSR_WRITE(sc, DGE_RDT, 0);
        CSR_WRITE(sc, DGE_RDTR, RDTR | 0x80000000);
        CSR_WRITE(sc, DGE_FCRTL, FCRTL | FCRTL_XONE);
        CSR_WRITE(sc, DGE_FCRTH, FCRTH);

        for (i = 0; i < DGE_NRXDESC; i++) {
                rxs = &sc->sc_rxsoft[i];
                if (rxs->rxs_mbuf == NULL) {
                        if ((error = dge_add_rxbuf(sc, i)) != 0) {
                                printf("%s: unable to allocate or map rx "
                                    "buffer %d, error = %d\n",
                                    device_xname(sc->sc_dev), i, error);
                                /*
                                 * XXX Should attempt to run with fewer receive
                                 * XXX buffers instead of just failing.
                                 */
                                dge_rxdrain(sc);
                                goto out;
                        }
                }
                DGE_INIT_RXDESC(sc, i);
        }
        sc->sc_rxptr = DGE_RXSPACE;
        sc->sc_rxdiscard = 0;
        DGE_RXCHAIN_RESET(sc);

        if (sc->sc_ethercom.ec_capabilities & ETHERCAP_JUMBO_MTU) {
                sc->sc_ctrl0 |= CTRL0_JFE;
                CSR_WRITE(sc, DGE_MFS, ETHER_MAX_LEN_JUMBO << 16);
        }

        /* Write the control registers. */
        CSR_WRITE(sc, DGE_CTRL0, sc->sc_ctrl0);

        /*
         * Set up checksum offload parameters.
         */
        reg = CSR_READ(sc, DGE_RXCSUM);
        if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx)
                reg |= RXCSUM_IPOFL;
        else
                reg &= ~RXCSUM_IPOFL;
        if (ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx))
                reg |= RXCSUM_IPOFL | RXCSUM_TUOFL;
        else {
                reg &= ~RXCSUM_TUOFL;
                if ((ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) == 0)
                        reg &= ~RXCSUM_IPOFL;
        }
        CSR_WRITE(sc, DGE_RXCSUM, reg);

        /*
         * Set up the interrupt registers.
         */
        CSR_WRITE(sc, DGE_IMC, 0xffffffffU);
        sc->sc_icr = ICR_TXDW | ICR_LSC | ICR_RXSEQ | ICR_RXDMT0 |
            ICR_RXO | ICR_RXT0;

        CSR_WRITE(sc, DGE_IMS, sc->sc_icr);

        /*
         * Set up the transmit control register.
         */
        sc->sc_tctl = TCTL_TCE | TCTL_TPDE | TCTL_TXEN;
        CSR_WRITE(sc, DGE_TCTL, sc->sc_tctl);

        /*
         * Set up the receive control register; we actually program
         * the register when we set the receive filter.  Use multicast
         * address offset type 0.
         */
        sc->sc_mchash_type = 0;

        sc->sc_rctl = RCTL_RXEN | RCTL_RDMTS_12 | RCTL_RPDA_MC |
            RCTL_CFF | RCTL_SECRC | RCTL_MO(sc->sc_mchash_type);

#ifdef DGE_OFFBYONE_RXBUG
        sc->sc_rctl |= RCTL_BSIZE_16k;
#else
        switch (MCLBYTES) {
        case 2048:
                sc->sc_rctl |= RCTL_BSIZE_2k;
                break;
        case 4096:
                sc->sc_rctl |= RCTL_BSIZE_4k;
                break;
        case 8192:
                sc->sc_rctl |= RCTL_BSIZE_8k;
                break;
        case 16384:
                sc->sc_rctl |= RCTL_BSIZE_16k;
                break;
        default:
                panic("dge_init: MCLBYTES %d unsupported", MCLBYTES);
        }
#endif

        /* Set the receive filter. */
        /* Also sets RCTL */
        dge_set_filter(sc);

        /* ...all done! */
        ifp->if_flags |= IFF_RUNNING;
        ifp->if_flags &= ~IFF_OACTIVE;

 out:
        if (error)
                printf("%s: interface not running\n", device_xname(sc->sc_dev));
        return error;
}

/*
 * dge_rxdrain:
 *
 *      Drain the receive queue.
 */
static void
dge_rxdrain(struct dge_softc *sc)
{
        struct dge_rxsoft *rxs;
        int i;

        for (i = 0; i < DGE_NRXDESC; i++) {
                rxs = &sc->sc_rxsoft[i];
                if (rxs->rxs_mbuf != NULL) {
                        bus_dmamap_unload(sc->sc_dmat, rxs->rxs_dmamap);
                        m_freem(rxs->rxs_mbuf);
                        rxs->rxs_mbuf = NULL;
                }
        }
}

/*
 * dge_stop:            [ifnet interface function]
 *
 *      Stop transmission on the interface.
 */
static void
dge_stop(struct ifnet *ifp, int disable)
{
        struct dge_softc *sc = ifp->if_softc;
        struct dge_txsoft *txs;
        int i;

        /* Stop the transmit and receive processes. */
        CSR_WRITE(sc, DGE_TCTL, 0);
        CSR_WRITE(sc, DGE_RCTL, 0);

        /* Release any queued transmit buffers. */
        for (i = 0; i < DGE_TXQUEUELEN; i++) {
                txs = &sc->sc_txsoft[i];
                if (txs->txs_mbuf != NULL) {
                        bus_dmamap_unload(sc->sc_dmat, txs->txs_dmamap);
                        m_freem(txs->txs_mbuf);
                        txs->txs_mbuf = NULL;
                }
        }

        /* Mark the interface as down and cancel the watchdog timer. */
        ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
        ifp->if_timer = 0;

        if (disable)
                dge_rxdrain(sc);
}

/*
 * dge_add_rxbuf:
 *
 *      Add a receive buffer to the indicated descriptor.
 */
static int
dge_add_rxbuf(struct dge_softc *sc, int idx)
{
        struct dge_rxsoft *rxs = &sc->sc_rxsoft[idx];
        struct mbuf *m;
        int error;
#ifdef DGE_OFFBYONE_RXBUG
        void *buf;
#endif

        MGETHDR(m, M_DONTWAIT, MT_DATA);
        if (m == NULL)
                return ENOBUFS;

#ifdef DGE_OFFBYONE_RXBUG
        if ((buf = dge_getbuf(sc)) == NULL)
                return ENOBUFS;

        m->m_len = m->m_pkthdr.len = DGE_BUFFER_SIZE;
        MEXTADD(m, buf, DGE_BUFFER_SIZE, M_DEVBUF, dge_freebuf, sc);
        m->m_flags |= M_EXT_RW;

        if (rxs->rxs_mbuf != NULL)
                bus_dmamap_unload(sc->sc_dmat, rxs->rxs_dmamap);
        rxs->rxs_mbuf = m;

        error = bus_dmamap_load(sc->sc_dmat, rxs->rxs_dmamap, buf,
            DGE_BUFFER_SIZE, NULL, BUS_DMA_READ | BUS_DMA_NOWAIT);
#else
        MCLGET(m, M_DONTWAIT);
        if ((m->m_flags & M_EXT) == 0) {
                m_freem(m);
                return ENOBUFS;
        }

        if (rxs->rxs_mbuf != NULL)
                bus_dmamap_unload(sc->sc_dmat, rxs->rxs_dmamap);

        rxs->rxs_mbuf = m;

        m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
        error = bus_dmamap_load_mbuf(sc->sc_dmat, rxs->rxs_dmamap, m,
            BUS_DMA_READ | BUS_DMA_NOWAIT);
#endif
        if (error) {
                printf("%s: unable to load rx DMA map %d, error = %d\n",
                    device_xname(sc->sc_dev), idx, error);
                panic("dge_add_rxbuf"); /* XXX XXX XXX */
        }
        bus_dmamap_sync(sc->sc_dmat, rxs->rxs_dmamap, 0,
            rxs->rxs_dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);

        return 0;
}

/*
 * dge_set_ral:
 *
 *      Set an entry in the receive address list.
 */
static void
dge_set_ral(struct dge_softc *sc, const uint8_t *enaddr, int idx)
{
        uint32_t ral_lo, ral_hi;

        if (enaddr != NULL) {
                ral_lo = enaddr[0] | (enaddr[1] << 8) | (enaddr[2] << 16) |
                    (enaddr[3] << 24);
                ral_hi = enaddr[4] | (enaddr[5] << 8);
                ral_hi |= RAH_AV;
        } else {
                ral_lo = 0;
                ral_hi = 0;
        }
        CSR_WRITE(sc, RA_ADDR(DGE_RAL, idx), ral_lo);
        CSR_WRITE(sc, RA_ADDR(DGE_RAH, idx), ral_hi);
}

/*
 * dge_mchash:
 *
 *      Compute the hash of the multicast address for the 4096-bit
 *      multicast filter.
 */
static uint32_t
dge_mchash(struct dge_softc *sc, const uint8_t *enaddr)
{
        static const int lo_shift[4] = { 4, 3, 2, 0 };
        static const int hi_shift[4] = { 4, 5, 6, 8 };
        uint32_t hash;

        hash = (enaddr[4] >> lo_shift[sc->sc_mchash_type]) |
            (((uint16_t) enaddr[5]) << hi_shift[sc->sc_mchash_type]);

        return (hash & 0xfff);
}

/*
 * dge_set_filter:
 *
 *      Set up the receive filter.
 */
static void
dge_set_filter(struct dge_softc *sc)
{
        struct ethercom *ec = &sc->sc_ethercom;
        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
        struct ether_multi *enm;
        struct ether_multistep step;
        uint32_t hash, reg, bit;
        int i;

        sc->sc_rctl &= ~(RCTL_BAM | RCTL_UPE | RCTL_MPE);

        if (ifp->if_flags & IFF_BROADCAST)
                sc->sc_rctl |= RCTL_BAM;
        if (ifp->if_flags & IFF_PROMISC) {
                sc->sc_rctl |= RCTL_UPE;
                goto allmulti;
        }

        /*
         * Set the station address in the first RAL slot, and
         * clear the remaining slots.
         */
        dge_set_ral(sc, CLLADDR(ifp->if_sadl), 0);
        for (i = 1; i < RA_TABSIZE; i++)
                dge_set_ral(sc, NULL, i);

        /* Clear out the multicast table. */
        for (i = 0; i < MC_TABSIZE; i++)
                CSR_WRITE(sc, DGE_MTA + (i << 2), 0);

        ETHER_LOCK(ec);
        ETHER_FIRST_MULTI(step, ec, enm);
        while (enm != NULL) {
                if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
                        /*
                         * We must listen to a range of multicast addresses.
                         * For now, just accept all multicasts, rather than
                         * trying to set only those filter bits needed to match
                         * the range.  (At this time, the only use of address
                         * ranges is for IP multicast routing, for which the
                         * range is big enough to require all bits set.)
                         */
                        ETHER_UNLOCK(ec);
                        goto allmulti;
                }

                hash = dge_mchash(sc, enm->enm_addrlo);

                reg = (hash >> 5) & 0x7f;
                bit = hash & 0x1f;

                hash = CSR_READ(sc, DGE_MTA + (reg << 2));
                hash |= 1U << bit;

                CSR_WRITE(sc, DGE_MTA + (reg << 2), hash);

                ETHER_NEXT_MULTI(step, enm);
        }
        ETHER_UNLOCK(ec);

        ifp->if_flags &= ~IFF_ALLMULTI;
        goto setit;

 allmulti:
        ifp->if_flags |= IFF_ALLMULTI;
        sc->sc_rctl |= RCTL_MPE;

 setit:
        CSR_WRITE(sc, DGE_RCTL, sc->sc_rctl);
}

/*
 * Read in the EEPROM info and verify checksum.
 */
int
dge_read_eeprom(struct dge_softc *sc)
{
        uint16_t cksum;
        int i;

        cksum = 0;
        for (i = 0; i < EEPROM_SIZE; i++) {
                sc->sc_eeprom[i] = dge_eeprom_word(sc, i);
                cksum += sc->sc_eeprom[i];
        }
        return cksum != EEPROM_CKSUM;
}


/*
 * Read a 16-bit word from address addr in the serial EEPROM.
 */
uint16_t
dge_eeprom_word(struct dge_softc *sc, int addr)
{
        uint32_t reg;
        uint16_t rval = 0;
        int i;

        reg = CSR_READ(sc, DGE_EECD) & ~(EECD_SK | EECD_DI | EECD_CS);

        /* Lower clock pulse (and data in to chip) */
        CSR_WRITE(sc, DGE_EECD, reg);
        /* Select chip */
        CSR_WRITE(sc, DGE_EECD, reg | EECD_CS);

        /* Send read command */
        dge_eeprom_clockout(sc, 1);
        dge_eeprom_clockout(sc, 1);
        dge_eeprom_clockout(sc, 0);

        /* Send address */
        for (i = 5; i >= 0; i--)
                dge_eeprom_clockout(sc, (addr >> i) & 1);

        /* Read data */
        for (i = 0; i < 16; i++) {
                rval <<= 1;
                rval |= dge_eeprom_clockin(sc);
        }

        /* Deselect chip */
        CSR_WRITE(sc, DGE_EECD, reg);

        return rval;
}

/*
 * Clock out a single bit to the EEPROM.
 */
void
dge_eeprom_clockout(struct dge_softc *sc, int bit)
{
        int reg;

        reg = CSR_READ(sc, DGE_EECD) & ~(EECD_DI | EECD_SK);
        if (bit)
                reg |= EECD_DI;

        CSR_WRITE(sc, DGE_EECD, reg);
        delay(2);
        CSR_WRITE(sc, DGE_EECD, reg | EECD_SK);
        delay(2);
        CSR_WRITE(sc, DGE_EECD, reg);
        delay(2);
}

/*
 * Clock in a single bit from EEPROM.
 */
int
dge_eeprom_clockin(struct dge_softc *sc)
{
        int reg, rv;

        reg = CSR_READ(sc, DGE_EECD) & ~(EECD_DI | EECD_DO | EECD_SK);

        CSR_WRITE(sc, DGE_EECD, reg | EECD_SK); /* Raise clock */
        delay(2);
        rv = (CSR_READ(sc, DGE_EECD) & EECD_DO) != 0; /* Get bit */
        CSR_WRITE(sc, DGE_EECD, reg); /* Lower clock */
        delay(2);

        return rv;
}

static void
dge_xgmii_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
{
        struct dge_softc *sc = ifp->if_softc;

        ifmr->ifm_status = IFM_AVALID;
        if (sc->sc_dgep->flags & DGEP_F_10G_SR ) {
                ifmr->ifm_active = IFM_ETHER | IFM_10G_SR;
        } else {
                ifmr->ifm_active = IFM_ETHER | IFM_10G_LR;
        }

        if (CSR_READ(sc, DGE_STATUS) & STATUS_LINKUP)
                ifmr->ifm_status |= IFM_ACTIVE;
}

static inline int
phwait(struct dge_softc *sc, int p, int r, int d, int type)
{
        int i, mdic;

        CSR_WRITE(sc, DGE_MDIO,
            MDIO_PHY(p) | MDIO_REG(r) | MDIO_DEV(d) | type | MDIO_CMD);
        for (i = 0; i < 10; i++) {
                delay(10);
                if (((mdic = CSR_READ(sc, DGE_MDIO)) & MDIO_CMD) == 0)
                        break;
        }
        return mdic;
}

static void
dge_xgmii_writereg(struct dge_softc *sc, int phy, int reg, int val)
{
        int mdic;

        CSR_WRITE(sc, DGE_MDIRW, val);
        if (((mdic = phwait(sc, phy, reg, 1, MDIO_ADDR)) & MDIO_CMD)) {
                printf("%s: address cycle timeout; phy %d reg %d\n",
                    device_xname(sc->sc_dev), phy, reg);
                return;
        }
        if (((mdic = phwait(sc, phy, reg, 1, MDIO_WRITE)) & MDIO_CMD)) {
                printf("%s: write cycle timeout; phy %d reg %d\n",
                    device_xname(sc->sc_dev), phy, reg);
                return;
        }
}

static void
dge_xgmii_reset(struct dge_softc *sc)
{
        dge_xgmii_writereg(sc, 0, 0, BMCR_RESET);
}

static int
dge_xgmii_mediachange(struct ifnet *ifp)
{
        return 0;
}