/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2010-2016 Solarflare Communications Inc.
 * All rights reserved.
 *
 * This software was developed in part by Philip Paeps under contract for
 * Solarflare Communications, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * The views and conclusions contained in the software and documentation are
 * those of the authors and should not be interpreted as representing official
 * policies, either expressed or implied, of the FreeBSD Project.
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/param.h>
#include <sys/queue.h>
#include <sys/systm.h>
#include <sys/taskqueue.h>

#include "common/efx.h"

#include "sfxge.h"

static void
sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
{
        struct sfxge_softc *sc;
        unsigned int index;
        struct sfxge_rxq *rxq;
        struct sfxge_txq *txq;

        SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);

        sc = evq->sc;
        index = evq->index;
        rxq = sc->rxq[index];

        if ((txq = evq->txq) != NULL) {
                evq->txq = NULL;
                evq->txqs = &(evq->txq);

                do {
                        struct sfxge_txq *next;

                        next = txq->next;
                        txq->next = NULL;

                        KASSERT(txq->evq_index == index,
                            ("txq->evq_index != index"));

                        if (txq->pending != txq->completed)
                                sfxge_tx_qcomplete(txq, evq);

                        txq = next;
                } while (txq != NULL);
        }

        if (rxq->pending != rxq->completed)
                sfxge_rx_qcomplete(rxq, eop);
}

static struct sfxge_rxq *
sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label)
{
        struct sfxge_rxq *rxq;

        KASSERT(label == 0, ("unexpected rxq label != 0"));

        rxq = evq->sc->rxq[evq->index];

        KASSERT(rxq != NULL, ("rxq == NULL"));
        KASSERT(evq->index == rxq->index, ("evq->index != rxq->index"));

        return (rxq);
}

static boolean_t
sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
            uint16_t flags)
{
        struct sfxge_evq *evq;
        struct sfxge_softc *sc;
        struct sfxge_rxq *rxq;
        unsigned int stop;
        unsigned int delta;
        struct sfxge_rx_sw_desc *rx_desc;

        evq = arg;
        SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);

        sc = evq->sc;

        if (evq->exception)
                goto done;

        rxq = sfxge_get_rxq_by_label(evq, label);
        if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
                goto done;

        stop = (id + 1) & rxq->ptr_mask;
        id = rxq->pending & rxq->ptr_mask;
        delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
        rxq->pending += delta;

        if (delta != 1) {
                if ((delta <= 0) ||
                    (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
                        evq->exception = B_TRUE;

                        device_printf(sc->dev, "RX completion out of order"
                                                  " (id=%#x delta=%u flags=%#x); resetting\n",
                                                  id, delta, flags);
                        sfxge_schedule_reset(sc);

                        goto done;
                }
        }

        rx_desc = &rxq->queue[id];

        prefetch_read_many(rx_desc->mbuf);

        for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
                rx_desc = &rxq->queue[id];
                KASSERT(rx_desc->flags == EFX_DISCARD,
                                ("rx_desc->flags != EFX_DISCARD"));
                rx_desc->flags = flags;

                KASSERT(size < (1 << 16), ("size > (1 << 16)"));
                rx_desc->size = (uint16_t)size;
        }

        evq->rx_done++;

        if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
                sfxge_ev_qcomplete(evq, B_FALSE);

done:
        return (evq->rx_done >= SFXGE_EV_BATCH);
}

static boolean_t
sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
{
        struct sfxge_evq *evq;
        struct sfxge_softc *sc;

        evq = (struct sfxge_evq *)arg;
        SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);

        sc = evq->sc;

        DBGPRINT(sc->dev, "[%d] %s", evq->index,
                          (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
                          (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
                          (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
                          (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" :
                          (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
                          (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
                          (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
                          (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
                          (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
                          "UNKNOWN");

        evq->exception = B_TRUE;

        if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
                device_printf(sc->dev,
                              "hardware exception (code=%u); resetting\n",
                              code);
                sfxge_schedule_reset(sc);
        }

        return (B_FALSE);
}

static boolean_t
sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
{
        struct sfxge_evq *evq;
        struct sfxge_softc *sc;
        struct sfxge_rxq *rxq;
        unsigned int index;
        uint16_t magic;

        evq = (struct sfxge_evq *)arg;
        SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);

        sc = evq->sc;
        rxq = sc->rxq[rxq_index];

        KASSERT(rxq != NULL, ("rxq == NULL"));

        /* Resend a software event on the correct queue */
        index = rxq->index;
        if (index == evq->index) {
                sfxge_rx_qflush_done(rxq);
                return (B_FALSE);
        }

        evq = sc->evq[index];
        magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq);

        KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
            ("evq not started"));
        efx_ev_qpost(evq->common, magic);

        return (B_FALSE);
}

static boolean_t
sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
{
        struct sfxge_evq *evq;
        struct sfxge_softc *sc;
        struct sfxge_rxq *rxq;
        unsigned int index;
        uint16_t magic;

        evq = (struct sfxge_evq *)arg;
        SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);

        sc = evq->sc;
        rxq = sc->rxq[rxq_index];

        KASSERT(rxq != NULL, ("rxq == NULL"));

        /* Resend a software event on the correct queue */
        index = rxq->index;
        evq = sc->evq[index];
        magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq);

        KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
            ("evq not started"));
        efx_ev_qpost(evq->common, magic);

        return (B_FALSE);
}

static struct sfxge_txq *
sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
{
        unsigned int index;

        KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) :
                ((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
                 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM)),
                ("unexpected txq label"));

        index = (evq->index == 0) ?
                label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc));
        return (evq->sc->txq[index]);
}

static boolean_t
sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
{
        struct sfxge_evq *evq;
        struct sfxge_txq *txq;
        unsigned int stop;
        unsigned int delta;

        evq = (struct sfxge_evq *)arg;
        SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);

        txq = sfxge_get_txq_by_label(evq, label);

        KASSERT(txq != NULL, ("txq == NULL"));
        KASSERT(evq->index == txq->evq_index,
            ("evq->index != txq->evq_index"));

        if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
                goto done;

        stop = (id + 1) & txq->ptr_mask;
        id = txq->pending & txq->ptr_mask;

        delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
        txq->pending += delta;

        evq->tx_done++;

        if (txq->next == NULL &&
            evq->txqs != &(txq->next)) {
                *(evq->txqs) = txq;
                evq->txqs = &(txq->next);
        }

        if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
                sfxge_tx_qcomplete(txq, evq);

done:
        return (evq->tx_done >= SFXGE_EV_BATCH);
}

static boolean_t
sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
{
        struct sfxge_evq *evq;
        struct sfxge_softc *sc;
        struct sfxge_txq *txq;
        uint16_t magic;

        evq = (struct sfxge_evq *)arg;
        SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);

        sc = evq->sc;
        txq = sc->txq[txq_index];

        KASSERT(txq != NULL, ("txq == NULL"));
        KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
            ("txq not initialized"));

        if (txq->evq_index == evq->index) {
                sfxge_tx_qflush_done(txq);
                return (B_FALSE);
        }

        /* Resend a software event on the correct queue */
        evq = sc->evq[txq->evq_index];
        magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq);

        KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
            ("evq not started"));
        efx_ev_qpost(evq->common, magic);

        return (B_FALSE);
}

static boolean_t
sfxge_ev_software(void *arg, uint16_t magic)
{
        struct sfxge_evq *evq;
        unsigned int label;

        evq = (struct sfxge_evq *)arg;
        SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);

        label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
        magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;

        switch (magic) {
        case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE):
                sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label));
                break;

        case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED):
                sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label));
                break;

        case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL):
                sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label));
                break;

        case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): {
                struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);

                KASSERT(txq != NULL, ("txq == NULL"));
                KASSERT(evq->index == txq->evq_index,
                    ("evq->index != txq->evq_index"));

                sfxge_tx_qflush_done(txq);
                break;
        }
        default:
                break;
        }

        return (B_FALSE);
}

static boolean_t
sfxge_ev_sram(void *arg, uint32_t code)
{
        (void)arg;
        (void)code;

        switch (code) {
        case EFX_SRAM_UPDATE:
                EFSYS_PROBE(sram_update);
                break;

        case EFX_SRAM_CLEAR:
                EFSYS_PROBE(sram_clear);
                break;

        case EFX_SRAM_ILLEGAL_CLEAR:
                EFSYS_PROBE(sram_illegal_clear);
                break;

        default:
                KASSERT(B_FALSE, ("Impossible SRAM event"));
                break;
        }

        return (B_FALSE);
}

static boolean_t
sfxge_ev_timer(void *arg, uint32_t index)
{
        (void)arg;
        (void)index;

        return (B_FALSE);
}

static boolean_t
sfxge_ev_wake_up(void *arg, uint32_t index)
{
        (void)arg;
        (void)index;

        return (B_FALSE);
}

#if EFSYS_OPT_QSTATS

static void
sfxge_evq_stat_update(struct sfxge_evq *evq)
{
        clock_t now;

        SFXGE_EVQ_LOCK(evq);

        if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED))
                goto out;

        now = ticks;
        if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz)
                goto out;

        evq->stats_update_time = now;
        efx_ev_qstats_update(evq->common, evq->stats);

out:
        SFXGE_EVQ_UNLOCK(evq);
}

static int
sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS)
{
        struct sfxge_evq *evq = arg1;
        struct sfxge_softc *sc = evq->sc;
        unsigned int id = arg2;

        SFXGE_ADAPTER_LOCK(sc);

        sfxge_evq_stat_update(evq);

        SFXGE_ADAPTER_UNLOCK(sc);

        return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id])));
}

static int
sfxge_evq_stat_init(struct sfxge_evq *evq)
{
        struct sfxge_softc *sc = evq->sc;
        struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
        char name[16];
        struct sysctl_oid *evq_stats_node;
        unsigned int id;

        snprintf(name, sizeof(name), "%u", evq->index);
        evq_stats_node = SYSCTL_ADD_NODE(ctx,
            SYSCTL_CHILDREN(sc->evqs_stats_node), OID_AUTO, name,
            CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
        if (evq_stats_node == NULL)
                return (ENOMEM);

        for (id = 0; id < EV_NQSTATS; id++) {
                SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(evq_stats_node),
                    OID_AUTO, efx_ev_qstat_name(sc->enp, id),
                    CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, evq, id,
                    sfxge_evq_stat_handler, "Q", "");
        }

        return (0);
}

static void
sfxge_ev_stat_update(struct sfxge_softc *sc)
{
        struct sfxge_evq *evq;
        unsigned int index;
        clock_t now;
        unsigned int id;

        SFXGE_ADAPTER_LOCK(sc);

        now = ticks;
        if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz)
                goto out;

        sc->ev_stats_update_time = now;

        memset(sc->ev_stats, 0, sizeof(sc->ev_stats));

        /* Update and add event counts from each event queue in turn */
        for (index = 0; index < sc->evq_count; index++) {
                evq = sc->evq[index];
                sfxge_evq_stat_update(evq);
                for (id = 0; id < EV_NQSTATS; id++)
                        sc->ev_stats[id] += evq->stats[id];
        }
out:
        SFXGE_ADAPTER_UNLOCK(sc);
}

static int
sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
{
        struct sfxge_softc *sc = arg1;
        unsigned int id = arg2;

        sfxge_ev_stat_update(sc);

        return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
}

static void
sfxge_ev_stat_init(struct sfxge_softc *sc)
{
        struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
        struct sysctl_oid_list *stat_list;
        unsigned int id;
        char name[40];

        stat_list = SYSCTL_CHILDREN(sc->stats_node);

        for (id = 0; id < EV_NQSTATS; id++) {
                snprintf(name, sizeof(name), "ev_%s",
                         efx_ev_qstat_name(sc->enp, id));
                SYSCTL_ADD_PROC(ctx, stat_list, OID_AUTO, name,
                    CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
                    sc, id, sfxge_ev_stat_handler, "Q", "");
        }
}

#endif /* EFSYS_OPT_QSTATS */

static void
sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
{
        struct sfxge_evq *evq;
        efx_evq_t *eep;

        evq = sc->evq[idx];
        eep = evq->common;

        KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
            ("evq->init_state != SFXGE_EVQ_STARTED"));

        (void)efx_ev_qmoderate(eep, us);
}

static int
sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
{
        struct sfxge_softc *sc = arg1;
        struct sfxge_intr *intr = &sc->intr;
        unsigned int moderation;
        int error;
        unsigned int index;

        SFXGE_ADAPTER_LOCK(sc);

        if (req->newptr != NULL) {
                if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
                    != 0)
                        goto out;

                /* We may not be calling efx_ev_qmoderate() now,
                 * so we have to range-check the value ourselves.
                 */
                if (moderation >
                    efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
                        error = EINVAL;
                        goto out;
                }

                sc->ev_moderation = moderation;
                if (intr->state == SFXGE_INTR_STARTED) {
                        for (index = 0; index < sc->evq_count; index++)
                                sfxge_ev_qmoderate(sc, index, moderation);
                }
        } else {
                error = SYSCTL_OUT(req, &sc->ev_moderation,
                                   sizeof(sc->ev_moderation));
        }

out:
        SFXGE_ADAPTER_UNLOCK(sc);

        return (error);
}

static boolean_t
sfxge_ev_initialized(void *arg)
{
        struct sfxge_evq *evq;

        evq = (struct sfxge_evq *)arg;
        SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);

        /* Init done events may be duplicated on 7xxx */
        KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
                evq->init_state == SFXGE_EVQ_STARTED,
            ("evq not starting"));

        evq->init_state = SFXGE_EVQ_STARTED;

        return (0);
}

static boolean_t
sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
{
        struct sfxge_evq *evq;
        struct sfxge_softc *sc;

        evq = (struct sfxge_evq *)arg;
        SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);

        sc = evq->sc;

        sfxge_mac_link_update(sc, link_mode);

        return (0);
}

static const efx_ev_callbacks_t sfxge_ev_callbacks = {
        .eec_initialized        = sfxge_ev_initialized,
        .eec_rx                 = sfxge_ev_rx,
        .eec_tx                 = sfxge_ev_tx,
        .eec_exception          = sfxge_ev_exception,
        .eec_rxq_flush_done     = sfxge_ev_rxq_flush_done,
        .eec_rxq_flush_failed   = sfxge_ev_rxq_flush_failed,
        .eec_txq_flush_done     = sfxge_ev_txq_flush_done,
        .eec_software           = sfxge_ev_software,
        .eec_sram               = sfxge_ev_sram,
        .eec_wake_up            = sfxge_ev_wake_up,
        .eec_timer              = sfxge_ev_timer,
        .eec_link_change        = sfxge_ev_link_change,
};

int
sfxge_ev_qpoll(struct sfxge_evq *evq)
{
        int rc;

        SFXGE_EVQ_LOCK(evq);

        if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
                            evq->init_state != SFXGE_EVQ_STARTED)) {
                rc = EINVAL;
                goto fail;
        }

        /* Synchronize the DMA memory for reading */
        bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
            BUS_DMASYNC_POSTREAD);

        KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
        KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
        KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
        KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));

        /* Poll the queue */
        efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);

        evq->rx_done = 0;
        evq->tx_done = 0;

        /* Perform any pending completion processing */
        sfxge_ev_qcomplete(evq, B_TRUE);

        /* Re-prime the event queue for interrupts */
        if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
                goto fail;

        SFXGE_EVQ_UNLOCK(evq);

        return (0);

fail:
        SFXGE_EVQ_UNLOCK(evq);
        return (rc);
}

static void
sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
{
        struct sfxge_evq *evq;

        evq = sc->evq[index];

        KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
            ("evq->init_state != SFXGE_EVQ_STARTED"));

        SFXGE_EVQ_LOCK(evq);
        evq->init_state = SFXGE_EVQ_INITIALIZED;
        evq->read_ptr = 0;
        evq->exception = B_FALSE;

#if EFSYS_OPT_QSTATS
        /* Add event counts before discarding the common evq state */
        efx_ev_qstats_update(evq->common, evq->stats);
#endif

        efx_ev_qdestroy(evq->common);
        efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
            EFX_EVQ_NBUFS(evq->entries));
        SFXGE_EVQ_UNLOCK(evq);
}

static int
sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
{
        struct sfxge_evq *evq;
        efsys_mem_t *esmp;
        int count;
        int rc;

        evq = sc->evq[index];
        esmp = &evq->mem;

        KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
            ("evq->init_state != SFXGE_EVQ_INITIALIZED"));

        /* Clear all events. */
        (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));

        /* Program the buffer table. */
        if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
            EFX_EVQ_NBUFS(evq->entries))) != 0)
                return (rc);

        /* Create the common code event queue. */
        if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
            evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO,
            &evq->common)) != 0)
                goto fail;

        SFXGE_EVQ_LOCK(evq);

        /* Prime the event queue for interrupts */
        if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
                goto fail2;

        evq->init_state = SFXGE_EVQ_STARTING;

        SFXGE_EVQ_UNLOCK(evq);

        /* Wait for the initialization event */
        count = 0;
        do {
                /* Pause for 100 ms */
                pause("sfxge evq init", hz / 10);

                /* Check to see if the test event has been processed */
                if (evq->init_state == SFXGE_EVQ_STARTED)
                        goto done;

        } while (++count < 20);

        rc = ETIMEDOUT;
        goto fail3;

done:
        return (0);

fail3:
        SFXGE_EVQ_LOCK(evq);
        evq->init_state = SFXGE_EVQ_INITIALIZED;
fail2:
        SFXGE_EVQ_UNLOCK(evq);
        efx_ev_qdestroy(evq->common);
fail:
        efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
            EFX_EVQ_NBUFS(evq->entries));

        return (rc);
}

void
sfxge_ev_stop(struct sfxge_softc *sc)
{
        struct sfxge_intr *intr __diagused;
        efx_nic_t *enp;
        int index;

        intr = &sc->intr;
        enp = sc->enp;

        KASSERT(intr->state == SFXGE_INTR_STARTED,
            ("Interrupts not started"));

        /* Stop the event queue(s) */
        index = sc->evq_count;
        while (--index >= 0)
                sfxge_ev_qstop(sc, index);

        /* Tear down the event module */
        efx_ev_fini(enp);
}

int
sfxge_ev_start(struct sfxge_softc *sc)
{
        struct sfxge_intr *intr __diagused;
        int index;
        int rc;

        intr = &sc->intr;

        KASSERT(intr->state == SFXGE_INTR_STARTED,
            ("intr->state != SFXGE_INTR_STARTED"));

        /* Initialize the event module */
        if ((rc = efx_ev_init(sc->enp)) != 0)
                return (rc);

        /* Start the event queues */
        for (index = 0; index < sc->evq_count; index++) {
                if ((rc = sfxge_ev_qstart(sc, index)) != 0)
                        goto fail;
        }

        return (0);

fail:
        /* Stop the event queue(s) */
        while (--index >= 0)
                sfxge_ev_qstop(sc, index);

        /* Tear down the event module */
        efx_ev_fini(sc->enp);

        return (rc);
}

static void
sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
{
        struct sfxge_evq *evq;

        evq = sc->evq[index];

        KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
            ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
        KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));

        sfxge_dma_free(&evq->mem);

        sc->evq[index] = NULL;

        SFXGE_EVQ_LOCK_DESTROY(evq);

        free(evq, M_SFXGE);
}

static int
sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
{
        struct sfxge_evq *evq;
        efsys_mem_t *esmp;
        int rc;

        KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));

        evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
        evq->sc = sc;
        evq->index = index;
        sc->evq[index] = evq;
        esmp = &evq->mem;

        /* Build an event queue with room for one event per tx and rx buffer,
         * plus some extra for link state events and MCDI completions.
         * There are three tx queues in the first event queue and one in
         * other.
         */
        if (index == 0)
                evq->entries =
                        ROUNDUP_POW_OF_TWO(sc->rxq_entries +
                                           3 * sc->txq_entries +
                                           128);
        else
                evq->entries =
                        ROUNDUP_POW_OF_TWO(sc->rxq_entries +
                                           sc->txq_entries +
                                           128);

        /* Initialise TX completion list */
        evq->txqs = &evq->txq;

        /* Allocate DMA space. */
        if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
                return (rc);

        /* Allocate buffer table entries. */
        sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
                                 &evq->buf_base_id);

        SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);

        evq->init_state = SFXGE_EVQ_INITIALIZED;

#if EFSYS_OPT_QSTATS
        rc = sfxge_evq_stat_init(evq);
        if (rc != 0)
                goto fail_evq_stat_init;
#endif

        return (0);

#if EFSYS_OPT_QSTATS
fail_evq_stat_init:
        evq->init_state = SFXGE_EVQ_UNINITIALIZED;
        SFXGE_EVQ_LOCK_DESTROY(evq);
        sfxge_dma_free(esmp);
        sc->evq[index] = NULL;
        free(evq, M_SFXGE);

        return (rc);
#endif
}

void
sfxge_ev_fini(struct sfxge_softc *sc)
{
        struct sfxge_intr *intr __diagused;
        int index;

        intr = &sc->intr;

        KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
            ("intr->state != SFXGE_INTR_INITIALIZED"));

        sc->ev_moderation = 0;

        /* Tear down the event queue(s). */
        index = sc->evq_count;
        while (--index >= 0)
                sfxge_ev_qfini(sc, index);

        sc->evq_count = 0;
}

int
sfxge_ev_init(struct sfxge_softc *sc)
{
        struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
        struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
        struct sfxge_intr *intr;
        int index;
        int rc;

        intr = &sc->intr;

        sc->evq_count = intr->n_alloc;

        KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
            ("intr->state != SFXGE_INTR_INITIALIZED"));

        /* Set default interrupt moderation; add a sysctl to
         * read and change it.
         */
        sc->ev_moderation = SFXGE_MODERATION;
        SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
            OID_AUTO, "int_mod", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
            sc, 0, sfxge_int_mod_handler, "IU",
            "sfxge interrupt moderation (us)");

#if EFSYS_OPT_QSTATS
        sc->evqs_stats_node = SYSCTL_ADD_NODE(
            device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node),
            OID_AUTO, "evq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
            "Event queues stats");
        if (sc->evqs_stats_node == NULL) {
                rc = ENOMEM;
                goto fail_evqs_stats_node;
        }
#endif

        /*
         * Initialize the event queue(s) - one per interrupt.
         */
        for (index = 0; index < sc->evq_count; index++) {
                if ((rc = sfxge_ev_qinit(sc, index)) != 0)
                        goto fail;
        }

#if EFSYS_OPT_QSTATS
        sfxge_ev_stat_init(sc);
#endif

        return (0);

fail:
        while (--index >= 0)
                sfxge_ev_qfini(sc, index);

#if EFSYS_OPT_QSTATS
fail_evqs_stats_node:
#endif
        sc->evq_count = 0;
        return (rc);
}