/*
 * Copyright (c) 2008-2016 Solarflare Communications Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * The views and conclusions contained in the software and documentation are
 * those of the authors and should not be interpreted as representing official
 * policies, either expressed or implied, of the FreeBSD Project.
 */

#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/atomic.h>
#include <sys/stream.h>
#include <sys/strsun.h>
#include <sys/strsubr.h>
#include <sys/strft.h>
#include <sys/ksynch.h>
#include <sys/ethernet.h>
#include <sys/crc32.h>
#include <sys/pattr.h>
#include <sys/cpu.h>

#include <sys/ethernet.h>
#include <inet/ip.h>

#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/tcp.h>

#include "sfxge.h"

#include "efx.h"

/* RXQ flush response timeout (in microseconds) */
#define SFXGE_RX_QFLUSH_USEC    (2000000)

/* RXQ flush tries in the case of failure */
#define SFXGE_RX_QFLUSH_TRIES   (5)

/* RXQ default packet buffer preallocation (number of packet buffers) */
#define SFXGE_RX_QPREALLOC      (0)

/* Receive packet DMA attributes */
static ddi_device_acc_attr_t sfxge_rx_packet_devacc = {

        DDI_DEVICE_ATTR_V0,     /* devacc_attr_version */
        DDI_NEVERSWAP_ACC,      /* devacc_attr_endian_flags */
        DDI_STRICTORDER_ACC     /* devacc_attr_dataorder */
};

static ddi_dma_attr_t sfxge_rx_packet_dma_attr = {
        DMA_ATTR_V0,            /* dma_attr_version     */
        0,                      /* dma_attr_addr_lo     */
        0xffffffffffffffffull,  /* dma_attr_addr_hi     */
        0xffffffffffffffffull,  /* dma_attr_count_max   */
        SFXGE_CPU_CACHE_SIZE,   /* dma_attr_align       */
        0xffffffff,             /* dma_attr_burstsizes  */
        1,                      /* dma_attr_minxfer     */
        0xffffffffffffffffull,  /* dma_attr_maxxfer     */
        0xffffffffffffffffull,  /* dma_attr_seg         */
        1,                      /* dma_attr_sgllen      */
        1,                      /* dma_attr_granular    */
        0                       /* dma_attr_flags       */
};

/* Receive queue DMA attributes */
static ddi_device_acc_attr_t sfxge_rxq_devacc = {

        DDI_DEVICE_ATTR_V0,     /* devacc_attr_version */
        DDI_NEVERSWAP_ACC,      /* devacc_attr_endian_flags */
        DDI_STRICTORDER_ACC     /* devacc_attr_dataorder */
};

static ddi_dma_attr_t sfxge_rxq_dma_attr = {
        DMA_ATTR_V0,            /* dma_attr_version     */
        0,                      /* dma_attr_addr_lo     */
        0xffffffffffffffffull,  /* dma_attr_addr_hi     */
        0xffffffffffffffffull,  /* dma_attr_count_max   */
        EFX_BUF_SIZE,           /* dma_attr_align       */
        0xffffffff,             /* dma_attr_burstsizes  */
        1,                      /* dma_attr_minxfer     */
        0xffffffffffffffffull,  /* dma_attr_maxxfer     */
        0xffffffffffffffffull,  /* dma_attr_seg         */
        1,                      /* dma_attr_sgllen      */
        1,                      /* dma_attr_granular    */
        0                       /* dma_attr_flags       */
};

/* Forward declaration */
static void sfxge_rx_qpreallocate(sfxge_rxq_t *srp, int nprealloc);

static int
sfxge_rx_packet_ctor(void *buf, void *arg, int kmflags)
{
        sfxge_rx_packet_t *srpp = buf;
        sfxge_t *sp = arg;
        dev_info_t *dip = sp->s_dip;
        int err;

        ASSERT3U(sizeof (srpp->__srp_u1.__srp_s1), <=,
            sizeof (srpp->__srp_u1.__srp_pad));
        ASSERT3U(sizeof (srpp->__srp_u2.__srp_s2), <=,
            sizeof (srpp->__srp_u2.__srp_pad));

        bzero(buf, sizeof (sfxge_rx_packet_t));

        /* Allocate a DMA handle */
        err = ddi_dma_alloc_handle(dip, &sfxge_rx_packet_dma_attr,
            (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
            NULL, &(srpp->srp_dma_handle));
        if (err != DDI_SUCCESS)
                goto fail1;

        return (0);

fail1:
        DTRACE_PROBE1(fail1, int, err);

        SFXGE_OBJ_CHECK(srpp, sfxge_rx_packet_t);

        return (-1);
}

static void
sfxge_rx_packet_dtor(void *buf, void *arg)
{
        sfxge_rx_packet_t *srpp = buf;

        _NOTE(ARGUNUSED(arg))

        /* Free the DMA handle */
        ddi_dma_free_handle(&(srpp->srp_dma_handle));
        srpp->srp_dma_handle = NULL;

        SFXGE_OBJ_CHECK(srpp, sfxge_rx_packet_t);
}

static int
sfxge_rx_qctor(void *buf, void *arg, int kmflags)
{
        sfxge_rxq_t *srp = buf;
        efsys_mem_t *esmp = &(srp->sr_mem);
        sfxge_t *sp = arg;
        sfxge_dma_buffer_attr_t dma_attr;
        sfxge_rx_fpp_t *srfppp;
        int nprealloc;
        unsigned int id;
        int rc;

        /* Compile-time structure layout checks */
        EFX_STATIC_ASSERT(sizeof (srp->__sr_u1.__sr_s1) <=
            sizeof (srp->__sr_u1.__sr_pad));
        EFX_STATIC_ASSERT(sizeof (srp->__sr_u2.__sr_s2) <=
            sizeof (srp->__sr_u2.__sr_pad));
        EFX_STATIC_ASSERT(sizeof (srp->__sr_u3.__sr_s3) <=
            sizeof (srp->__sr_u3.__sr_pad));

        bzero(buf, sizeof (sfxge_rxq_t));

        srp->sr_sp = sp;

        dma_attr.sdba_dip        = sp->s_dip;
        dma_attr.sdba_dattrp     = &sfxge_rxq_dma_attr;
        dma_attr.sdba_callback   = DDI_DMA_SLEEP;
        dma_attr.sdba_length     = EFX_RXQ_SIZE(sp->s_rxq_size);
        dma_attr.sdba_memflags   = DDI_DMA_CONSISTENT;
        dma_attr.sdba_devaccp    = &sfxge_rxq_devacc;
        dma_attr.sdba_bindflags  = DDI_DMA_READ | DDI_DMA_CONSISTENT;
        dma_attr.sdba_maxcookies = 1;
        dma_attr.sdba_zeroinit   = B_FALSE;

        if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
                goto fail1;

        /* Allocate some buffer table entries */
        if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_RXQ_NBUFS(sp->s_rxq_size),
            &(srp->sr_id))) != 0)
                goto fail2;

        /* Allocate the context array */
        if ((srp->sr_srpp = kmem_zalloc(sizeof (sfxge_rx_packet_t *) *
            sp->s_rxq_size, kmflags)) == NULL) {
                rc = ENOMEM;
                goto fail3;
        }

        /* Allocate the flow table */
        if ((srp->sr_flow = kmem_zalloc(sizeof (sfxge_rx_flow_t) *
            SFXGE_MAX_FLOW, kmflags)) == NULL) {
                rc = ENOMEM;
                goto fail4;
        }

        srp->sr_srfpp = &(srp->sr_srfp);
        srp->sr_rto = drv_usectohz(200000);

        srp->sr_mpp = &(srp->sr_mp);

        /* Initialize the free packet pool */
        srfppp = &(srp->sr_fpp);
        if ((srfppp->srfpp_putp = kmem_zalloc(SFXGE_CPU_CACHE_SIZE *
            SFXGE_RX_FPP_NSLOTS, kmflags)) == NULL) {
                rc = ENOMEM;
                goto fail5;
        }
        for (id = 0; id < SFXGE_RX_FPP_NSLOTS; id++) {
                sfxge_rx_fpp_putlist_t *putp;
                size_t off;

                off = id * SFXGE_CPU_CACHE_SIZE;
                putp = (void *)(srfppp->srfpp_putp + off);

                putp->srfpl_putp = NULL;
                putp->srfpl_putpp = &(putp->srfpl_putp);
                mutex_init(&(putp->srfpl_lock), NULL, MUTEX_DRIVER,
                    DDI_INTR_PRI(sp->s_intr.si_intr_pri));
        }

        cv_init(&(srp->sr_flush_kv), NULL, CV_DRIVER, NULL);

        /* Preallocate some packets on the free packet pool */
        nprealloc = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
            DDI_PROP_DONTPASS, "rx_prealloc_pkt_buffers", SFXGE_RX_QPREALLOC);
        sfxge_rx_qpreallocate(srp, nprealloc);


        return (0);

fail5:
        DTRACE_PROBE(fail5);

        srp->sr_mpp = NULL;

        srp->sr_rto = 0;
        srp->sr_srfpp = NULL;

        /* Free the flow table */
        kmem_free(srp->sr_flow, sizeof (sfxge_rx_flow_t) *
            SFXGE_MAX_FLOW);
        srp->sr_flow = NULL;

fail4:
        DTRACE_PROBE(fail4);

        /* Free the context array */
        kmem_free(srp->sr_srpp, sizeof (sfxge_rx_packet_t *) *
            sp->s_rxq_size);
        srp->sr_srpp = NULL;

fail3:
        DTRACE_PROBE(fail3);

        /* Free the buffer table entries */
        sfxge_sram_buf_tbl_free(sp, srp->sr_id,
            EFX_RXQ_NBUFS(sp->s_rxq_size));
        srp->sr_id = 0;

fail2:
        DTRACE_PROBE(fail2);
        /* Remove dma setup */
        sfxge_dma_buffer_destroy(esmp);

fail1:
        DTRACE_PROBE1(fail1, int, rc);

        srp->sr_sp = NULL;

        SFXGE_OBJ_CHECK(srp, sfxge_rxq_t);

        return (-1);
}

static void
sfxge_rx_qdtor(void *buf, void *arg)
{
        sfxge_rxq_t *srp = buf;
        efsys_mem_t *esmp = &(srp->sr_mem);
        sfxge_t *sp = srp->sr_sp;
        sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
        unsigned int id;

        _NOTE(ARGUNUSED(arg))

        cv_destroy(&(srp->sr_flush_kv));

        /* Tear down the free packet pool */
        for (id = 0; id < SFXGE_RX_FPP_NSLOTS; id++) {
                sfxge_rx_fpp_putlist_t *putp;
                size_t off;

                off = id * SFXGE_CPU_CACHE_SIZE;
                putp = (void *)(srfppp->srfpp_putp + off);

                putp->srfpl_putpp = NULL;
                mutex_destroy(&(putp->srfpl_lock));

                SFXGE_OBJ_CHECK(putp, sfxge_rx_fpp_putlist_t);
        }
        kmem_free(srfppp->srfpp_putp, SFXGE_CPU_CACHE_SIZE *
            SFXGE_RX_FPP_NSLOTS);
        srfppp->srfpp_putp = NULL;

        srp->sr_mpp = NULL;

        srp->sr_rto = 0;
        srp->sr_srfpp = NULL;

        /* Free the flow table */
        kmem_free(srp->sr_flow, sizeof (sfxge_rx_flow_t) *
            SFXGE_MAX_FLOW);
        srp->sr_flow = NULL;

        /* Free the context array */
        kmem_free(srp->sr_srpp, sizeof (sfxge_rx_packet_t *) *
            sp->s_rxq_size);
        srp->sr_srpp = NULL;

        /* Free the buffer table entries */
        sfxge_sram_buf_tbl_free(sp, srp->sr_id,
            EFX_RXQ_NBUFS(sp->s_rxq_size));
        srp->sr_id = 0;

        /* Tear down dma setup */
        sfxge_dma_buffer_destroy(esmp);

        SFXGE_OBJ_CHECK(srp, sfxge_rxq_t);
}

/* Note: This function takes ownership of *srpp. */
static inline void
sfxge_rx_qfpp_put(sfxge_rxq_t *srp, sfxge_rx_packet_t *srpp)
{
        sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
        mblk_t *mp = srpp->srp_mp;
        unsigned int id;
        size_t off;
        sfxge_rx_fpp_putlist_t *putp;

        ASSERT3P(mp->b_next, ==, NULL);
        ASSERT3P(mp->b_prev, ==, NULL);

        id = CPU->cpu_seqid & SFXGE_RX_FPP_MASK;
        off = id * SFXGE_CPU_CACHE_SIZE;

        ASSERT3P(srpp->srp_putp, ==, srfppp->srfpp_putp);
        putp = (void *)(srpp->srp_putp + off);

        mutex_enter(&(putp->srfpl_lock));
        putp->srfpl_count++;
        *putp->srfpl_putpp = mp;
        putp->srfpl_putpp = &(mp->b_next);
        mutex_exit(&(putp->srfpl_lock));
}

static unsigned int
sfxge_rx_qfpp_swizzle(sfxge_rxq_t *srp)
{
        sfxge_t *sp = srp->sr_sp;
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];
        sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
        unsigned int start;
        unsigned int id;
        mblk_t *p;
        mblk_t **pp;
        unsigned int count;
        unsigned int loaned;

        ASSERT(mutex_owned(&(sep->se_lock)));

        /* We want to access the put list for the current CPU last */
        id = start = (CPU->cpu_seqid + 1) & SFXGE_RX_FPP_MASK;

        do {
                sfxge_rx_fpp_putlist_t *putp;
                size_t off;

                off = id * SFXGE_CPU_CACHE_SIZE;
                id  = (id + 1) & SFXGE_RX_FPP_MASK;

                putp = (void *)(srfppp->srfpp_putp + off);

                /* Acquire the put list */
                mutex_enter(&(putp->srfpl_lock));

                p = putp->srfpl_putp;
                pp = putp->srfpl_putpp;
                count = putp->srfpl_count;

                putp->srfpl_putp = NULL;
                putp->srfpl_putpp = &(putp->srfpl_putp);
                putp->srfpl_count = 0;

                mutex_exit(&(putp->srfpl_lock));

                if (p == NULL)
                        continue;

                /* Add the list to the head of the get list */
                *pp = srfppp->srfpp_get;
                srfppp->srfpp_get = p;

                /* Adjust the counters */
                ASSERT3U(srfppp->srfpp_loaned, >=, count);
                srfppp->srfpp_loaned -= count;
                srfppp->srfpp_count += count;

#if 0
                /* NOTE: this probe is disabled because it is expensive!! */
                DTRACE_PROBE2(count,
                    unsigned int, (id - 1) & SFXGE_RX_FPP_MASK,
                    unsigned int, count);
#endif

        } while (id != start);

        /* Return the number of packets yet to appear in the put list */
        loaned = srfppp->srfpp_loaned;


        return (loaned);
}


#define DB_FRTNP(mp)    ((mp)->b_datap->db_frtnp)

static void
sfxge_rx_qfpp_empty(sfxge_rxq_t *srp)
{
        sfxge_t *sp = srp->sr_sp;
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];
        sfxge_rx_fpp_t *srfppp;
        mblk_t *mp;

        mutex_enter(&(sep->se_lock));
        srfppp = &(srp->sr_fpp);

        /* Swizzle put list to get list */
        (void) sfxge_rx_qfpp_swizzle(srp);
        ASSERT3U(srfppp->srfpp_loaned, ==, 0);

        mp = srfppp->srfpp_get;
        srfppp->srfpp_get = NULL;

        /* Free the remainder */
        while (mp != NULL) {
                mblk_t *next;
                frtn_t *freep;
                sfxge_rx_packet_t *srpp;

                next = mp->b_next;
                mp->b_next = NULL;

                ASSERT3U(srfppp->srfpp_count, >, 0);
                srfppp->srfpp_count--;

                freep = DB_FRTNP(mp);
                /*
                 * ASSERT3P(freep->free_func, ==, sfxge_rx_qpacket_free);
                 *   is implied by srpp test below
                 */
                /*LINTED*/
                srpp = (sfxge_rx_packet_t *)(freep->free_arg);
                ASSERT3P(srpp->srp_mp, ==, mp);
                ASSERT3P(mp->b_cont, ==, NULL);
                srpp->srp_recycle = B_FALSE;

                freeb(mp);

                mp = next;
        }
        ASSERT3U(srfppp->srfpp_count, ==, 0);

        srfppp->srfpp_min = 0;

        mutex_exit(&(sep->se_lock));
}

/*
 * This is an estimate of all memory consumed per RX packet
 * it can be inaccurate but but sp->s_rx_pkt_mem_alloc mustn't drift
 */
static uint64_t
sfxge_rx_pkt_mem_approx(const sfxge_rx_packet_t *srpp)
{
        return (srpp->srp_mblksize + sizeof (mblk_t) + sizeof (dblk_t) +
            sizeof (sfxge_rx_packet_t));
}

static void
sfxge_rx_qpacket_destroy(sfxge_rxq_t *srp, sfxge_rx_packet_t *srpp)
{
        sfxge_t *sp = srp->sr_sp;
        int64_t delta = sfxge_rx_pkt_mem_approx(srpp);

        ASSERT(!(srpp->srp_recycle));
        ASSERT3P(srpp->srp_mp, ==, NULL);

        srpp->srp_off = 0;
        srpp->srp_thp = NULL;
        srpp->srp_iphp = NULL;
        srpp->srp_etherhp = NULL;
        srpp->srp_size = 0;
        srpp->srp_flags = 0;

        bzero(&(srpp->srp_free), sizeof (frtn_t));

        srpp->srp_mblksize = 0;
        srpp->srp_base = NULL;

        /* Unbind the DMA memory from the DMA handle */
        srpp->srp_addr = 0;
        (void) ddi_dma_unbind_handle(srpp->srp_dma_handle);

        /* Free the DMA memory */
        srpp->srp_base = NULL;
        ddi_dma_mem_free(&(srpp->srp_acc_handle));
        srpp->srp_acc_handle = NULL;

        srpp->srp_putp = NULL;
        srpp->srp_srp = NULL;

        kmem_cache_free(sp->s_rpc, srpp);
        if (sp->s_rx_pkt_mem_max)
                atomic_add_64(&sp->s_rx_pkt_mem_alloc, -delta);
}

static void
sfxge_rx_qpacket_free(void *arg)
{
        sfxge_rx_packet_t *srpp = arg;
        sfxge_rxq_t *srp = srpp->srp_srp;

        /*
         * WARNING "man -s 9f esballoc"  states:
         * => runs sync from the thread calling freeb()
         * => must not sleep, or access data structures that could be freed
         */

        /* Check whether we want to recycle the receive packets */
        if (srpp->srp_recycle) {
                frtn_t *freep;
                mblk_t *mp;
                size_t size;

                freep = &(srpp->srp_free);
                ASSERT3P(freep->free_func, ==, sfxge_rx_qpacket_free);
                ASSERT3P(freep->free_arg, ==, (caddr_t)srpp);

                /*
                 * Allocate a matching mblk_t before the current one is
                 * freed.
                 */
                size = srpp->srp_mblksize;

                if ((mp = desballoc(srpp->srp_base, size, BPRI_HI,
                    freep)) != NULL) {
                        srpp->srp_mp = mp;

                        /* NORMAL recycled case */
                        sfxge_rx_qfpp_put(srp, srpp);
                        return;
                }
        }

        srpp->srp_mp = NULL;

        sfxge_rx_qpacket_destroy(srp, srpp);
}

static sfxge_rx_packet_t *
sfxge_rx_qpacket_create(sfxge_rxq_t *srp)
{
        sfxge_t *sp = srp->sr_sp;
        sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
        sfxge_rx_packet_t *srpp;
        size_t size;
        caddr_t base;
        size_t unit;
        ddi_dma_cookie_t dmac;
        unsigned int ncookies;
        frtn_t *freep;
        mblk_t *mp;
        int err;
        int rc;

        size = sp->s_rx_buffer_size;

        if (sp->s_rx_pkt_mem_max &&
            (sp->s_rx_pkt_mem_alloc + size >= sp->s_rx_pkt_mem_max)) {
                DTRACE_PROBE(rx_pkt_mem_max);
                srp->sr_kstat.srk_rx_pkt_mem_limit++;
                return (NULL);
        }

        /* Allocate a new packet */
        if ((srpp = kmem_cache_alloc(sp->s_rpc, KM_NOSLEEP)) == NULL) {
                srp->sr_kstat.srk_kcache_alloc_nomem++;
                rc = ENOMEM;
                goto fail1;
        }

        srpp->srp_srp = srp;
        srpp->srp_putp = srfppp->srfpp_putp;

        /* Allocate some DMA memory */
        err = ddi_dma_mem_alloc(srpp->srp_dma_handle, size,
            &sfxge_rx_packet_devacc, DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
            NULL, &base, &unit, &(srpp->srp_acc_handle));
        switch (err) {
        case DDI_SUCCESS:
                break;

        case DDI_FAILURE:
                srp->sr_kstat.srk_dma_alloc_nomem++;
                rc = ENOMEM;
                goto fail2;

        default:
                srp->sr_kstat.srk_dma_alloc_fail++;
                rc = EFAULT;
                goto fail2;
        }

        /* Adjust the buffer to align the start of the DMA area correctly */
        base += sp->s_rx_buffer_align;
        size -= sp->s_rx_buffer_align;

        /* Bind the DMA memory to the DMA handle */
        err = ddi_dma_addr_bind_handle(srpp->srp_dma_handle, NULL,
            base, size, DDI_DMA_READ | DDI_DMA_STREAMING,
            DDI_DMA_DONTWAIT, NULL, &dmac, &ncookies);
        switch (err) {
        case DDI_DMA_MAPPED:
                break;

        case DDI_DMA_INUSE:
                srp->sr_kstat.srk_dma_bind_fail++;
                rc = EEXIST;
                goto fail3;

        case DDI_DMA_NORESOURCES:
                srp->sr_kstat.srk_dma_bind_nomem++;
                rc = ENOMEM;
                goto fail3;

        case DDI_DMA_NOMAPPING:
                srp->sr_kstat.srk_dma_bind_fail++;
                rc = ENOTSUP;
                goto fail3;

        case DDI_DMA_TOOBIG:
                srp->sr_kstat.srk_dma_bind_fail++;
                rc = EFBIG;
                goto fail3;

        default:
                srp->sr_kstat.srk_dma_bind_fail++;
                rc = EFAULT;
                goto fail3;
        }
        ASSERT3U(ncookies, ==, 1);

        srpp->srp_addr = dmac.dmac_laddress;

        srpp->srp_base = (unsigned char *)base;
        srpp->srp_mblksize = size;

        /*
         * Allocate a STREAMS block: We use size 1 so that the allocator will
         * use the first (and smallest) dblk cache.
         */
        freep = &(srpp->srp_free);
        freep->free_func = sfxge_rx_qpacket_free;
        freep->free_arg  = (caddr_t)srpp;

        if ((mp = desballoc(srpp->srp_base, size, BPRI_HI, freep)) == NULL) {
                srp->sr_kstat.srk_desballoc_fail++;
                rc = ENOMEM;
                goto fail4;
        }

        srpp->srp_mp = mp;
        srpp->srp_recycle = B_TRUE;

        if (sp->s_rx_pkt_mem_max) {
                int64_t delta = sfxge_rx_pkt_mem_approx(srpp);
                atomic_add_64(&sp->s_rx_pkt_mem_alloc, delta);
        }

        return (srpp);

fail4:
        DTRACE_PROBE(fail4);

        bzero(&(srpp->srp_free), sizeof (frtn_t));

        srpp->srp_mblksize = 0;
        srpp->srp_base = NULL;

        /* Unbind the DMA memory from the DMA handle */
        srpp->srp_addr = 0;
        (void) ddi_dma_unbind_handle(srpp->srp_dma_handle);

fail3:
        DTRACE_PROBE(fail3);

        /* Free the DMA memory */
        ddi_dma_mem_free(&(srpp->srp_acc_handle));
        srpp->srp_acc_handle = NULL;

fail2:
        DTRACE_PROBE(fail2);

        srpp->srp_putp = NULL;
        srpp->srp_srp = NULL;

        kmem_cache_free(sp->s_rpc, srpp);

fail1:
        DTRACE_PROBE1(fail1, int, rc);

        return (NULL);
}

#define SFXGE_REFILL_BATCH  64

/* Try to refill the RX descriptor ring from the associated free pkt pool */
static void
sfxge_rx_qrefill(sfxge_rxq_t *srp, unsigned int target)
{
        sfxge_t *sp = srp->sr_sp;
        sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];
        efsys_dma_addr_t addr[SFXGE_REFILL_BATCH];
        mblk_t *mp;
        int ntodo;
        unsigned int count;
        unsigned int batch;
        unsigned int rxfill;
        unsigned int mblksize;

        prefetch_read_many(sp->s_enp);
        prefetch_read_many(srp->sr_erp);

        ASSERT(mutex_owned(&(sep->se_lock)));

        if (srp->sr_state != SFXGE_RXQ_STARTED)
                return;

        rxfill = srp->sr_added - srp->sr_completed;
        ASSERT3U(rxfill, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
        ntodo = min(EFX_RXQ_LIMIT(sp->s_rxq_size) - rxfill, target);
        ASSERT3U(ntodo, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));

        if (ntodo == 0)
                goto out;

        (void) sfxge_rx_qfpp_swizzle(srp);

        mp = srfppp->srfpp_get;
        count = srfppp->srfpp_count;
        mblksize = sp->s_rx_buffer_size - sp->s_rx_buffer_align;

        batch = 0;
        while (ntodo-- > 0) {
                mblk_t *next;
                frtn_t *freep;
                sfxge_rx_packet_t *srpp;
                unsigned int id;

                if (mp == NULL)
                        break;

                next = mp->b_next;
                mp->b_next = NULL;

                if (next != NULL)
                        prefetch_read_many(next);

                freep = DB_FRTNP(mp);
                /*LINTED*/
                srpp = (sfxge_rx_packet_t *)(freep->free_arg);
                ASSERT3P(srpp->srp_mp, ==, mp);

                /* The MTU may have changed since the packet was allocated */
                if (MBLKSIZE(mp) != mblksize) {
                        srpp->srp_recycle = B_FALSE;

                        freeb(mp);

                        --count;
                        mp = next;
                        continue;
                }

                srpp->srp_off = 0;
                srpp->srp_thp = NULL;
                srpp->srp_iphp = NULL;
                srpp->srp_etherhp = NULL;
                srpp->srp_size = 0;
                srpp->srp_flags = EFX_DISCARD;

                id = (srp->sr_added + batch) & (sp->s_rxq_size - 1);
                ASSERT(srp->sr_srpp[id] == NULL);
                srp->sr_srpp[id] = srpp;

                addr[batch++] = srpp->srp_addr;
                if (batch == SFXGE_REFILL_BATCH) {
                        efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
                            srp->sr_completed, srp->sr_added);
                        srp->sr_added += batch;
                        batch = 0;
                }

                --count;
                mp = next;
        }

        srfppp->srfpp_get = mp;
        srfppp->srfpp_count = count;

        if (batch != 0) {
                efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
                    srp->sr_completed, srp->sr_added);
                srp->sr_added += batch;
        }

        efx_rx_qpush(srp->sr_erp, srp->sr_added, &srp->sr_pushed);

out:
        if (srfppp->srfpp_count < srfppp->srfpp_min)
                srfppp->srfpp_min = srfppp->srfpp_count;
}

/* Preallocate packets and put them in the free packet pool */
static void
sfxge_rx_qpreallocate(sfxge_rxq_t *srp, int nprealloc)
{
        sfxge_rx_fpp_t *srfppp = &((srp)->sr_fpp);
        srfppp->srfpp_lowat = nprealloc;
        while (nprealloc-- > 0) {
                sfxge_rx_packet_t *srpp;

                if ((srpp = sfxge_rx_qpacket_create(srp)) == NULL)
                        break;
                sfxge_rx_qfpp_put(srp, srpp);
        }
}

/* Try to refill the RX descriptor ring by allocating new packets */
static void
sfxge_rx_qfill(sfxge_rxq_t *srp, unsigned int target)
{
        sfxge_t *sp = srp->sr_sp;
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];
        unsigned int batch;
        unsigned int rxfill;
        unsigned int mblksize;
        int ntodo;
        efsys_dma_addr_t addr[SFXGE_REFILL_BATCH];
        mblk_t *mp = NULL;

        prefetch_read_many(sp->s_enp);
        prefetch_read_many(srp->sr_erp);

        ASSERT(mutex_owned(&(sep->se_lock)));

        if (srp->sr_state != SFXGE_RXQ_STARTED)
                return;

        rxfill = srp->sr_added - srp->sr_completed;
        ASSERT3U(rxfill, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
        ntodo = min(EFX_RXQ_LIMIT(sp->s_rxq_size) - rxfill, target);
        ASSERT3U(ntodo, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));

        if (ntodo == 0)
                return;

        mblksize = sp->s_rx_buffer_size - sp->s_rx_buffer_align;

        batch = 0;
        while (ntodo-- > 0) {
                sfxge_rx_packet_t *srpp;
                unsigned int id;

                if ((srpp = sfxge_rx_qpacket_create(srp)) == NULL)
                        break;

                mp = srpp->srp_mp;

                ASSERT3U(MBLKSIZE(mp), ==, mblksize);

                ASSERT3U(srpp->srp_off, ==, 0);
                ASSERT3P(srpp->srp_thp, ==, NULL);
                ASSERT3P(srpp->srp_iphp, ==, NULL);
                ASSERT3P(srpp->srp_etherhp, ==, NULL);
                ASSERT3U(srpp->srp_size, ==, 0);

                srpp->srp_flags = EFX_DISCARD;

                id = (srp->sr_added + batch) & (sp->s_rxq_size - 1);
                ASSERT(srp->sr_srpp[id] == NULL);
                srp->sr_srpp[id] = srpp;

                addr[batch++] = srpp->srp_addr;
                if (batch == SFXGE_REFILL_BATCH) {
                        efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
                            srp->sr_completed, srp->sr_added);
                        srp->sr_added += batch;
                        batch = 0;
                }
        }

        if (batch != 0) {
                efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
                    srp->sr_completed, srp->sr_added);
                srp->sr_added += batch;
        }

        efx_rx_qpush(srp->sr_erp, srp->sr_added, &srp->sr_pushed);
}

void
sfxge_rx_qfpp_trim(sfxge_rxq_t *srp)
{
        sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
        sfxge_t *sp = srp->sr_sp;
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];
        mblk_t *p;
        mblk_t **pp;
        int count;

        ASSERT(mutex_owned(&(sep->se_lock)));

        if (srp->sr_state != SFXGE_RXQ_STARTED)
                goto done;

        /* Make sure the queue is full */
        sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));

        /* The refill may have emptied the pool */
        if (srfppp->srfpp_min == 0)
                goto done;

        /* Don't trim below the pool's low water mark */
        if (srfppp->srfpp_count <= srfppp->srfpp_lowat)
                goto done;

        ASSERT(srfppp->srfpp_min <= srfppp->srfpp_count);

        /* Trim to the largest of srfppp->srfpp_min and srfpp->srfpp_lowat */
        if (srfppp->srfpp_lowat > srfppp->srfpp_min)
                count = srfppp->srfpp_count - srfppp->srfpp_lowat;
        else
                count = srfppp->srfpp_count - srfppp->srfpp_min;

        /* Walk the get list */
        pp = &(srfppp->srfpp_get);
        while (--count >= 0) {
                ASSERT(pp);
                p = *pp;
                ASSERT(p != NULL);

                pp = &(p->b_next);
        }
        ASSERT(pp);
        p = *pp;

        /* Truncate the get list */
        *pp = NULL;

        /* Free the remainder */
        while (p != NULL) {
                mblk_t *next;
                frtn_t *freep;
                sfxge_rx_packet_t *srpp;

                next = p->b_next;
                p->b_next = NULL;

                ASSERT3U(srfppp->srfpp_min, >, 0);
                srfppp->srfpp_min--;
                srfppp->srfpp_count--;

                freep = DB_FRTNP(p);
                /*LINTED*/
                srpp = (sfxge_rx_packet_t *)(freep->free_arg);
                ASSERT3P(srpp->srp_mp, ==, p);

                srpp->srp_recycle = B_FALSE;

                freeb(p);

                p = next;
        }

done:
        srfppp->srfpp_min = srfppp->srfpp_count;
}

static void
sfxge_rx_qpoll(void *arg)
{
        sfxge_rxq_t *srp = arg;
        sfxge_t *sp = srp->sr_sp;
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];
        uint16_t magic;

        /*
         * man timeout(9f) states that this code should adhere to the
         * same requirements as a softirq handler - DO NOT BLOCK
         */

        /*
         * Post an event to the event queue to cause the free packet pool to be
         * trimmed if it is oversize.
         */
        magic = SFXGE_MAGIC_RX_QFPP_TRIM | index;

#if defined(DEBUG)
        /* This is guaranteed due to the start/stop order of rx and ev */
        ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
        ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
#else
        /*
         * Bug22691 WORKAROUND:
         * This handler has been observed in the field to be invoked for a
         * queue in the INITIALIZED state, which should never happen.
         * Until the mechanism for this is properly understood, add defensive
         * checks.
         */
        if ((sep->se_state != SFXGE_EVQ_STARTED) ||
            (srp->sr_state != SFXGE_RXQ_STARTED) ||
            (!sep->se_eep)) {
                dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
                    "RXQ[%d] bad state in sfxge_rx_qpoll %d %d %p",
                    index, sep->se_state, srp->sr_state, sep->se_eep);
                return;
        }
#endif
        efx_ev_qpost(sep->se_eep, magic);

        srp->sr_tid = timeout(sfxge_rx_qpoll, srp,
            drv_usectohz(sp->s_rxq_poll_usec));
}

static void
sfxge_rx_qpoll_start(sfxge_rxq_t *srp)
{
        sfxge_t *sp = srp->sr_sp;
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];

        ASSERT(mutex_owned(&(sep->se_lock)));
        ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);

        /* Schedule a poll */
        ASSERT3P(srp->sr_tid, ==, 0);
        srp->sr_tid = timeout(sfxge_rx_qpoll, srp, 0);
}

static void
sfxge_rx_qpoll_stop(sfxge_rxq_t *srp)
{
        sfxge_t *sp = srp->sr_sp;
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];
        timeout_id_t tid;

        ASSERT(mutex_owned(&(sep->se_lock)));
        ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);

        /*
         * Cancel the qpoll timer. Care is needed as this function
         * can race with sfxge_rx_qpoll() for timeout id updates.
         *
         * Do not hold locks used by any timeout(9f) handlers across
         * calls to untimeout(9f) as this will deadlock.
         */
        tid = 0;
        while ((srp->sr_tid != 0) && (srp->sr_tid != tid)) {
                tid = srp->sr_tid;
                (void) untimeout(tid);
        }
        srp->sr_tid = 0;
}

static int
sfxge_rx_kstat_update(kstat_t *ksp, int rw)
{
        sfxge_rxq_t *srp = ksp->ks_private;
        sfxge_t *sp = srp->sr_sp;
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];
        kstat_named_t *knp;
        int rc;

        if (rw != KSTAT_READ) {
                rc = EACCES;
                goto fail1;
        }

        ASSERT(mutex_owned(&(sep->se_lock)));
        if (srp->sr_state != SFXGE_RXQ_STARTED)
                goto done;

        knp = ksp->ks_data;
        /* NB pointer post-increment below */
        knp++->value.ui32 = srp->sr_kstat.srk_rx_pkt_mem_limit;
        knp++->value.ui32 = srp->sr_kstat.srk_kcache_alloc_nomem;
        knp++->value.ui32 = srp->sr_kstat.srk_dma_alloc_nomem;
        knp++->value.ui32 = srp->sr_kstat.srk_dma_alloc_fail;
        knp++->value.ui32 = srp->sr_kstat.srk_dma_bind_nomem;
        knp++->value.ui32 = srp->sr_kstat.srk_dma_bind_fail;
        knp++->value.ui32 = srp->sr_kstat.srk_desballoc_fail;
        knp++->value.ui32 = srp->sr_kstat.srk_rxq_empty_discard;

done:
        return (0);

fail1:
        DTRACE_PROBE1(fail1, int, rc);

        return (rc);
}

static int
sfxge_rx_kstat_init(sfxge_rxq_t *srp)
{
        sfxge_t *sp = srp->sr_sp;
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];
        dev_info_t *dip = sp->s_dip;
        char name[MAXNAMELEN];
        kstat_t *ksp;
        kstat_named_t *knp;
        int rc;

        /* Create the set */
        (void) snprintf(name, MAXNAMELEN - 1, "%s_rxq%04d",
            ddi_driver_name(dip), index);

        if ((ksp = kstat_create((char *)ddi_driver_name(dip),
            ddi_get_instance(dip), name, "rxq", KSTAT_TYPE_NAMED,
            SFXGE_RX_NSTATS, 0)) == NULL) {
                rc = ENOMEM;
                goto fail1;
        }

        srp->sr_ksp = ksp;

        ksp->ks_update = sfxge_rx_kstat_update;
        ksp->ks_private = srp;
        ksp->ks_lock = &(sep->se_lock);

        /* Initialise the named stats */
        knp = ksp->ks_data;
        kstat_named_init(knp, "rx_pkt_mem_limit", KSTAT_DATA_UINT32);
        knp++;
        kstat_named_init(knp, "kcache_alloc_nomem", KSTAT_DATA_UINT32);
        knp++;
        kstat_named_init(knp, "dma_alloc_nomem", KSTAT_DATA_UINT32);
        knp++;
        kstat_named_init(knp, "dma_alloc_fail", KSTAT_DATA_UINT32);
        knp++;
        kstat_named_init(knp, "dma_bind_nomem", KSTAT_DATA_UINT32);
        knp++;
        kstat_named_init(knp, "dma_bind_fail", KSTAT_DATA_UINT32);
        knp++;
        kstat_named_init(knp, "desballoc_fail", KSTAT_DATA_UINT32);
        knp++;
        kstat_named_init(knp, "rxq_empty_discard", KSTAT_DATA_UINT32);

        kstat_install(ksp);
        return (0);

fail1:
        DTRACE_PROBE1(fail1, int, rc);

        return (rc);
}

static int
sfxge_rx_qinit(sfxge_t *sp, unsigned int index)
{
        sfxge_rxq_t *srp;
        int rc;

        ASSERT3U(index, <, SFXGE_RX_SCALE_MAX);

        if ((srp = kmem_cache_alloc(sp->s_rqc, KM_SLEEP)) == NULL) {
                rc = ENOMEM;
                goto fail1;
        }
        ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_UNINITIALIZED);

        srp->sr_index = index;
        sp->s_srp[index] = srp;

        if ((rc = sfxge_rx_kstat_init(srp)) != 0)
                goto fail2;

        srp->sr_state = SFXGE_RXQ_INITIALIZED;

        return (0);

fail2:
        DTRACE_PROBE(fail2);
        kmem_cache_free(sp->s_rqc, srp);

fail1:
        DTRACE_PROBE1(fail1, int, rc);

        return (rc);
}

static int
sfxge_rx_qstart(sfxge_t *sp, unsigned int index)
{
        sfxge_evq_t *sep = sp->s_sep[index];
        sfxge_rxq_t *srp;
        efsys_mem_t *esmp;
        efx_nic_t *enp;
        unsigned int level;
        int rc;

        mutex_enter(&(sep->se_lock));
        srp = sp->s_srp[index];
        enp = sp->s_enp;
        esmp = &(srp->sr_mem);

        ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_INITIALIZED);
        ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);

        /* Zero the memory */
        bzero(esmp->esm_base, EFX_RXQ_SIZE(sp->s_rxq_size));

        /* Program the buffer table */
        if ((rc = sfxge_sram_buf_tbl_set(sp, srp->sr_id, esmp,
            EFX_RXQ_NBUFS(sp->s_rxq_size))) != 0)
                goto fail1;

        /* Create the receive queue */
        if ((rc = efx_rx_qcreate(enp, index, index, EFX_RXQ_TYPE_DEFAULT,
            esmp, sp->s_rxq_size, srp->sr_id, sep->se_eep, &(srp->sr_erp)))
            != 0)
                goto fail2;

        /* Enable the receive queue */
        efx_rx_qenable(srp->sr_erp);

        /* Set the water marks */
        srp->sr_hiwat = EFX_RXQ_LIMIT(sp->s_rxq_size) * 9 / 10;
        srp->sr_lowat = srp->sr_hiwat / 2;

        srp->sr_state = SFXGE_RXQ_STARTED;
        srp->sr_flush = SFXGE_FLUSH_INACTIVE;

        sfxge_rx_qpoll_start(srp);

        /* Try to fill the queue from the pool */
        sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));

        /*
         * If there were insufficient buffers in the pool to reach the at
         * least a batch then allocate some.
         */
        level = srp->sr_added - srp->sr_completed;
        if (level < SFXGE_RX_BATCH)
                sfxge_rx_qfill(srp, SFXGE_RX_BATCH);

        mutex_exit(&(sep->se_lock));

        return (0);

fail2:
        DTRACE_PROBE(fail2);

        /* Clear entries from the buffer table */
        sfxge_sram_buf_tbl_clear(sp, srp->sr_id,
            EFX_RXQ_NBUFS(sp->s_rxq_size));

fail1:
        DTRACE_PROBE1(fail1, int, rc);

        mutex_exit(&(sep->se_lock));

        return (rc);
}

static void
sfxge_rx_qflow_complete(sfxge_rxq_t *srp, sfxge_rx_flow_t *srfp)
{
        mblk_t *mp;
        struct ether_header *etherhp;
        struct ip *iphp;
        struct tcphdr *thp;

        if (srfp->srf_mp == NULL)
                return;

        mp = srfp->srf_mp;
        etherhp = srfp->srf_etherhp;
        iphp = srfp->srf_iphp;
        thp = srfp->srf_last_thp;

        ASSERT3U(((etherhp->ether_type == htons(ETHERTYPE_VLAN)) ?
            sizeof (struct ether_vlan_header) :
            sizeof (struct ether_header)) +
            srfp->srf_len, ==, msgdsize(mp));

        ASSERT3U(srfp->srf_len & 0xffff, ==, srfp->srf_len);
        iphp->ip_len = htons(srfp->srf_len);

        srfp->srf_first_thp->th_ack = thp->th_ack;
        srfp->srf_first_thp->th_win = thp->th_win;
        srfp->srf_first_thp->th_flags = thp->th_flags;

        DTRACE_PROBE2(flow_complete, uint32_t, srfp->srf_tag,
            size_t, srfp->srf_len);

        srfp->srf_mp = NULL;
        srfp->srf_len = 0;

        ASSERT(mp->b_next == NULL);
        *(srp->sr_mpp) = mp;
        srp->sr_mpp = &(mp->b_next);
}

static boolean_t
sfxge_rx_qflow_add(sfxge_rxq_t *srp, sfxge_rx_flow_t *srfp,
    sfxge_rx_packet_t *srpp, clock_t now)
{
        sfxge_t *sp = srp->sr_sp;
        struct ether_header *etherhp = srpp->srp_etherhp;
        struct ip *iphp = srpp->srp_iphp;
        struct tcphdr *thp = srpp->srp_thp;
        size_t off = srpp->srp_off;
        size_t size = (size_t)(srpp->srp_size);
        mblk_t *mp = srpp->srp_mp;
        uint32_t seq;
        unsigned int shift;

        ASSERT3U(MBLKL(mp), ==, off + size);
        ASSERT3U(DB_CKSUMFLAGS(mp), ==,
            HCK_FULLCKSUM | HCK_FULLCKSUM_OK | HCK_IPV4_HDRCKSUM);

        seq = htonl(thp->th_seq);

        /*
         * If the time between this segment and the last is greater than RTO
         * then consider this a new flow.
         */
        if (now - srfp->srf_lbolt > srp->sr_rto) {
                srfp->srf_count = 1;
                srfp->srf_seq = seq + size;

                goto fail1;
        }

        if (seq != srfp->srf_seq) {
                if (srfp->srf_count > SFXGE_SLOW_START)
                        srfp->srf_count = SFXGE_SLOW_START;

                srfp->srf_count >>= 1;

                srfp->srf_count++;
                srfp->srf_seq = seq + size;

                goto fail2;
        }

        /* Update the in-order segment count and sequence number */
        srfp->srf_count++;
        srfp->srf_seq = seq + size;

        /* Don't merge across pure ACK, URG, SYN or RST segments */
        if (size == 0 || thp->th_flags & (TH_URG | TH_SYN | TH_RST) ||
            thp->th_urp != 0)
                goto fail3;

        /*
         * If the in-order segment count has not yet reached the slow-start
         * threshold then we cannot coalesce.
         */
        if (srfp->srf_count < SFXGE_SLOW_START)
                goto fail4;

        /* Scale up the packet size from 4k (the maximum being 64k) */
        ASSERT3U(srfp->srf_count, >=, SFXGE_SLOW_START);
        shift = MIN(srfp->srf_count - SFXGE_SLOW_START + 12, 16);
        if (srfp->srf_len + size >= (1 << shift))
                sfxge_rx_qflow_complete(srp, srfp);

        ASSERT(mp->b_cont == NULL);

        if (srfp->srf_mp == NULL) {
                /* First packet in this flow */
                srfp->srf_etherhp = etherhp;
                srfp->srf_iphp = iphp;
                srfp->srf_first_thp = srfp->srf_last_thp = thp;

                ASSERT3P(mp->b_cont, ==, NULL);
                srfp->srf_mp = mp;
                srfp->srf_mpp = &(mp->b_cont);

                srfp->srf_len = ntohs(iphp->ip_len);

                /*
                 * If the flow is not already in the list of occupied flows then
                 * add it.
                 */
                if (srfp->srf_next == NULL &&
                    srp->sr_srfpp != &(srfp->srf_next)) {
                        *(srp->sr_srfpp) = srfp;
                        srp->sr_srfpp = &(srfp->srf_next);
                }
        } else {
                /* Later packet in this flow - skip TCP header */
                srfp->srf_last_thp = thp;

                mp->b_rptr += off;
                ASSERT3U(MBLKL(mp), ==, size);

                ASSERT3P(mp->b_cont, ==, NULL);
                *(srfp->srf_mpp) = mp;
                srfp->srf_mpp = &(mp->b_cont);

                srfp->srf_len += size;

                ASSERT(srfp->srf_next != NULL ||
                    srp->sr_srfpp == &(srfp->srf_next));
        }

        DTRACE_PROBE2(flow_add, uint32_t, srfp->srf_tag, size_t, size);

        /*
         * Try to align coalesced segments on push boundaries, unless they
         * are too frequent.
         */
        if (sp->s_rx_coalesce_mode == SFXGE_RX_COALESCE_ALLOW_PUSH &&
            thp->th_flags & TH_PUSH)
                sfxge_rx_qflow_complete(srp, srfp);

        srfp->srf_lbolt = now;
        return (B_TRUE);

fail4:
fail3:
fail2:
fail1:
        sfxge_rx_qflow_complete(srp, srfp);

        srfp->srf_lbolt = now;
        return (B_FALSE);
}

void
sfxge_rx_qpacket_coalesce(sfxge_rxq_t *srp)
{
        sfxge_t *sp = srp->sr_sp;
        clock_t now;
        mblk_t *mp;
        sfxge_rx_flow_t *srfp;

        ASSERT(sp->s_rx_coalesce_mode != SFXGE_RX_COALESCE_OFF);

        now = ddi_get_lbolt();

        mp = srp->sr_mp;

        srp->sr_mp = NULL;
        srp->sr_mpp = &(srp->sr_mp);

        /* Start with the last flow to be appended to */
        srfp = *(srp->sr_srfpp);

        while (mp != NULL) {
                frtn_t *freep;
                sfxge_rx_packet_t *srpp;
                struct ether_header *etherhp;
                struct ip *iphp;
                struct tcphdr *thp;
                size_t off;
                size_t size;
                uint16_t ether_tci;
                uint32_t hash;
                uint32_t tag;
                mblk_t *next;
                sfxge_packet_type_t pkt_type;
                uint16_t sport, dport;

                next = mp->b_next;
                mp->b_next = NULL;

                if (next != NULL)
                        prefetch_read_many(next);

                freep = DB_FRTNP(mp);
                /*LINTED*/
                srpp = (sfxge_rx_packet_t *)(freep->free_arg);
                ASSERT3P(srpp->srp_mp, ==, mp);

                /* If the packet is not TCP then we cannot coalesce it */
                if (~(srpp->srp_flags) & EFX_PKT_TCP)
                        goto reject;

                /*
                 * If the packet is not fully checksummed then we cannot
                 * coalesce it.
                 */
                if (~(srpp->srp_flags) & (EFX_CKSUM_TCPUDP | EFX_CKSUM_IPV4))
                        goto reject;

                /* Parse the TCP header */
                pkt_type = sfxge_pkthdr_parse(mp, &etherhp, &iphp, &thp, &off,
                    &size, &sport, &dport);
                ASSERT(pkt_type == SFXGE_PACKET_TYPE_IPV4_TCP);
                ASSERT(etherhp != NULL);
                ASSERT(iphp != NULL);
                ASSERT(thp != NULL);
                ASSERT(off != 0);

                if ((iphp->ip_off & ~htons(IP_DF)) != 0)
                        goto reject;

                if (etherhp->ether_type == htons(ETHERTYPE_VLAN)) {
                        struct ether_vlan_header *ethervhp;

                        ethervhp = (struct ether_vlan_header *)etherhp;
                        ether_tci = ethervhp->ether_tci;
                } else {
                        ether_tci = 0;
                }

                /*
                 * Make sure any minimum length padding is stripped
                 * before we try to add the packet to a flow.
                 */
                ASSERT3U(sp->s_rx_prefix_size + MBLKL(mp), ==,
                    (size_t)(srpp->srp_size));
                ASSERT3U(sp->s_rx_prefix_size + off + size, <=,
                    (size_t)(srpp->srp_size));

                if (sp->s_rx_prefix_size + off + size <
                    (size_t)(srpp->srp_size))
                        mp->b_wptr = mp->b_rptr + off + size;

                /*
                 * If there is no current flow, or the segment does not match
                 * the current flow then we must attempt to look up the
                 * correct flow in the table.
                 */
                if (srfp == NULL)
                        goto lookup;

                if (srfp->srf_saddr != iphp->ip_src.s_addr ||
                    srfp->srf_daddr != iphp->ip_dst.s_addr)
                        goto lookup;

                if (srfp->srf_sport != thp->th_sport ||
                    srfp->srf_dport != thp->th_dport)
                        goto lookup;

                if (srfp->srf_tci != ether_tci)
                        goto lookup;

add:
                ASSERT(srfp != NULL);

                srpp->srp_etherhp = etherhp;
                srpp->srp_iphp = iphp;
                srpp->srp_thp = thp;
                srpp->srp_off = off;

                ASSERT3U(size, <, (1 << 16));
                srpp->srp_size = (uint16_t)size;

                /* Try to append the packet to the flow */
                if (!sfxge_rx_qflow_add(srp, srfp, srpp, now))
                        goto reject;

                mp = next;
                continue;

lookup:
                /*
                 * If there is a prefix area then read the hash from that,
                 * otherwise calculate it.
                 */
                if (sp->s_rx_prefix_size != 0) {
                        hash = efx_psuedo_hdr_hash_get(sp->s_enp,
                            EFX_RX_HASHALG_TOEPLITZ,
                            DB_BASE(mp));
                } else {
                        SFXGE_TCP_HASH(sp,
                            &iphp->ip_src.s_addr,
                            thp->th_sport,
                            &iphp->ip_dst.s_addr,
                            thp->th_dport,
                            hash);
                }

                srfp = &(srp->sr_flow[(hash >> 6) % SFXGE_MAX_FLOW]);
                tag = hash + 1; /* Make sure it's not zero */

                /*
                 * If the flow we have found does not match the hash then
                 * it may be an unused flow, or it may be stale.
                 */
                if (tag != srfp->srf_tag) {
                        if (srfp->srf_count != 0) {
                                if (now - srfp->srf_lbolt <= srp->sr_rto)
                                        goto reject;
                        }

                        if (srfp->srf_mp != NULL)
                                goto reject;

                        /* Start a new flow */
                        ASSERT(srfp->srf_next == NULL);

                        srfp->srf_tag = tag;

                        srfp->srf_saddr = iphp->ip_src.s_addr;
                        srfp->srf_daddr = iphp->ip_dst.s_addr;
                        srfp->srf_sport = thp->th_sport;
                        srfp->srf_dport = thp->th_dport;
                        srfp->srf_tci = ether_tci;

                        srfp->srf_count = 0;
                        srfp->srf_seq = ntohl(thp->th_seq);

                        srfp->srf_lbolt = now;
                        goto add;
                }

                /*
                 * If the flow we have found does match the hash then it could
                 * still be an alias.
                 */
                if (srfp->srf_saddr != iphp->ip_src.s_addr ||
                    srfp->srf_daddr != iphp->ip_dst.s_addr)
                        goto reject;

                if (srfp->srf_sport != thp->th_sport ||
                    srfp->srf_dport != thp->th_dport)
                        goto reject;

                if (srfp->srf_tci != ether_tci)
                        goto reject;

                goto add;

reject:
                *(srp->sr_mpp) = mp;
                srp->sr_mpp = &(mp->b_next);

                mp = next;
        }
}

void
sfxge_rx_qcomplete(sfxge_rxq_t *srp, boolean_t eop)
{
        sfxge_t *sp = srp->sr_sp;
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];
        unsigned int completed;
        sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
        unsigned int level;

        ASSERT(mutex_owned(&(sep->se_lock)));

        ASSERT(srp->sr_mp == NULL);
        ASSERT(srp->sr_mpp == &(srp->sr_mp));

        completed = srp->sr_completed;
        while (completed != srp->sr_pending) {
                unsigned int id;
                sfxge_rx_packet_t *srpp;
                mblk_t *mp;
                size_t size;
                uint16_t flags;
                int rc;

                id = completed++ & (sp->s_rxq_size - 1);

                if (srp->sr_pending - completed >= 4) {
                        unsigned int prefetch;

                        prefetch = (id + 4) & (sp->s_rxq_size - 1);

                        srpp = srp->sr_srpp[prefetch];
                        ASSERT(srpp != NULL);

                        mp = srpp->srp_mp;
                        prefetch_read_many(mp->b_datap);
                } else if (completed == srp->sr_pending) {
                        prefetch_read_many(srp->sr_mp);
                }

                srpp = srp->sr_srpp[id];
                ASSERT(srpp != NULL);

                srp->sr_srpp[id] = NULL;

                mp = srpp->srp_mp;
                ASSERT(mp->b_cont == NULL);

                /* when called from sfxge_rx_qstop() */
                if (srp->sr_state != SFXGE_RXQ_STARTED)
                        goto discard;

                if (srpp->srp_flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
                        goto discard;

                /* Make the data visible to the kernel */
                rc = ddi_dma_sync(srpp->srp_dma_handle, 0,
                    sp->s_rx_buffer_size, DDI_DMA_SYNC_FORKERNEL);
                ASSERT3P(rc, ==, DDI_SUCCESS);

                /* Read the length from the psuedo header if required */
                if (srpp->srp_flags & EFX_PKT_PREFIX_LEN) {
                        rc = efx_psuedo_hdr_pkt_length_get(sp->s_enp,
                            mp->b_rptr,
                            &srpp->srp_size);
                        ASSERT3P(rc, ==, 0);
                        srpp->srp_size += sp->s_rx_prefix_size;
                }

                /* Set up the packet length */
                ASSERT3P(mp->b_rptr, ==, DB_BASE(mp));
                mp->b_rptr += sp->s_rx_prefix_size;

                prefetch_read_many(mp->b_rptr);

                ASSERT3P(mp->b_wptr, ==, DB_BASE(mp));
                mp->b_wptr += (size_t)(srpp->srp_size);
                ASSERT3P(mp->b_wptr, <=, DB_LIM(mp));

                /* Calculate the maximum packet size */
                size = sp->s_mtu;
                size += (srpp->srp_flags & EFX_PKT_VLAN_TAGGED) ?
                    sizeof (struct ether_vlan_header) :
                    sizeof (struct ether_header);

                if (MBLKL(mp) > size)
                        goto discard;

                /* Check for loopback packets */
                if (!(srpp->srp_flags & EFX_PKT_IPV4) &&
                    !(srpp->srp_flags & EFX_PKT_IPV6)) {
                        struct ether_header *etherhp;

                        /*LINTED*/
                        etherhp = (struct ether_header *)(mp->b_rptr);

                        if (etherhp->ether_type ==
                            htons(SFXGE_ETHERTYPE_LOOPBACK)) {
                                DTRACE_PROBE(loopback);

                                srp->sr_loopback++;
                                goto discard;
                        }
                }

                /* Set up the checksum information */
                flags = 0;

                if (srpp->srp_flags & EFX_CKSUM_IPV4) {
                        ASSERT(srpp->srp_flags & EFX_PKT_IPV4);
                        flags |= HCK_IPV4_HDRCKSUM;
                }

                if (srpp->srp_flags & EFX_CKSUM_TCPUDP) {
                        ASSERT(srpp->srp_flags & EFX_PKT_TCP ||
                            srpp->srp_flags & EFX_PKT_UDP);
                        flags |= HCK_FULLCKSUM | HCK_FULLCKSUM_OK;
                }

                DB_CKSUMSTART(mp) = 0;
                DB_CKSUMSTUFF(mp) = 0;
                DB_CKSUMEND(mp) = 0;
                DB_CKSUMFLAGS(mp) = flags;
                DB_CKSUM16(mp) = 0;

                /* Add the packet to the tail of the chain */
                srfppp->srfpp_loaned++;

                ASSERT(mp->b_next == NULL);
                *(srp->sr_mpp) = mp;
                srp->sr_mpp = &(mp->b_next);

                continue;

discard:
                /* Return the packet to the pool */
                srfppp->srfpp_loaned++;
                freeb(mp); /* Equivalent to freemsg() as b_cont==0 */
        }
        srp->sr_completed = completed;

        /* Attempt to coalesce any TCP packets */
        if (sp->s_rx_coalesce_mode != SFXGE_RX_COALESCE_OFF)
                sfxge_rx_qpacket_coalesce(srp);

        /*
         * If there are any pending flows and this is the end of the
         * poll then they must be completed.
         */
        if (srp->sr_srfp != NULL && eop) {
                sfxge_rx_flow_t *srfp;

                srfp = srp->sr_srfp;

                srp->sr_srfp = NULL;
                srp->sr_srfpp = &(srp->sr_srfp);

                do {
                        sfxge_rx_flow_t *next;

                        next = srfp->srf_next;
                        srfp->srf_next = NULL;

                        sfxge_rx_qflow_complete(srp, srfp);

                        srfp = next;
                } while (srfp != NULL);
        }

        level = srp->sr_pushed - srp->sr_completed;

        /* If there are any packets then pass them up the stack */
        if (srp->sr_mp != NULL) {
                mblk_t *mp;

                mp = srp->sr_mp;

                srp->sr_mp = NULL;
                srp->sr_mpp = &(srp->sr_mp);

                if (level == 0) {
                        /* Try to refill ASAP */
                        sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
                        level = srp->sr_pushed - srp->sr_completed;
                }

                /*
                 * If the RXQ is still empty, discard and recycle the
                 * current entry to ensure that the ring always
                 * contains at least one descriptor. This ensures that
                 * the next hardware RX will trigger an event
                 * (possibly delayed by interrupt moderation) and
                 * trigger another refill/fill attempt.
                 *
                 * Note this drops a complete LRO fragment from the
                 * start of the batch.
                 *
                 * Note also that copymsgchain() does not help with
                 * resource starvation here, unless we are short of DMA
                 * mappings.
                 */
                if (level == 0) {
                        mblk_t *nmp;

                        srp->sr_kstat.srk_rxq_empty_discard++;
                        DTRACE_PROBE1(rxq_empty_discard, int, index);
                        nmp = mp->b_next;
                        if (nmp)
                                sfxge_gld_rx_post(sp, index, nmp);
                        /* as level==0 will swizzle,rxpost below */
                        freemsg(mp);
                } else {
                        sfxge_gld_rx_post(sp, index, mp);
                }
        }

        /* Top up the queue if necessary */
        if (level < srp->sr_hiwat) {
                sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));

                level = srp->sr_added - srp->sr_completed;
                if (level < srp->sr_lowat)
                        sfxge_rx_qfill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
        }
}

void
sfxge_rx_qflush_done(sfxge_rxq_t *srp)
{
        sfxge_t *sp = srp->sr_sp;
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];
        boolean_t flush_pending;

        ASSERT(mutex_owned(&(sep->se_lock)));

        /*
         * Flush successful: wakeup sfxge_rx_qstop() if flush is pending.
         *
         * A delayed flush event received after RxQ stop has timed out
         * will be ignored, as then the flush state will not be PENDING
         * (see SFCbug22989).
         */
        flush_pending = (srp->sr_flush == SFXGE_FLUSH_PENDING);
        srp->sr_flush = SFXGE_FLUSH_DONE;
        if (flush_pending)
                cv_broadcast(&(srp->sr_flush_kv));
}

void
sfxge_rx_qflush_failed(sfxge_rxq_t *srp)
{
        sfxge_t *sp = srp->sr_sp;
        unsigned int index = srp->sr_index;
        sfxge_evq_t *sep = sp->s_sep[index];
        boolean_t flush_pending;

        ASSERT(mutex_owned(&(sep->se_lock)));

        /*
         * Flush failed: wakeup sfxge_rx_qstop() if flush is pending.
         *
         * A delayed flush event received after RxQ stop has timed out
         * will be ignored, as then the flush state will not be PENDING
         * (see SFCbug22989).
         */
        flush_pending = (srp->sr_flush == SFXGE_FLUSH_PENDING);
        srp->sr_flush = SFXGE_FLUSH_FAILED;
        if (flush_pending)
                cv_broadcast(&(srp->sr_flush_kv));
}

static void
sfxge_rx_qstop(sfxge_t *sp, unsigned int index)
{
        dev_info_t *dip = sp->s_dip;
        sfxge_evq_t *sep = sp->s_sep[index];
        sfxge_rxq_t *srp;
        clock_t timeout;
        unsigned int flush_tries = SFXGE_RX_QFLUSH_TRIES;
        int rc;

        ASSERT(mutex_owned(&(sp->s_state_lock)));

        mutex_enter(&(sep->se_lock));

        srp = sp->s_srp[index];
        ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);

        sfxge_rx_qpoll_stop(srp);

        /* Further packets are discarded by sfxge_rx_qcomplete() */
        srp->sr_state = SFXGE_RXQ_INITIALIZED;

        if (sp->s_hw_err != SFXGE_HW_OK) {
                /*
                 * Flag indicates possible hardware failure.
                 * Attempt flush but do not wait for it to complete.
                 */
                srp->sr_flush = SFXGE_FLUSH_DONE;
                (void) efx_rx_qflush(srp->sr_erp);
        }

        /* Wait upto 2sec for queue flushing to complete */
        timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_RX_QFLUSH_USEC);

        while (srp->sr_flush != SFXGE_FLUSH_DONE && flush_tries-- > 0) {
                if ((rc = efx_rx_qflush(srp->sr_erp)) != 0) {
                        if (rc == EALREADY)
                                srp->sr_flush = SFXGE_FLUSH_DONE;
                        else
                                srp->sr_flush = SFXGE_FLUSH_FAILED;
                        break;
                }
                srp->sr_flush = SFXGE_FLUSH_PENDING;
                if (cv_timedwait(&(srp->sr_flush_kv), &(sep->se_lock),
                    timeout) < 0) {
                        /* Timeout waiting for successful or failed flush */
                        dev_err(dip, CE_NOTE,
                            SFXGE_CMN_ERR "rxq[%d] flush timeout", index);
                        break;
                }
        }

        if (srp->sr_flush == SFXGE_FLUSH_FAILED)
                dev_err(dip, CE_NOTE,
                    SFXGE_CMN_ERR "rxq[%d] flush failed", index);

        DTRACE_PROBE1(flush, sfxge_flush_state_t, srp->sr_flush);
        srp->sr_flush = SFXGE_FLUSH_DONE;

        /* Destroy the receive queue */
        efx_rx_qdestroy(srp->sr_erp);
        srp->sr_erp = NULL;

        /* Clear entries from the buffer table */
        sfxge_sram_buf_tbl_clear(sp, srp->sr_id,
            EFX_RXQ_NBUFS(sp->s_rxq_size));

        /*
         * Free any unused RX packets which had descriptors on the RXQ
         * Packets will be discard as state != STARTED
         */
        srp->sr_pending = srp->sr_added;
        sfxge_rx_qcomplete(srp, B_TRUE);

        ASSERT3U(srp->sr_completed, ==, srp->sr_pending);

        srp->sr_added = 0;
        srp->sr_pushed = 0;
        srp->sr_pending = 0;
        srp->sr_completed = 0;
        srp->sr_loopback = 0;

        srp->sr_lowat = 0;
        srp->sr_hiwat = 0;

        mutex_exit(&(sep->se_lock));
}

static void
sfxge_rx_kstat_fini(sfxge_rxq_t *srp)
{
        kstat_delete(srp->sr_ksp);
        srp->sr_ksp = NULL;
}

static void
sfxge_rx_qfini(sfxge_t *sp, unsigned int index)
{
        sfxge_rxq_t *srp = sp->s_srp[index];

        ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_INITIALIZED);

        sp->s_srp[index] = NULL;
        srp->sr_state = SFXGE_RXQ_UNINITIALIZED;

        sfxge_rx_kstat_fini(srp);

        /* Empty the pool */
        sfxge_rx_qfpp_empty(srp);

        srp->sr_index = 0;

        kmem_cache_free(sp->s_rqc, srp);
}

static int
sfxge_rx_scale_kstat_update(kstat_t *ksp, int rw)
{
        sfxge_t *sp = ksp->ks_private;
        sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
        sfxge_intr_t *sip = &(sp->s_intr);
        kstat_named_t *knp;
        unsigned int index;
        unsigned int entry;
        unsigned int *freq;
        int rc;

        ASSERT(mutex_owned(&(srsp->srs_lock)));

        if (rw != KSTAT_READ) {
                rc = EACCES;
                goto fail1;
        }

        if ((freq = kmem_zalloc(sizeof (unsigned int) * sip->si_nalloc,
            KM_NOSLEEP)) == NULL) {
                rc = ENOMEM;
                goto fail2;
        }

        for (entry = 0; entry < SFXGE_RX_SCALE_MAX; entry++) {
                index = srsp->srs_tbl[entry];

                freq[index]++;
        }

        knp = ksp->ks_data;
        for (index = 0; index < sip->si_nalloc; index++) {
                knp->value.ui64 = freq[index];
                knp++;
        }

        knp->value.ui64 = srsp->srs_count;

        kmem_free(freq, sizeof (unsigned int) * sip->si_nalloc);

        return (0);

fail2:
        DTRACE_PROBE(fail2);
fail1:
        DTRACE_PROBE1(fail1, int, rc);
        return (rc);
}

static int
sfxge_rx_scale_kstat_init(sfxge_t *sp)
{
        dev_info_t *dip = sp->s_dip;
        sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
        sfxge_intr_t *sip = &(sp->s_intr);
        char name[MAXNAMELEN];
        kstat_t *ksp;
        kstat_named_t *knp;
        unsigned int index;
        int rc;

        /* Create the set */
        (void) snprintf(name, MAXNAMELEN - 1, "%s_rss", ddi_driver_name(dip));

        if ((ksp = kstat_create((char *)ddi_driver_name(dip),
            ddi_get_instance(dip), name, "rss", KSTAT_TYPE_NAMED,
            sip->si_nalloc + 1, 0)) == NULL) {
                rc = ENOMEM;
                goto fail1;
        }

        srsp->srs_ksp = ksp;

        ksp->ks_update = sfxge_rx_scale_kstat_update;
        ksp->ks_private = sp;
        ksp->ks_lock = &(srsp->srs_lock);

        /* Initialise the named stats */
        knp = ksp->ks_data;
        for (index = 0; index < sip->si_nalloc; index++) {
                char name[MAXNAMELEN];

                (void) snprintf(name, MAXNAMELEN - 1, "evq%04d_count", index);
                kstat_named_init(knp, name, KSTAT_DATA_UINT64);
                knp++;
        }

        kstat_named_init(knp, "scale", KSTAT_DATA_UINT64);

        kstat_install(ksp);
        return (0);

fail1:
        DTRACE_PROBE1(fail1, int, rc);

        return (rc);
}

static void
sfxge_rx_scale_kstat_fini(sfxge_t *sp)
{
        sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);

        /* Destroy the set */
        kstat_delete(srsp->srs_ksp);
        srsp->srs_ksp = NULL;
}


unsigned int
sfxge_rx_scale_prop_get(sfxge_t *sp)
{
        int rx_scale;

        rx_scale = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
            DDI_PROP_DONTPASS, "rx_scale_count", SFXGE_RX_SCALE_MAX);
        /* 0 and all -ve numbers sets to number of logical CPUs */
        if (rx_scale <= 0)
                rx_scale = ncpus;

        return (rx_scale);
}


static int
sfxge_rx_scale_init(sfxge_t *sp)
{
        sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
        sfxge_intr_t *sip = &(sp->s_intr);
        int rc;

        ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_UNINITIALIZED);

        /* Create tables for CPU, core, cache and chip counts */
        srsp->srs_cpu = kmem_zalloc(sizeof (unsigned int) * NCPU, KM_SLEEP);

        mutex_init(&(srsp->srs_lock), NULL, MUTEX_DRIVER, NULL);

        /* We need at least one event queue */
        srsp->srs_count = sfxge_rx_scale_prop_get(sp);
        if (srsp->srs_count > sip->si_nalloc)
                srsp->srs_count = sip->si_nalloc;
        if (srsp->srs_count < 1)
                srsp->srs_count = 1;

        /* Set up the kstats */
        if ((rc = sfxge_rx_scale_kstat_init(sp)) != 0)
                goto fail1;

        srsp->srs_state = SFXGE_RX_SCALE_INITIALIZED;

        return (0);

fail1:
        DTRACE_PROBE1(fail1, int, rc);
        mutex_destroy(&(srsp->srs_lock));

        return (rc);
}

void
sfxge_rx_scale_update(void *arg)
{
        sfxge_t *sp = arg;
        sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
        sfxge_intr_t *sip;
        processorid_t id;
        unsigned int count;
        unsigned int *tbl;
        unsigned int *rating;
        unsigned int entry;
        int rc;

        mutex_enter(&(srsp->srs_lock));

        if (srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
                rc = EFAULT;
                goto fail1;
        }

        if ((tbl =  kmem_zalloc(sizeof (unsigned int) * SFXGE_RX_SCALE_MAX,
            KM_NOSLEEP)) == NULL) {
                rc = ENOMEM;
                goto fail2;
        }

        sip = &(sp->s_intr);
        if ((rating = kmem_zalloc(sizeof (unsigned int) * sip->si_nalloc,
            KM_NOSLEEP)) == NULL) {
                rc = ENOMEM;
                goto fail3;
        }

        mutex_enter(&cpu_lock);

        /*
         * Substract any current CPU, core, cache and chip usage from the
         * global contention tables.
         */
        for (id = 0; id < NCPU; id++) {
                ASSERT3U(sfxge_cpu[id], >=, srsp->srs_cpu[id]);
                sfxge_cpu[id] -= srsp->srs_cpu[id];
                srsp->srs_cpu[id] = 0;
        }

        ASSERT(srsp->srs_count != 0);

        /* Choose as many event queues as we need */
        for (count = 0; count < srsp->srs_count; count++) {
                unsigned int index;
                sfxge_evq_t *sep;
                unsigned int choice;
                unsigned int choice_rating;

                bzero(rating, sizeof (unsigned int) * sip->si_nalloc);

                /*
                 * Rate each event queue on its global level of CPU
                 * contention.
                 */
                for (index = 0; index < sip->si_nalloc; index++) {
                        sep = sp->s_sep[index];

                        id = sep->se_cpu_id;
                        rating[index] += sfxge_cpu[id];
                }

                /* Choose the queue with the lowest CPU contention */
                choice = 0;
                choice_rating = rating[0];

                for (index = 1; index < sip->si_nalloc; index++) {
                        if (rating[index] < choice_rating) {
                                choice = index;
                                choice_rating = rating[index];
                        }
                }

                /* Add our choice to the condensed RSS table */
                tbl[count] = choice;

                /* Add information to the global contention tables */
                sep = sp->s_sep[choice];

                id = sep->se_cpu_id;
                srsp->srs_cpu[id]++;
                sfxge_cpu[id]++;
        }

        mutex_exit(&cpu_lock);

        /* Build the expanded RSS table */
        count = 0;
        for (entry = 0; entry < SFXGE_RX_SCALE_MAX; entry++) {
                unsigned int index;

                index = tbl[count];
                count = (count + 1) % srsp->srs_count;

                srsp->srs_tbl[entry] = index;
        }

        /* Program the expanded RSS table into the hardware */
        (void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
            SFXGE_RX_SCALE_MAX);

        mutex_exit(&(srsp->srs_lock));
        kmem_free(rating, sizeof (unsigned int) * sip->si_nalloc);
        kmem_free(tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
        return;

fail3:
        DTRACE_PROBE(fail3);
        kmem_free(tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
fail2:
        DTRACE_PROBE(fail2);
fail1:
        DTRACE_PROBE1(fail1, int, rc);

        mutex_exit(&(srsp->srs_lock));
}

static int
sfxge_rx_scale_start(sfxge_t *sp)
{
        sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
        int rc;

        mutex_enter(&(srsp->srs_lock));

        ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_INITIALIZED);

        /* Clear down the RSS table */
        bzero(srsp->srs_tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);

        (void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
            SFXGE_RX_SCALE_MAX);

        if ((rc = sfxge_toeplitz_hash_init(sp)) != 0)
                goto fail1;

        srsp->srs_state = SFXGE_RX_SCALE_STARTED;

        mutex_exit(&(srsp->srs_lock));

        /* sfxge_t->s_state_lock held */
        (void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
            DDI_SLEEP);

        return (0);

fail1:
        DTRACE_PROBE1(fail1, int, rc);

        mutex_exit(&(srsp->srs_lock));

        return (rc);
}

int
sfxge_rx_scale_count_get(sfxge_t *sp, unsigned int *countp)
{
        sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
        int rc;

        mutex_enter(&(srsp->srs_lock));

        if (srsp->srs_state != SFXGE_RX_SCALE_INITIALIZED &&
            srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
                rc = ENOTSUP;
                goto fail1;
        }

        *countp = srsp->srs_count;

        mutex_exit(&(srsp->srs_lock));

        return (0);

fail1:
        DTRACE_PROBE1(fail1, int, rc);

        mutex_exit(&(srsp->srs_lock));

        return (rc);
}

int
sfxge_rx_scale_count_set(sfxge_t *sp, unsigned int count)
{
        sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
        sfxge_intr_t *sip = &(sp->s_intr);
        int dispatch = 1;
        int rc;

        if (count < 1 || count > sip->si_nalloc) {
                rc = EINVAL;
                goto fail1;
        }

        mutex_enter(&(srsp->srs_lock));

        if (srsp->srs_state != SFXGE_RX_SCALE_INITIALIZED &&
            srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
                rc = ENOTSUP;
                goto fail2;
        }

        srsp->srs_count = count;

        if (srsp->srs_state != SFXGE_RX_SCALE_STARTED)
                dispatch = 0;

        mutex_exit(&(srsp->srs_lock));

        if (dispatch)
                /* no locks held */
                (void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
                    DDI_SLEEP);

        return (0);

fail2:
        DTRACE_PROBE(fail2);

        mutex_exit(&(srsp->srs_lock));

fail1:
        DTRACE_PROBE1(fail1, int, rc);

        return (rc);
}

static void
sfxge_rx_scale_stop(sfxge_t *sp)
{
        sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
        processorid_t id;

        mutex_enter(&(srsp->srs_lock));

        ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_STARTED);

        srsp->srs_state = SFXGE_RX_SCALE_INITIALIZED;

        mutex_enter(&cpu_lock);

        /*
         * Substract any current CPU, core, cache and chip usage from the
         * global contention tables.
         */
        for (id = 0; id < NCPU; id++) {
                ASSERT3U(sfxge_cpu[id], >=, srsp->srs_cpu[id]);
                sfxge_cpu[id] -= srsp->srs_cpu[id];
                srsp->srs_cpu[id] = 0;
        }

        mutex_exit(&cpu_lock);

        /* Clear down the RSS table */
        bzero(srsp->srs_tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);

        (void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
            SFXGE_RX_SCALE_MAX);

        mutex_exit(&(srsp->srs_lock));
}

static void
sfxge_rx_scale_fini(sfxge_t *sp)
{
        sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);

        ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_INITIALIZED);

        srsp->srs_state = SFXGE_RX_SCALE_UNINITIALIZED;

        /* Tear down the kstats */
        sfxge_rx_scale_kstat_fini(sp);

        srsp->srs_count = 0;

        mutex_destroy(&(srsp->srs_lock));

        /* Destroy tables */
        kmem_free(srsp->srs_cpu, sizeof (unsigned int) * NCPU);
        srsp->srs_cpu = NULL;

        sfxge_toeplitz_hash_fini(sp);
}

int
sfxge_rx_init(sfxge_t *sp)
{
        sfxge_intr_t *sip = &(sp->s_intr);
        char name[MAXNAMELEN];
        int index;
        int rc;

        if (sip->si_state == SFXGE_INTR_UNINITIALIZED) {
                rc = EINVAL;
                goto fail1;
        }

        if ((rc = sfxge_rx_scale_init(sp)) != 0)
                goto fail2;

        (void) snprintf(name, MAXNAMELEN - 1, "%s%d_rx_packet_cache",
            ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));

        sp->s_rpc = kmem_cache_create(name, sizeof (sfxge_rx_packet_t),
            SFXGE_CPU_CACHE_SIZE, sfxge_rx_packet_ctor, sfxge_rx_packet_dtor,
            NULL, sp, NULL, 0);
        ASSERT(sp->s_rpc != NULL);

        (void) snprintf(name, MAXNAMELEN - 1, "%s%d_rxq_cache",
            ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));

        sp->s_rqc = kmem_cache_create(name, sizeof (sfxge_rxq_t),
            SFXGE_CPU_CACHE_SIZE, sfxge_rx_qctor, sfxge_rx_qdtor, NULL, sp,
            NULL, 0);
        ASSERT(sp->s_rqc != NULL);

        sp->s_rx_pkt_mem_max = ddi_prop_get_int64(DDI_DEV_T_ANY, sp->s_dip,
            DDI_PROP_DONTPASS, "rx_pkt_mem_max", 0); /* disabled */

        /* Initialize the receive queue(s) */
        for (index = 0; index < sip->si_nalloc; index++) {
                if ((rc = sfxge_rx_qinit(sp, index)) != 0)
                        goto fail3;
        }

        sp->s_rx_coalesce_mode = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
            DDI_PROP_DONTPASS, "rx_coalesce_mode", SFXGE_RX_COALESCE_OFF);

        return (0);

fail3:
        DTRACE_PROBE(fail3);

        /* Tear down the receive queue(s) */
        while (--index >= 0)
                sfxge_rx_qfini(sp, index);

        kmem_cache_destroy(sp->s_rqc);
        sp->s_rqc = NULL;

        kmem_cache_destroy(sp->s_rpc);
        sp->s_rpc = NULL;

        sfxge_rx_scale_fini(sp);

fail2:
        DTRACE_PROBE(fail2);
fail1:
        DTRACE_PROBE1(fail1, int, rc);

        return (rc);
}

int
sfxge_rx_start(sfxge_t *sp)
{
        sfxge_mac_t *smp = &(sp->s_mac);
        sfxge_intr_t *sip;
        const efx_nic_cfg_t *encp;
        size_t hdrlen, align;
        int index;
        int rc;

        mutex_enter(&(smp->sm_lock));

        /* Calculate the receive packet buffer size and alignment */
        sp->s_rx_buffer_size = EFX_MAC_PDU(sp->s_mtu);

        encp = efx_nic_cfg_get(sp->s_enp);

        /* Packet buffer allocations are cache line aligned */
        EFSYS_ASSERT3U(encp->enc_rx_buf_align_start, <=, SFXGE_CPU_CACHE_SIZE);

        if (sp->s_family == EFX_FAMILY_HUNTINGTON) {
                sp->s_rx_prefix_size = encp->enc_rx_prefix_size;

                hdrlen = sp->s_rx_prefix_size + sizeof (struct ether_header);

                /* Ensure IP headers are 32bit aligned */
                sp->s_rx_buffer_align = P2ROUNDUP(hdrlen, 4) - hdrlen;
                sp->s_rx_buffer_size += sp->s_rx_buffer_align;

        } else if (encp->enc_features & EFX_FEATURE_LFSR_HASH_INSERT) {
                sp->s_rx_prefix_size = encp->enc_rx_prefix_size;

                /*
                 * Place the start of the buffer a prefix length minus 2
                 * before the start of a cache line. This ensures that the
                 * last two bytes of the prefix (which is where the LFSR hash
                 * is located) are in the same cache line as the headers, and
                 * the IP header is 32-bit aligned.
                 */
                sp->s_rx_buffer_align =
                    SFXGE_CPU_CACHE_SIZE - (encp->enc_rx_prefix_size - 2);
                sp->s_rx_buffer_size += sp->s_rx_buffer_align;
        } else {
                sp->s_rx_prefix_size = 0;

                /*
                 * Place the start of the buffer 2 bytes after a cache line
                 * boundary so that the headers fit into the cache line and
                 * the IP header is 32-bit aligned.
                 */
                hdrlen = sp->s_rx_prefix_size + sizeof (struct ether_header);

                sp->s_rx_buffer_align = P2ROUNDUP(hdrlen, 4) - hdrlen;
                sp->s_rx_buffer_size += sp->s_rx_buffer_align;
        }

        /* Align end of packet buffer for RX DMA end padding */
        align = MAX(1, encp->enc_rx_buf_align_end);
        EFSYS_ASSERT(ISP2(align));
        sp->s_rx_buffer_size = P2ROUNDUP(sp->s_rx_buffer_size, align);

        /* Initialize the receive module */
        if ((rc = efx_rx_init(sp->s_enp)) != 0)
                goto fail1;

        mutex_exit(&(smp->sm_lock));

        if ((rc = sfxge_rx_scale_start(sp)) != 0)
                goto fail2;

        /* Start the receive queue(s) */
        sip = &(sp->s_intr);
        for (index = 0; index < sip->si_nalloc; index++) {
                if ((rc = sfxge_rx_qstart(sp, index)) != 0)
                        goto fail3;
        }

        ASSERT3U(sp->s_srp[0]->sr_state, ==, SFXGE_RXQ_STARTED);
        /* It is sufficient to have Rx scale initialized */
        ASSERT3U(sp->s_rx_scale.srs_state, ==, SFXGE_RX_SCALE_STARTED);
        rc = efx_mac_filter_default_rxq_set(sp->s_enp, sp->s_srp[0]->sr_erp,
            sp->s_rx_scale.srs_count > 1);
        if (rc != 0)
                goto fail4;

        return (0);

fail4:
        DTRACE_PROBE(fail4);

fail3:
        DTRACE_PROBE(fail3);

        /* Stop the receive queue(s) */
        while (--index >= 0)
                sfxge_rx_qstop(sp, index);

        sfxge_rx_scale_stop(sp);

fail2:
        DTRACE_PROBE(fail2);

        mutex_enter(&(smp->sm_lock));

        /* Tear down the receive module */
        efx_rx_fini(sp->s_enp);

fail1:
        DTRACE_PROBE1(fail1, int, rc);

        mutex_exit(&(smp->sm_lock));

        return (rc);
}

void
sfxge_rx_coalesce_mode_get(sfxge_t *sp, sfxge_rx_coalesce_mode_t *modep)
{
        *modep = sp->s_rx_coalesce_mode;
}

int
sfxge_rx_coalesce_mode_set(sfxge_t *sp, sfxge_rx_coalesce_mode_t mode)
{
        int rc;

        switch (mode) {
        case SFXGE_RX_COALESCE_OFF:
        case SFXGE_RX_COALESCE_DISALLOW_PUSH:
        case SFXGE_RX_COALESCE_ALLOW_PUSH:
                break;

        default:
                rc = EINVAL;
                goto fail1;
        }

        sp->s_rx_coalesce_mode = mode;

        return (0);

fail1:
        DTRACE_PROBE1(fail1, int, rc);

        return (rc);
}

void
sfxge_rx_stop(sfxge_t *sp)
{
        sfxge_mac_t *smp = &(sp->s_mac);
        sfxge_intr_t *sip = &(sp->s_intr);
        efx_nic_t *enp = sp->s_enp;
        int index;

        ASSERT(mutex_owned(&(sp->s_state_lock)));

        efx_mac_filter_default_rxq_clear(enp);

        /* Stop the receive queue(s) */
        index = sip->si_nalloc;
        while (--index >= 0) {
                /* TBD: Flush RXQs in parallel; HW has limit + may need retry */
                sfxge_rx_qstop(sp, index);
        }

        sfxge_rx_scale_stop(sp);

        mutex_enter(&(smp->sm_lock));

        /* Tear down the receive module */
        efx_rx_fini(enp);

        sp->s_rx_buffer_align = 0;
        sp->s_rx_prefix_size = 0;
        sp->s_rx_buffer_size = 0;

        mutex_exit(&(smp->sm_lock));
}

unsigned int
sfxge_rx_loaned(sfxge_t *sp)
{
        sfxge_intr_t *sip = &(sp->s_intr);
        int index;
        unsigned int loaned;

        ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);

        loaned = 0;
        for (index = 0; index < sip->si_nalloc; index++) {
                sfxge_rxq_t *srp = sp->s_srp[index];
                sfxge_evq_t *sep = sp->s_sep[srp->sr_index];

                mutex_enter(&(sep->se_lock));

                loaned += sfxge_rx_qfpp_swizzle(srp);

                mutex_exit(&(sep->se_lock));
        }

        return (loaned);
}

void
sfxge_rx_fini(sfxge_t *sp)
{
        sfxge_intr_t *sip = &(sp->s_intr);
        int index;

        ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);

        sp->s_rx_coalesce_mode = SFXGE_RX_COALESCE_OFF;

        /* Tear down the receive queue(s) */
        index = sip->si_nalloc;
        while (--index >= 0)
                sfxge_rx_qfini(sp, index);

        ASSERT3U(sp->s_rx_pkt_mem_alloc, ==, 0);

        kmem_cache_destroy(sp->s_rqc);
        sp->s_rqc = NULL;

        kmem_cache_destroy(sp->s_rpc);
        sp->s_rpc = NULL;

        sfxge_rx_scale_fini(sp);
}