/*
 * Copyright (c) 2011 NetApp, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 *
 * Copyright 2015 Pluribus Networks Inc.
 * Copyright 2019 Joyent, Inc.
 * Copyright 2022 OmniOS Community Edition (OmniOSce) Association.
 * Copyright 2026 Oxide Computer Company
 */

#include <sys/param.h>
#include <sys/linker_set.h>
#include <sys/ioctl.h>
#include <sys/uio.h>
#include <sys/viona_io.h>

#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <assert.h>
#include <pthread.h>
#include <signal.h>
#include <stdbool.h>
#include <poll.h>
#include <libdladm.h>
#include <libdllink.h>
#include <libdlvnic.h>

#include <machine/vmm.h>
#include <vmmapi.h>

#include "bhyverun.h"
#include "config.h"
#include "debug.h"
#include "pci_emul.h"
#include "virtio.h"
#include "iov.h"
#include "virtio_net.h"

/*
 * This is the default number of queues allocated and advertised via the
 * multi-queue feature. It can be overridden via the `qpair` device option.
 */
#define VIONA_DEFAULT_MAX_QPAIR         8

/*
 * These are the default TX and RX ring sizes. They can be overidden with the
 * `vqsize`, `rxvqsize` and `txvqsize` options. The first sets both values to
 * the same while the last two allow setting different values for TX and RX.
 *
 * It is common to have asymmetry here as the RX queue must absorb host-side
 * burstiness while some backpressure is desirable on the TX side.
 */
#define VIONA_DEFAULT_TX_RINGSZ         256
#define VIONA_DEFAULT_RX_RINGSZ         1024

#define VIONA_CTLQ_SIZE                 64
#define VIONA_CTLQ_MAXSEGS              32

#define VIONA_DEFAULT_LINK_SPEED        1000 /* In Mb/s, so this is 1Gb/s */

/*
 * These macros work in terms of TX/RX queues only, which is always what we
 * need when interfacing with viona since the control queue is implemented
 * entirely in userspace.
 */
#define VIONA_P2QS(p)           ((p) * 2)
#define VIONA_NRINGS(sc)        ((sc)->vsc_consts.vc_nvq - 1)
#define VIONA_USABLE_RINGS(sc)  (VIONA_P2QS((sc)->vsc_vq_usepairs))
#define VIONA_RING_VALID(sc, r) ((r) < VIONA_NRINGS(sc))

#define VIONA_RING(sc, n)       (&(sc)->vsc_queues[(n)])
#define VIONA_RXQ(sc, n)        (VIONA_RING(sc, (n) * 2))
#define VIONA_TXQ(sc, n)        (VIONA_RING(sc, (n) * 2 + 1))

#define IS_RXQ(i)               ((i) % 2 == 0)
#define IS_TXQ(i)               ((i) % 2 != 0)

/*
 * The control queue is always in the last slot of allocated rings, regardless
 * of how many rings are in use.
 */
#define VIONA_CTLQ_NUM(sc)      (VIONA_NRINGS(sc))
#define VIONA_RING_CTLQ(sc, r)  (r == VIONA_CTLQ_NUM(sc))
#define VIONA_CTLQ(sc)          (VIONA_RING(sc, VIONA_CTLQ_NUM(sc)))

/*
 * Debug printf
 */
static volatile int pci_viona_debug;
#define DPRINTF(fmt, arg...) \
        do { \
                if (pci_viona_debug) { \
                        FPRINTLN(stdout, fmt, ##arg); \
                        fflush(stdout); \
                } \
        } while (0)
#define WPRINTF(fmt, arg...) FPRINTLN(stderr, fmt, ##arg)

/*
 * Per-device softc
 */
struct pci_viona_softc {
        struct virtio_softc     vsc_vs;
        struct virtio_consts    vsc_consts;
        struct vqueue_info      *vsc_queues;
        pthread_mutex_t         vsc_mtx;

        struct virtio_net_config vsc_config;

        datalink_id_t   vsc_linkid;
        int             vsc_vnafd;

        /* Configurable parameters */
        char            vsc_linkname[MAXLINKNAMELEN];
        uint64_t        vsc_feature_mask;
        uint16_t        vsc_vq_usepairs; /* RX/TX pairs in use */
        uint16_t        vsc_vq_rxsize;  /* size of each RX queue */
        uint16_t        vsc_vq_txsize;  /* size of each TX queue */

        bool            vsc_resetting;
        bool            vsc_msix_active;

        viona_promisc_t vsc_promisc;            /* Current promisc mode */
        bool            vsc_promisc_promisc;    /* PROMISC enabled */
        bool            vsc_promisc_allmulti;   /* ALLMULTI enabled */
        bool            vsc_promisc_umac;       /* unicast MACs sent */
        bool            vsc_promisc_mmac;       /* multicast MACs sent */
};

static int pci_viona_cfgread(void *, int, int, uint32_t *);
static int pci_viona_cfgwrite(void *, int, int, uint32_t);
static uint64_t pci_viona_get_hv_features(void *, bool);
static void pci_viona_set_hv_features(void *, uint64_t *);
static void pci_viona_qnotify(void *, struct vqueue_info *);
static void pci_viona_ctlqnotify(void *, struct vqueue_info *);
static void pci_viona_ring_init(void *, uint64_t, bool);
static void pci_viona_reset(void *);
static void pci_viona_ring_set_msix(void *, int);
static void pci_viona_update_msix(void *, uint64_t);

static virtio_capstr_t viona_caps[] = {
        { VIRTIO_NET_F_CSUM,            "VIRTIO_NET_F_CSUM" },
        { VIRTIO_NET_F_GUEST_CSUM,      "VIRTIO_NET_F_GUEST_CSUM" },
        { VIRTIO_NET_F_MTU,             "VIRTIO_NET_F_MTU" },
        { VIRTIO_NET_F_MAC,             "VIRTIO_NET_F_MAC" },
        { VIRTIO_NET_F_GSO_DEPREC,      "VIRTIO_NET_F_GSO_DEPREC" },
        { VIRTIO_NET_F_GUEST_TSO4,      "VIRTIO_NET_F_GUEST_TSO4" },
        { VIRTIO_NET_F_GUEST_TSO6,      "VIRTIO_NET_F_GUEST_TSO6" },
        { VIRTIO_NET_F_GUEST_ECN,       "VIRTIO_NET_F_GUEST_ECN" },
        { VIRTIO_NET_F_GUEST_UFO,       "VIRTIO_NET_F_GUEST_UFO" },
        { VIRTIO_NET_F_HOST_TSO4,       "VIRTIO_NET_F_HOST_TSO4" },
        { VIRTIO_NET_F_HOST_TSO6,       "VIRTIO_NET_F_HOST_TSO6" },
        { VIRTIO_NET_F_HOST_ECN,        "VIRTIO_NET_F_HOST_ECN" },
        { VIRTIO_NET_F_HOST_UFO,        "VIRTIO_NET_F_HOST_UFO" },
        { VIRTIO_NET_F_MRG_RXBUF,       "VIRTIO_NET_F_MRG_RXBUF" },
        { VIRTIO_NET_F_STATUS,          "VIRTIO_NET_F_STATUS" },
        { VIRTIO_NET_F_CTRL_VQ,         "VIRTIO_NET_F_CTRL_VQ" },
        { VIRTIO_NET_F_CTRL_RX,         "VIRTIO_NET_F_CTRL_RX" },
        { VIRTIO_NET_F_CTRL_VLAN,       "VIRTIO_NET_F_CTRL_VLAN" },
        { VIRTIO_NET_F_GUEST_ANNOUNCE,  "VIRTIO_NET_F_GUEST_ANNOUNCE" },
        { VIRTIO_NET_F_MQ,              "VIRTIO_NET_F_MQ" },
        { VIRTIO_F_CTRL_MAC_ADDR,       "VIRTIO_F_CTRL_MAC_ADDR" },
        { VIRTIO_NET_F_SPEED_DUPLEX,    "VIRTIO_NET_F_SPEED_DUPLEX" },
};

static struct virtio_consts viona_vi_consts = {
        .vc_name                = "viona",
        .vc_nvq                 = 0,    /* set in pci_viona_qalloc() */
        .vc_max_nvq             = 0,    /* set in pci_viona_parse_opts() */
        .vc_cfgsize             = sizeof (struct virtio_net_config),
        .vc_cfgread             = pci_viona_cfgread,
        .vc_cfgwrite            = pci_viona_cfgwrite,
        .vc_set_msix            = pci_viona_ring_set_msix,
        .vc_update_msix         = pci_viona_update_msix,
        .vc_reset               = pci_viona_reset,
        .vc_qinit               = pci_viona_ring_init,
        .vc_qnotify             = pci_viona_qnotify,
        .vc_hv_features         = pci_viona_get_hv_features,
        .vc_apply_features      = pci_viona_set_hv_features,
        /*
         * The following fields are populated using the response from the
         * viona driver during initialisation, augmented with the additional
         * capabilities emulated in userspace.
         */
        .vc_hv_caps_legacy      = 0,
        .vc_hv_caps_modern      = 0,

        .vc_capstr =            viona_caps,
        .vc_ncapstr =           ARRAY_SIZE(viona_caps),
};

static void
pci_viona_ring_reset(struct pci_viona_softc *sc, int ring)
{
        DPRINTF("viona: ring reset 0x%x", ring);
        for (;;) {
                int res;

                res = ioctl(sc->vsc_vnafd, VNA_IOC_RING_RESET, ring);
                if (res == 0) {
                        break;
                } else if (errno != EINTR) {
                        WPRINTF("ioctl viona ring 0x%x reset failed %d",
                            ring, errno);
                        return;
                }
        }
}

static bool
pci_viona_set_usepairs(struct pci_viona_softc *sc, uint16_t pairs)
{
        const uint16_t opairs = sc->vsc_vq_usepairs;

        DPRINTF("QUSE pairs 0x%x -> 0x%x", opairs, pairs);

        if (opairs == pairs)
                return (true);

        if (ioctl(sc->vsc_vnafd, VNA_IOC_SET_USEPAIRS, pairs) != 0) {
                WPRINTF("error setting viona use pairs(0x%x): %s",
                    pairs, strerror(errno));
                return (false);
        }
        sc->vsc_vq_usepairs = pairs;
        return (true);
}

static bool
pci_viona_qalloc(struct pci_viona_softc *sc, int pairs)
{
        struct virtio_consts *vc = &sc->vsc_consts;
        struct vqueue_info *queues;
        int nqueues, oqueues;

        oqueues = vc->vc_nvq;
        /* Add one for the control queue */
        nqueues = VIONA_P2QS(pairs) + 1;
        DPRINTF("QALLOC pairs 0x%x (0x%x -> 0x%x)", pairs,
            oqueues, nqueues);

        if (oqueues == nqueues)
                return (true);

        queues = recallocarray(sc->vsc_queues, oqueues, nqueues,
            sizeof (struct vqueue_info));
        if (queues == NULL) {
                WPRINTF("Failed to allocate memory changing queues from "
                    "0x%x to 0x%x", oqueues, nqueues);
                return (false);
        }
        sc->vsc_queues = queues;
        vc->vc_nvq = nqueues;

        for (uint_t i = 0; i < vc->vc_nvq; i++) {
                sc->vsc_queues[i].vq_qsize =
                    IS_RXQ(i) ? sc->vsc_vq_rxsize : sc->vsc_vq_txsize;
                sc->vsc_queues[i].vq_notify = NULL;
        }
        VIONA_CTLQ(sc)->vq_qsize = VIONA_CTLQ_SIZE;
        VIONA_CTLQ(sc)->vq_notify = pci_viona_ctlqnotify;

        vi_queue_linkup(&sc->vsc_vs, sc->vsc_queues);

        if (ioctl(sc->vsc_vnafd, VNA_IOC_SET_PAIRS, pairs) != 0) {
                WPRINTF("error setting viona queue pairs(0x%x): %s",
                    pairs, strerror(errno));
                return (false);
        }

        return (true);
}

static void
pci_viona_reset(void *vsc)
{
        struct pci_viona_softc *sc = vsc;

        DPRINTF("viona: device reset requested !");

        vi_reset_dev(&sc->vsc_vs);

        /* Reset all TX/RX rings */
        for (uint16_t i = 0; i < VIONA_NRINGS(sc); i++)
                pci_viona_ring_reset(sc, i);

        /* Shrink back down to one queue pair */
        VERIFY(pci_viona_set_usepairs(sc, 1));
        VERIFY(pci_viona_qalloc(sc, 1));
}

static const char *
pci_viona_promisc_descr(viona_promisc_t mode)
{
        switch (mode) {
        case VIONA_PROMISC_NONE:
                return ("none");
        case VIONA_PROMISC_MULTI:
                return ("multicast");
        case VIONA_PROMISC_ALL:
                return ("all");
        default:
                abort();
        }
}

static int
pci_viona_eval_promisc(struct pci_viona_softc *sc)
{
        viona_promisc_t mode = VIONA_PROMISC_NONE;
        int err = 0;

        /*
         * If the guest has explicitly requested promiscuous mode or has sent a
         * non-empty unicast MAC address table, then set viona to promiscuous
         * mode. Otherwise, if the guest has explicitly requested multicast
         * promiscuity or has sent a non-empty multicast MAC address table,
         * then set viona to multicast promiscuous mode.
         */
        if (sc->vsc_promisc_promisc || sc->vsc_promisc_umac)
                mode = VIONA_PROMISC_ALL;
        else if (sc->vsc_promisc_allmulti || sc->vsc_promisc_mmac)
                mode = VIONA_PROMISC_MULTI;

        if (mode != sc->vsc_promisc) {
                DPRINTF("viona: setting promiscuous mode to %d (%s)",
                    mode, pci_viona_promisc_descr(mode));
                DPRINTF("       promisc=%u, umac=%u, allmulti=%u, mmac=%u",
                    sc->vsc_promisc_promisc, sc->vsc_promisc_umac,
                    sc->vsc_promisc_allmulti, sc->vsc_promisc_mmac);

                err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_PROMISC, mode);
                if (err == 0)
                        sc->vsc_promisc = mode;
                else
                        WPRINTF("ioctl viona set promisc failed %d", errno);
        }

        return (err);
}

static uint8_t
pci_viona_control_rx(struct vqueue_info *vq, const virtio_net_ctrl_hdr_t *hdr,
    iov_bunch_t *iob)
{
        struct pci_viona_softc *sc = (struct pci_viona_softc *)vq->vq_vs;
        uint8_t v;

        if (!iov_bunch_copy(iob, &v, sizeof (v))) {
                EPRINTLN("viona: bad control RX data");
                return (VIRTIO_NET_CQ_ERR);
        }

        switch (hdr->vnch_command) {
        case VIRTIO_NET_CTRL_RX_PROMISC:
                DPRINTF("viona: ctrl RX promisc %d", v);
                sc->vsc_promisc_promisc = (v != 0);
                break;
        case VIRTIO_NET_CTRL_RX_ALLMULTI:
                DPRINTF("viona: ctrl RX allmulti %d", v);
                sc->vsc_promisc_allmulti = (v != 0);
                break;
        default:
                /*
                 * VIRTIO_NET_F_CTRL_RX_EXTRA was not offered so no other
                 * commands are expected.
                 */
                EPRINTLN("viona: unrecognised RX control cmd %u",
                    hdr->vnch_command);
                return (VIRTIO_NET_CQ_ERR);
        }

        if (pci_viona_eval_promisc(sc) == 0)
                return (VIRTIO_NET_CQ_OK);
        return (VIRTIO_NET_CQ_ERR);
}

static void
pci_viona_control_mac_dump(const char *tag, iov_bunch_t *iob, uint32_t cnt)
{
        if (!pci_viona_debug) {
                (void) iov_bunch_skip(iob, cnt * ETHERADDRL);
                return;
        }

        DPRINTF("-- %s MAC TABLE (entries: %u)", tag, cnt);

        for (uint32_t i = 0; i < cnt; i++) {
                ether_addr_t mac;

                if (!iov_bunch_copy(iob, &mac, sizeof (mac)))
                        return;

                DPRINTF("   [%2d] %s", i, ether_ntoa((struct ether_addr *)mac));
        }
}

static uint8_t
pci_viona_control_mac(struct vqueue_info *vq, const virtio_net_ctrl_hdr_t *hdr,
    iov_bunch_t *iob)
{
        struct pci_viona_softc *sc = (struct pci_viona_softc *)vq->vq_vs;

        switch (hdr->vnch_command) {
        case VIRTIO_NET_CTRL_MAC_TABLE_SET: {
                virtio_net_ctrl_mac_t table;

                DPRINTF("viona: ctrl MAC table set");

                /*
                 * We advertise VIRTIO_NET_F_CTRL_RX and therefore need to
                 * accept VIRTIO_NET_CTRL_MAC, but we don't support passing
                 * changes in the MAC address lists down to viona.
                 * Instead, we set flags to indicate if the guest has sent
                 * any MAC addresses for each table, and use these to determine
                 * the resulting promiscuous mode, see pci_viona_eval_promisc()
                 * above.
                 */

                /* Unicast MAC table */
                if (!iov_bunch_copy(iob,
                    &table.vncm_entries, sizeof (table.vncm_entries))) {
                        EPRINTLN("viona: bad control MAC unicast header");
                        return (VIRTIO_NET_CQ_ERR);
                }
                sc->vsc_promisc_umac = (table.vncm_entries != 0);
                pci_viona_control_mac_dump("UNICAST", iob, table.vncm_entries);

                /* Multicast MAC table */
                if (!iov_bunch_copy(iob,
                    &table.vncm_entries, sizeof (table.vncm_entries))) {
                        EPRINTLN("viona: bad control MAC multicast header");
                        return (VIRTIO_NET_CQ_ERR);
                }
                sc->vsc_promisc_mmac = (table.vncm_entries != 0);
                pci_viona_control_mac_dump("MULTICAST", iob,
                    table.vncm_entries);

                break;
        }
        case VIRTIO_NET_CTRL_MAC_ADDR_SET:
                /* disallow setting the primary filter MAC address */
                DPRINTF("viona: ctrl MAC addr set with 0x%x bytes",
                    iob->ib_remain);
                return (VIRTIO_NET_CQ_ERR);
        default:
                EPRINTLN("viona: unrecognised MAC control cmd %u",
                    hdr->vnch_command);
                return (VIRTIO_NET_CQ_ERR);
        }

        if (pci_viona_eval_promisc(sc) == 0)
                return (VIRTIO_NET_CQ_OK);
        return (VIRTIO_NET_CQ_ERR);
}

static uint8_t
pci_viona_control_mq(struct vqueue_info *vq, const virtio_net_ctrl_hdr_t *hdr,
    iov_bunch_t *iob)
{
        struct pci_viona_softc *sc = (struct pci_viona_softc *)vq->vq_vs;

        switch (hdr->vnch_command) {
        case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET: {
                virtio_net_ctrl_mq_t mq;

                if (!iov_bunch_copy(iob, &mq, sizeof (mq))) {
                        EPRINTLN("viona: bad control MQ data");
                        return (VIRTIO_NET_CQ_ERR);
                }

                DPRINTF("VQ PAIRS SET 0x%x", mq.virtqueue_pairs);
                if (mq.virtqueue_pairs < 1 ||
                    VIONA_P2QS(mq.virtqueue_pairs) > VIONA_NRINGS(sc)) {
                        EPRINTLN("viona: invalid VQ pairs from guest 0x%x",
                            mq.virtqueue_pairs);
                        return (VIRTIO_NET_CQ_ERR);
                }
                if (!pci_viona_set_usepairs(sc, mq.virtqueue_pairs))
                        return (VIRTIO_NET_CQ_ERR);
                break;
        }
        default:
                EPRINTLN("viona: unrecognised MQ control cmd %u",
                    hdr->vnch_command);
                return (VIRTIO_NET_CQ_ERR);
        }

        return (VIRTIO_NET_CQ_OK);
}

static void
pci_viona_control(struct vqueue_info *vq)
{
        struct iovec iov[VIONA_CTLQ_MAXSEGS];
        virtio_net_ctrl_hdr_t hdr;
        struct vi_req req = { 0 };
        iov_bunch_t iob;
        uint8_t *ackp;
        uint32_t wlen = 0;
        size_t len;
        int niov;

        niov = vq_getchain(vq, iov, VIONA_CTLQ_MAXSEGS, &req);
        assert(niov >= 1 && niov <= VIONA_CTLQ_MAXSEGS);

        /*
         * Since we support the modern interface we must accept a flexible
         * layout here. Even with that we can do some basic checks - there have
         * to be at least two descriptors and only a single writable one sized
         * for one byte. Check the incoming message to make sure it matches
         * this layout and drop the entire chain if not.
         */
        if (niov < 2 || req.writable != 1 || req.readable + 1 != niov ||
            iov[req.readable].iov_len != sizeof (uint8_t)) {
                EPRINTLN("viona: bad control chain, niov=0x%x, w=0x%x, r=0x%x",
                    niov, req.writable, req.readable);
                goto drop;
        }

        len = iov_bunch_init(&iob, iov, niov);
        if (!iov_bunch_copy(&iob, &hdr, sizeof (hdr))) {
                EPRINTLN("viona: control header copy failed, len=0x%x", len);
                goto drop;
        }

        /*
         * Writable iovecs start at iov[req.readable], and we've already
         * checked that there is only one writable, it's at the end, and the
         * right size; it's the acknowledgement byte.
         */
        ackp = (uint8_t *)iov[req.readable].iov_base;
        iob.ib_remain--;

        switch (hdr.vnch_class) {
        case VIRTIO_NET_CTRL_RX:
                *ackp = pci_viona_control_rx(vq, &hdr, &iob);
                break;
        case VIRTIO_NET_CTRL_MAC:
                *ackp = pci_viona_control_mac(vq, &hdr, &iob);
                break;
        case VIRTIO_NET_CTRL_MQ:
                *ackp = pci_viona_control_mq(vq, &hdr, &iob);
                break;
        default:
                EPRINTLN("viona: unrecognised control class %u, cmd %u",
                    hdr.vnch_class, hdr.vnch_command);
                *ackp = VIRTIO_NET_CQ_ERR;
                break;
        }

        /* We've written the status byte */
        wlen++;

drop:
        vq_relchain(vq, req.idx, wlen);
}

static void
pci_viona_process_ctrlq(struct vqueue_info *vq)
{
        for (;;) {
                vq_kick_disable(vq);

                while (vq_has_descs(vq))
                        pci_viona_control(vq);

                vq_kick_enable(vq);

                /*
                 * One more check in case a late addition raced with
                 * re-enabling kicks. Note that vq_kick_enable() includes a
                 * memory barrier.
                 */

                if (!vq_has_descs(vq))
                        break;
        }

        vq_endchains(vq, /* used_all_avail= */1);
}

static void *
pci_viona_poll_thread(void *param)
{
        struct pci_viona_softc *sc = param;
        pollfd_t pollset;
        const int fd = sc->vsc_vnafd;

        pollset.fd = fd;
        pollset.events = POLLRDBAND;

        for (;;) {
                if (poll(&pollset, 1, -1) < 0) {
                        if (errno == EINTR || errno == EAGAIN) {
                                continue;
                        } else {
                                WPRINTF("pci_viona_poll_thread poll() error %d",
                                    errno);
                                break;
                        }
                }
                if (pollset.revents & POLLRDBAND) {
                        uint_t i;
                        int entries;
                        bool assert_lintr = false;
                        const bool do_msix = pci_msix_enabled(sc->vsc_vs.vs_pi);
                        vioc_intr_poll_mq_t vipm;

                        vipm.vipm_nrings = VIONA_USABLE_RINGS(sc);

                        entries = ioctl(fd, VNA_IOC_INTR_POLL_MQ, &vipm);
                        for (i = 0; entries > 0 && i < vipm.vipm_nrings &&
                            i < VIONA_USABLE_RINGS(sc); i++) {
                                if (!VIONA_INTR_TEST(&vipm, i))
                                        continue;
                                entries--;
                                if (do_msix) {
                                        pci_generate_msix(sc->vsc_vs.vs_pi,
                                            sc->vsc_queues[i].vq_msix_idx);
                                } else {
                                        assert_lintr = true;
                                }
                                if (ioctl(fd, VNA_IOC_RING_INTR_CLR, i) != 0) {
                                        WPRINTF("ioctl viona vq %d intr "
                                            "clear failed %d", i, errno);
                                }
                        }
                        if (assert_lintr) {
                                pthread_mutex_lock(&sc->vsc_mtx);
                                sc->vsc_vs.vs_isr |= VIRTIO_PCI_ISR_INTR;
                                pci_lintr_assert(sc->vsc_vs.vs_pi);
                                pthread_mutex_unlock(&sc->vsc_mtx);
                        }
                }
        }

        pthread_exit(NULL);
}

static void
pci_viona_ring_init(void *vsc, uint64_t pfn, bool modern)
{
        struct pci_viona_softc *sc = vsc;
        int qnum = sc->vsc_vs.vs_curq;
        struct vqueue_info *vq = &sc->vsc_queues[qnum];
        vioc_ring_init_modern_t vna_rim = { 0 };
        int error;
        const bool ctlq = VIONA_RING_CTLQ(sc, qnum);

        DPRINTF("viona: ring init 0x%x", qnum);

        if (!ctlq && !VIONA_RING_VALID(sc, qnum)) {
                DPRINTF("viona: bad ring 0x%d", qnum);
                return;
        }

        if (modern)
                vi_vq_init(&sc->vsc_vs);
        else
                vi_legacy_vq_init(&sc->vsc_vs, pfn);

        if (ctlq)
                return;

        vna_rim.rim_index = qnum;
        vna_rim.rim_qsize = vq->vq_qsize;
        vna_rim.rim_qaddr_desc = vq->vq_desc_gpa;
        vna_rim.rim_qaddr_avail = vq->vq_avail_gpa;
        vna_rim.rim_qaddr_used = vq->vq_used_gpa;

        error = ioctl(sc->vsc_vnafd, VNA_IOC_RING_INIT_MODERN, &vna_rim);

        if (error != 0) {
                WPRINTF("ioctl viona ring 0x%x init failed %d", qnum, errno);
        }
}

static int
pci_viona_viona_init(struct vmctx *ctx, struct pci_viona_softc *sc)
{
        vioc_create_t           vna_create;
        int                     error, version;

        sc->vsc_vnafd = open("/dev/viona", O_RDWR | O_EXCL);
        if (sc->vsc_vnafd == -1) {
                WPRINTF("open viona ctl failed: %d", errno);
                return (-1);
        }

        version = ioctl(sc->vsc_vnafd, VNA_IOC_VERSION, 0);
        if (version != VIONA_CURRENT_INTERFACE_VERSION) {
                (void) close(sc->vsc_vnafd);
                WPRINTF(
                    "ioctl interface version %d != expected %d",
                    version, VIONA_CURRENT_INTERFACE_VERSION);
                return (-1);
        }

        vna_create.c_linkid = sc->vsc_linkid;
        vna_create.c_vmfd = vm_get_device_fd(ctx);
        error = ioctl(sc->vsc_vnafd, VNA_IOC_CREATE, &vna_create);
        if (error != 0) {
                (void) close(sc->vsc_vnafd);
                WPRINTF("ioctl viona create failed %d", errno);
                return (-1);
        }

        return (0);
}

static int
pci_viona_legacy_config(nvlist_t *nvl, const char *opt)
{
        char *config, *name, *tofree, *value;

        if (opt == NULL)
                return (0);

        config = tofree = strdup(opt);
        while ((name = strsep(&config, ",")) != NULL) {
                value = strchr(name, '=');
                if (value != NULL) {
                        *value++ = '\0';
                        set_config_value_node(nvl, name, value);
                } else {
                        set_config_value_node(nvl, "vnic", name);
                }
        }
        free(tofree);
        return (0);
}

static int
pci_viona_parse_opts(struct pci_viona_softc *sc, nvlist_t *nvl)
{
        const char *value, *errstr;
        long long num;
        int err = 0;

        sc->vsc_config.vnc_max_qpair = VIONA_DEFAULT_MAX_QPAIR;
        sc->vsc_vq_txsize = VIONA_DEFAULT_TX_RINGSZ;
        sc->vsc_vq_rxsize = VIONA_DEFAULT_RX_RINGSZ;
        sc->vsc_config.vnc_speed = VIONA_DEFAULT_LINK_SPEED;
        sc->vsc_feature_mask = 0;
        sc->vsc_linkname[0] = '\0';

        value = get_config_value_node(nvl, "feature_mask");
        if (value != NULL) {
                num = strtonumx(value, 0, UINT64_MAX, &errstr, 0);
                if (errstr != NULL) {
                        EPRINTLN("viona: invalid feature_mask '%s': %s",
                            value, errstr);
                        err = -1;
                } else {
                        sc->vsc_feature_mask = num;
                }
        }

        value = get_config_value_node(nvl, "vqsize");
        if (value != NULL) {
                num = strtonumx(value, 2, 32768, &errstr, 0);
                if (errstr != NULL) {
                        EPRINTLN("viona: invalid vqsize '%s': %s",
                            value, errstr);
                        err = -1;
                } else if ((1 << (ffsll(num) - 1)) != num) {
                        EPRINTLN("viona: vqsize '%s' must be power of 2",
                            value);
                        err = -1;
                } else {
                        sc->vsc_vq_txsize = num;
                        sc->vsc_vq_rxsize = num;
                }
        }

        value = get_config_value_node(nvl, "txvqsize");
        if (value != NULL) {
                num = strtonumx(value, 2, 32768, &errstr, 0);
                if (errstr != NULL) {
                        EPRINTLN("viona: invalid txvqsize '%s': %s",
                            value, errstr);
                        err = -1;
                } else if ((1 << (ffsll(num) - 1)) != num) {
                        EPRINTLN("viona: txvqsize '%s' must be power of 2",
                            value);
                        err = -1;
                } else {
                        sc->vsc_vq_txsize = num;
                }
        }

        value = get_config_value_node(nvl, "rxvqsize");
        if (value != NULL) {
                num = strtonumx(value, 2, 32768, &errstr, 0);
                if (errstr != NULL) {
                        EPRINTLN("viona: invalid rxvqsize '%s': %s",
                            value, errstr);
                        err = -1;
                } else if ((1 << (ffsll(num) - 1)) != num) {
                        EPRINTLN("viona: rxvqsize '%s' must be power of 2",
                            value);
                        err = -1;
                } else {
                        sc->vsc_vq_rxsize = num;
                }
        }

        value = get_config_value_node(nvl, "qpair");
        if (value != NULL) {
                num = strtonumx(value, VIONA_MIN_QPAIR, VIONA_MAX_QPAIR,
                    &errstr, 0);
                if (errstr != NULL) {
                        EPRINTLN("viona: invalid qpair '%s': %s",
                            value, errstr);
                        err = -1;
                } else {
                        sc->vsc_config.vnc_max_qpair = num;
                }
        }

        value = get_config_value_node(nvl, "speed");
        if (value != NULL) {
                num = strtonumx(value, 0, INT32_MAX, &errstr, 0);
                if (errstr != NULL) {
                        EPRINTLN("viona: invalid speed '%s': %s",
                            value, errstr);
                        err = -1;
                } else {
                        sc->vsc_config.vnc_speed = num;
                }
        }

        value = get_config_value_node(nvl, "vnic");
        if (value == NULL) {
                EPRINTLN("viona: vnic name required");
                err = -1;
        } else {
                (void) strlcpy(sc->vsc_linkname, value, MAXLINKNAMELEN);
        }

        DPRINTF("viona=%p dev=%s vqsize(TX/RX)=0x%x/0x%x qpair=0x%x "
            "feature_mask=0x%" PRIx64,
            sc, sc->vsc_linkname, sc->vsc_vq_txsize, sc->vsc_vq_rxsize,
            sc->vsc_config.vnc_max_qpair, sc->vsc_feature_mask);
        return (err);
}

static uint16_t
pci_viona_query_mtu(dladm_handle_t handle, datalink_id_t linkid)
{
        char buf[DLADM_PROP_VAL_MAX];
        char *propval = buf;
        uint_t propcnt = 1;

        if (dladm_get_linkprop(handle, linkid, DLADM_PROP_VAL_CURRENT, "mtu",
            &propval, &propcnt) == DLADM_STATUS_OK && propcnt == 1) {
                ulong_t parsed = strtoul(buf, NULL, 10);

                /*
                 * The virtio spec notes that for devices implementing
                 * VIRTIO_NET_F_MTU, that the noted MTU MUST be between
                 * 68-65535, inclusive.
                 */
                if (parsed >= 68 && parsed <= 65535)
                        return (parsed);
        }

        /*
         * Default to 1500 if query is unsuccessful or the result is out of
         * bounds.
         */
        return (1500);
}

static int
pci_viona_free_softstate(struct pci_viona_softc *sc, int err)
{
        pthread_mutex_destroy(&sc->vsc_mtx);
        free(sc->vsc_queues);
        free(sc);

        return (err);
}

static int
pci_viona_init(struct pci_devinst *pi, nvlist_t *nvl)
{
        dladm_handle_t          handle;
        dladm_status_t          status;
        dladm_vnic_attr_t       attr;
        char                    errmsg[DLADM_STRSIZE];
        char                    tname[MAXCOMLEN + 1];
        int                     error;
        struct                  pci_viona_softc *sc;
        struct                  virtio_consts *vc;
        const char              *vnic;
        pthread_t               tid;

        vnic = get_config_value_node(nvl, "vnic");
        if (vnic == NULL) {
                WPRINTF("virtio-viona: vnic required");
                return (1);
        }

        sc = calloc(1, sizeof (struct pci_viona_softc));
        if (sc == NULL) {
                WPRINTF("Failed to allocate memory for soft state");
                return (1);
        }
        vc = &sc->vsc_consts;
        *vc = viona_vi_consts;

        pthread_mutex_init(&sc->vsc_mtx, NULL);

        if (get_config_bool_default("viona.debug", false))
                pci_viona_debug = 1;
        vi_set_debug(&sc->vsc_vs, pci_viona_debug);

        if (pci_viona_parse_opts(sc, nvl) != 0)
                return (pci_viona_free_softstate(sc, 1));

        /* Add one for the control queue */
        vc->vc_max_nvq = VIONA_P2QS(sc->vsc_config.vnc_max_qpair) + 1;

        if ((status = dladm_open(&handle)) != DLADM_STATUS_OK) {
                WPRINTF("could not open /dev/dld");
                return (pci_viona_free_softstate(sc, 1));
        }

        if ((status = dladm_name2info(handle, sc->vsc_linkname, &sc->vsc_linkid,
            NULL, NULL, NULL)) != DLADM_STATUS_OK) {
                WPRINTF("dladm_name2info() for %s failed: %s", vnic,
                    dladm_status2str(status, errmsg));
                dladm_close(handle);
                return (pci_viona_free_softstate(sc, 1));
        }

        if ((status = dladm_vnic_info(handle, sc->vsc_linkid, &attr,
            DLADM_OPT_ACTIVE)) != DLADM_STATUS_OK) {
                WPRINTF("dladm_vnic_info() for %s failed: %s", vnic,
                    dladm_status2str(status, errmsg));
                dladm_close(handle);
                return (pci_viona_free_softstate(sc, 1));
        }
        memcpy(sc->vsc_config.vnc_macaddr, attr.va_mac_addr, ETHERADDRL);
        sc->vsc_config.vnc_status = VIRTIO_NET_S_LINK_UP; /* link always up */
        sc->vsc_config.vnc_duplex = VIRTIO_NET_DUPLEX_FULL;
        sc->vsc_config.vnc_mtu = pci_viona_query_mtu(handle, sc->vsc_linkid);
        dladm_close(handle);

        error = pci_viona_viona_init(pi->pi_vmctx, sc);
        if (error != 0)
                return (pci_viona_free_softstate(sc, 1));

        if (ioctl(sc->vsc_vnafd, VNA_IOC_SET_MTU,
            sc->vsc_config.vnc_mtu) != 0) {
                WPRINTF("error setting viona MTU(%u): %s",
                    sc->vsc_config.vnc_mtu, strerror(errno));
        }

        /* link virtio to softc */
        vi_softc_linkup(&sc->vsc_vs, vc, sc, pi, sc->vsc_queues);
        sc->vsc_vs.vs_mtx = &sc->vsc_mtx;

        /*
         * We initially need to configure a single queue pair. Some legacy
         * drivers will set these up before completing feature negotiation,
         * which is before we know if they support multi-queue.
         */
        if (!pci_viona_qalloc(sc, 1))
                return (pci_viona_free_softstate(sc, 1));
        /* Until the guest tells us otherwise, we'll only use a single pair */
        sc->vsc_vq_usepairs = 1;

        error = pthread_create(&tid, NULL, pci_viona_poll_thread, sc);
        assert(error == 0);
        snprintf(tname, sizeof (tname), "vionapoll:%s", vnic);
        pthread_set_name_np(tid, tname);

        /* initialize config space */
        vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_NET,
            VIRTIO_ID_NETWORK, PCIC_NETWORK);

        /*
         * Guests that do not support CTRL_RX_MAC still generally need to
         * receive multicast packets. Guests that do support this feature will
         * end up setting this flag indirectly via messages on the control
         * queue but it does not hurt to default to multicast promiscuity here
         * and it is what older version of viona did.
         */
        sc->vsc_promisc_mmac = true;
        pci_viona_eval_promisc(sc);

        /* Viona always uses MSI-X */
        if (!vi_intr_init(&sc->vsc_vs, true))
                return (pci_viona_free_softstate(sc, 1));

        if (!vi_pcibar_setup(&sc->vsc_vs))
                return (pci_viona_free_softstate(sc, 1));

        return (0);
}

static int
pci_viona_cfgwrite(void *vsc, int offset, int size, uint32_t value)
{
        struct pci_viona_softc *sc = vsc;
        void *ptr;

        /* We will only ever end up here with an 8, 16 or 32-bit size */
        ASSERT(size == 1 || size == 2 || size == 4);

        /*
         * The driver is allowed to change the MAC address.
         * vnc_macaddr is the first element of vsc_config
         */
        if (offset < (int)sizeof (sc->vsc_config.vnc_macaddr) &&
            offset + size <= (int)sizeof (sc->vsc_config.vnc_macaddr)) {
                ptr = &sc->vsc_config.vnc_macaddr[offset];
                memcpy(ptr, &value, size);
                vq_devcfg_changed(&sc->vsc_vs);
        } else {
                /* silently ignore other writes */
                DPRINTF("viona: write to readonly reg 0x%x", offset);
        }

        return (0);
}

static int
pci_viona_cfgread(void *vsc, int offset, int size, uint32_t *retval)
{
        struct pci_viona_softc *sc = vsc;
        void *ptr;

        ptr = (uint8_t *)&sc->vsc_config + offset;
        memcpy(retval, ptr, size);
        return (0);
}

static void
pci_viona_ring_set_msix(void *vsc, int ring)
{
        struct pci_viona_softc *sc = vsc;
        struct pci_devinst *pi = sc->vsc_vs.vs_pi;
        struct msix_table_entry mte;
        uint16_t tab_index;
        vioc_ring_msi_t vrm;
        int res;

        if (!VIONA_RING_VALID(sc, ring))
                return;

        vrm.rm_index = ring;
        vrm.rm_addr = 0;
        vrm.rm_msg = 0;
        tab_index = sc->vsc_queues[ring].vq_msix_idx;

        if (tab_index != VIRTIO_MSI_NO_VECTOR && sc->vsc_msix_active) {
                mte = pi->pi_msix.table[tab_index];
                if ((mte.vector_control & PCIM_MSIX_VCTRL_MASK) == 0) {
                        vrm.rm_addr = mte.addr;
                        vrm.rm_msg = mte.msg_data;
                }
        }

        DPRINTF("SET MSI ring=0x%x addr=0x%" PRIx64 " msg=0x%" PRIx64,
            vrm.rm_index, vrm.rm_addr, vrm.rm_msg);

        res = ioctl(sc->vsc_vnafd, VNA_IOC_RING_SET_MSI, &vrm);
        if (res != 0) {
                WPRINTF("ioctl viona set_msi %d failed %d", ring, errno);
        }
}

static void
pci_viona_lintrupdate(struct pci_devinst *pi)
{
        struct pci_viona_softc *sc = pi->pi_arg;
        bool msix_on = false;

        pthread_mutex_lock(&sc->vsc_mtx);
        msix_on = pci_msix_enabled(pi) && (pi->pi_msix.function_mask == 0);
        if ((sc->vsc_msix_active && !msix_on) ||
            (msix_on && !sc->vsc_msix_active)) {
                DPRINTF("MSIX %u -> %u", sc->vsc_msix_active, msix_on);
                sc->vsc_msix_active = msix_on;
                /* Update in-kernel ring configs */
                for (uint16_t i = 0; i < VIONA_NRINGS(sc); i++)
                        pci_viona_ring_set_msix(sc, VIONA_RING(sc, i)->vq_num);
        }
        pthread_mutex_unlock(&sc->vsc_mtx);
}

static void
pci_viona_update_msix(void *vsc, uint64_t offset)
{
        struct pci_viona_softc *sc = vsc;
        uint_t tab_index;

        pthread_mutex_lock(&sc->vsc_mtx);
        if (!sc->vsc_msix_active) {
                pthread_mutex_unlock(&sc->vsc_mtx);
                return;
        }

        /*
         * Rather than update every possible MSI-X vector, cheat and use the
         * offset to calculate the entry within the table.  Since this should
         * only be called when a write to the table succeeds, the index should
         * be valid.
         */
        tab_index = offset / MSIX_TABLE_ENTRY_SIZE;

        for (uint16_t i = 0; i < VIONA_NRINGS(sc); i++) {
                struct vqueue_info *vq = VIONA_RING(sc, i);

                if (vq->vq_msix_idx == tab_index)
                        pci_viona_ring_set_msix(vsc, vq->vq_num);
        }

        pthread_mutex_unlock(&sc->vsc_mtx);
}

static void
pci_viona_ctlqnotify(void *vsc, struct vqueue_info *vq)
{
        if (vq_has_descs(vq))
                pci_viona_process_ctrlq(vq);
}

static void
pci_viona_qnotify(void *vsc, struct vqueue_info *vq)
{
        struct pci_viona_softc *sc = vsc;
        int ring = vq->vq_num;
        int error;

        if (!VIONA_RING_VALID(sc, ring))
                return;

        error = ioctl(sc->vsc_vnafd, VNA_IOC_RING_KICK, ring);
        if (error != 0)
                WPRINTF("ioctl viona ring 0x%x kick failed %d", ring, errno);
}

static void
pci_viona_baraddr(struct pci_devinst *pi, int baridx, int enabled,
    uint64_t address)
{
        struct pci_viona_softc *sc = pi->pi_arg;
        int err;

        DPRINTF("BAR%d ADDRESS %" PRIx64 " (%s)",
            baridx, address, enabled == 1 ? "enable": "disable");

        switch (baridx) {
        case VIRTIO_LEGACY_BAR: {
                uint64_t ioport;

                if (enabled == 0) {
                        err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_IOP, 0);
                        if (err != 0)
                                WPRINTF("Uninstall ioport hook fail %d", errno);
                        break;
                }

                /*
                 * Install ioport hook for virtqueue notification.
                 * This is part of the virtio common configuration area so the
                 * address does not change with MSI-X status.
                 */
                ioport = address + VIRTIO_PCI_QUEUE_NOTIFY;
                err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_IOP, ioport);
                if (err != 0) {
                        WPRINTF("Install ioport hook at 0x%x failed %d",
                            ioport, errno);
                }
                break;
        }
        case VIRTIO_MODERN_BAR: {
                virtio_pci_capcfg_t *cfg;
                vioc_notify_mmio_t vim;

                if (enabled == 0) {
                        err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_MMIO, 0);
                        if (err != 0)
                                WPRINTF("Uninstall MMIO hook fail %d", errno);
                        break;
                }

                cfg = vi_pci_cfg_bytype(&sc->vsc_vs, VIRTIO_PCI_CAP_NOTIFY_CFG);
                if (cfg == NULL)
                        break;

                vim.vim_address = address + cfg->c_baroff;
                vim.vim_size = cfg->c_barlen;

                DPRINTF("MODERN BAR NOTIFY address 0x%" PRIx64 " size 0x%x",
                    vim.vim_address, vim.vim_size);

                err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_MMIO, &vim);
                if (err != 0) {
                        WPRINTF(
                            "Install MMIO hook at 0x%" PRIx64 "+0x%x failed %d",
                            vim.vim_address, vim.vim_size, errno);
                }
                break;
        }
        default:
                break;
        }
}

static uint64_t
pci_viona_get_hv_features(void *vsc, bool modern)
{
        struct pci_viona_softc *sc = vsc;
        uint64_t value;
        int err;

        err = ioctl(sc->vsc_vnafd, VNA_IOC_GET_FEATURES, &value);
        if (err != 0)
                WPRINTF("ioctl get host features returned err = %d", errno);
        /*
         * Supplementary device capabilities provided in the userspace
         * component.
         */
        value |= VIRTIO_NET_F_MAC;
        value |= VIRTIO_NET_F_STATUS;
        value |= VIRTIO_NET_F_MTU;
        value |= VIRTIO_NET_F_CTRL_VQ;
        value |= VIRTIO_NET_F_CTRL_RX;
        value |= VIRTIO_NET_F_MQ;
        if (modern)
                value |= VIRTIO_NET_F_SPEED_DUPLEX;

        value &= ~sc->vsc_feature_mask;

        if (modern) {
                value |= VIRTIO_F_VERSION_1;
                sc->vsc_consts.vc_hv_caps_modern = value;
        } else {
                /*
                 * To be a conforming transitional device we must support
                 * arbitrary descriptor layouts on the legacy interface. The
                 * specification is a little ambiguous as it mandates this but
                 * also provides details on how descriptors should be laid out
                 * by transitional drivers that don't negotiate this. Since we
                 * don't make any assumptions about descriptor layout we may as
                 * well set this.
                 */
                value |= VIRTIO_F_ANY_LAYOUT;
                sc->vsc_consts.vc_hv_caps_legacy = value;
        }

        return (value);
}

static void
pci_viona_set_hv_features(void *vsc, uint64_t *value)
{
        struct pci_viona_softc *sc = vsc;
        int err;

        *value &= ~sc->vsc_feature_mask;
        err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_FEATURES, value);
        if (err != 0)
                WPRINTF("ioctl feature negotiation returned err = %d", errno);

        if (*value & VIRTIO_NET_F_MQ) {
                /*
                 * If multi-queue is negotiated then we need to provision all
                 * of the queues we can support. Even if the guest chooses not
                 * to use all of them it will still set them up.
                 */
                DPRINTF("Going MULTIQUEUE with 0x%x pairs!",
                    sc->vsc_config.vnc_max_qpair);
                if (!pci_viona_qalloc(sc, sc->vsc_config.vnc_max_qpair)) {
                        vi_error(&sc->vsc_vs,
                            "Failed to allocate 0x%x queue pairs",
                            sc->vsc_config.vnc_max_qpair);
                }
        }
}

struct pci_devemu pci_de_viona = {
        .pe_emu =               "virtio-net-viona",
        .pe_init =              pci_viona_init,
        .pe_legacy_config =     pci_viona_legacy_config,
        .pe_cfgwrite =          vi_pci_cfgwrite,
        .pe_cfgread =           vi_pci_cfgread,
        .pe_barwrite =          vi_pci_write,
        .pe_barread =           vi_pci_read,
        .pe_baraddr =           pci_viona_baraddr,
        .pe_lintrupdate =       pci_viona_lintrupdate
};
PCI_EMUL_SET(pci_de_viona);