/*-
 * Copyright (c) 2018 VMware, Inc.
 *
 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
 */

/* Driver for VMware Virtual Machine Communication Interface (VMCI) device. */

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/rman.h>
#include <sys/systm.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>

#include <machine/bus.h>

#include "vmci.h"
#include "vmci_doorbell.h"
#include "vmci_driver.h"
#include "vmci_kernel_defs.h"
#include "vmci_queue_pair.h"

static int      vmci_probe(device_t);
static int      vmci_attach(device_t);
static int      vmci_detach(device_t);
static int      vmci_shutdown(device_t);

static int      vmci_map_bars(struct vmci_softc *);
static void     vmci_unmap_bars(struct vmci_softc *);

static int      vmci_config_capabilities(struct vmci_softc *);

static int      vmci_dma_malloc_int(struct vmci_softc *, bus_size_t,
                    bus_size_t, struct vmci_dma_alloc *);
static void     vmci_dma_free_int(struct vmci_softc *,
                    struct vmci_dma_alloc *);

static int      vmci_config_interrupts(struct vmci_softc *);
static int      vmci_config_interrupt(struct vmci_softc *);
static int      vmci_check_intr_cnt(struct vmci_softc *);
static int      vmci_allocate_interrupt_resources(struct vmci_softc *);
static int      vmci_setup_interrupts(struct vmci_softc *);
static void     vmci_dismantle_interrupts(struct vmci_softc *);
static void     vmci_interrupt(void *);
static void     vmci_interrupt_bm(void *);
static void     dispatch_datagrams(void *, int);
static void     process_bitmap(void *, int);

static void     vmci_delayed_work_fn_cb(void *context, int data);

static device_method_t vmci_methods[] = {
        /* Device interface. */
        DEVMETHOD(device_probe,         vmci_probe),
        DEVMETHOD(device_attach,        vmci_attach),
        DEVMETHOD(device_detach,        vmci_detach),
        DEVMETHOD(device_shutdown,      vmci_shutdown),

        DEVMETHOD_END
};

static driver_t vmci_driver = {
        "vmci", vmci_methods, sizeof(struct vmci_softc)
};

DRIVER_MODULE(vmci, pci, vmci_driver, 0, 0);
MODULE_VERSION(vmci, VMCI_VERSION);
const struct {
        uint16_t vendor;
        uint16_t device;
        const char *desc;
} vmci_ids[] = {
        { VMCI_VMWARE_VENDOR_ID, VMCI_VMWARE_DEVICE_ID,
            "VMware Virtual Machine Communication Interface" },
};
MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, vmci, vmci_ids,
    nitems(vmci_ids));

MODULE_DEPEND(vmci, pci, 1, 1, 1);

static struct vmci_softc *vmci_sc;

#define LGPFX   "vmci: "
/*
 * Allocate a buffer for incoming datagrams globally to avoid repeated
 * allocation in the interrupt handler's atomic context.
 */
static uint8_t *data_buffer = NULL;
static uint32_t data_buffer_size = VMCI_MAX_DG_SIZE;

struct vmci_delayed_work_info {
        vmci_work_fn    *work_fn;
        void            *data;
        vmci_list_item(vmci_delayed_work_info) entry;
};

/*
 *------------------------------------------------------------------------------
 *
 * vmci_probe --
 *
 *     Probe to see if the VMCI device is present.
 *
 * Results:
 *     BUS_PROBE_DEFAULT if device exists, ENXIO otherwise.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_probe(device_t dev)
{

        if (pci_get_vendor(dev) == vmci_ids[0].vendor &&
            pci_get_device(dev) == vmci_ids[0].device) {
                device_set_desc(dev, vmci_ids[0].desc);

                return (BUS_PROBE_DEFAULT);
        }

        return (ENXIO);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_attach --
 *
 *     Attach VMCI device to the system after vmci_probe() has been called and
 *     the device has been detected.
 *
 * Results:
 *     0 if success, ENXIO otherwise.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_attach(device_t dev)
{
        struct vmci_softc *sc;
        int error, i;

        sc = device_get_softc(dev);
        sc->vmci_dev = dev;
        vmci_sc = sc;

        data_buffer = NULL;
        sc->vmci_num_intr = 0;
        for (i = 0; i < VMCI_MAX_INTRS; i++) {
                sc->vmci_intrs[i].vmci_irq = NULL;
                sc->vmci_intrs[i].vmci_handler = NULL;
        }

        TASK_INIT(&sc->vmci_interrupt_dq_task, 0, dispatch_datagrams, sc);
        TASK_INIT(&sc->vmci_interrupt_bm_task, 0, process_bitmap, sc);

        TASK_INIT(&sc->vmci_delayed_work_task, 0, vmci_delayed_work_fn_cb, sc);

        pci_enable_busmaster(dev);

        mtx_init(&sc->vmci_spinlock, "VMCI Spinlock", NULL, MTX_SPIN);
        mtx_init(&sc->vmci_delayed_work_lock, "VMCI Delayed Work Lock",
            NULL, MTX_DEF);

        error = vmci_map_bars(sc);
        if (error) {
                VMCI_LOG_ERROR(LGPFX"Failed to map PCI BARs.\n");
                goto fail;
        }

        error = vmci_config_capabilities(sc);
        if (error) {
                VMCI_LOG_ERROR(LGPFX"Failed to configure capabilities.\n");
                goto fail;
        }

        vmci_list_init(&sc->vmci_delayed_work_infos);

        vmci_components_init();
        vmci_util_init();
        error = vmci_qp_guest_endpoints_init();
        if (error) {
                VMCI_LOG_ERROR(LGPFX"vmci_qp_guest_endpoints_init failed.\n");
                goto fail;
        }

        error = vmci_config_interrupts(sc);
        if (error)
                VMCI_LOG_ERROR(LGPFX"Failed to enable interrupts.\n");

fail:
        if (error) {
                vmci_detach(dev);
                return (ENXIO);
        }

        return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_detach --
 *
 *     Detach the VMCI device.
 *
 * Results:
 *     0
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_detach(device_t dev)
{
        struct vmci_softc *sc;

        sc = device_get_softc(dev);

        vmci_qp_guest_endpoints_exit();
        vmci_util_exit();

        vmci_dismantle_interrupts(sc);

        vmci_components_cleanup();

        if mtx_initialized(&sc->vmci_spinlock) {
                taskqueue_drain(taskqueue_thread, &sc->vmci_delayed_work_task);
                mtx_destroy(&sc->vmci_delayed_work_lock);
        }

        if (sc->vmci_res0 != NULL)
                bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
                    VMCI_CONTROL_ADDR, VMCI_CONTROL_RESET);

        if (sc->vmci_notifications_bitmap.dma_vaddr != NULL)
                vmci_dma_free(&sc->vmci_notifications_bitmap);

        vmci_unmap_bars(sc);

        if mtx_initialized(&sc->vmci_spinlock)
                mtx_destroy(&sc->vmci_spinlock);

        pci_disable_busmaster(dev);

        return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_shutdown --
 *
 *     This function is called during system shutdown. We don't do anything.
 *
 * Results:
 *     0
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_shutdown(device_t dev)
{

        return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_map_bars --
 *
 *     Maps the PCI I/O and MMIO BARs.
 *
 * Results:
 *     0 on success, ENXIO otherwise.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_map_bars(struct vmci_softc *sc)
{
        int rid;

        /* Map the PCI I/O BAR: BAR0 */
        rid = PCIR_BAR(0);
        sc->vmci_res0 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IOPORT,
            &rid, RF_ACTIVE);
        if (sc->vmci_res0 == NULL) {
                VMCI_LOG_ERROR(LGPFX"Could not map: BAR0\n");
                return (ENXIO);
        }

        sc->vmci_iot0 = rman_get_bustag(sc->vmci_res0);
        sc->vmci_ioh0 = rman_get_bushandle(sc->vmci_res0);
        sc->vmci_ioaddr = rman_get_start(sc->vmci_res0);

        /* Map the PCI MMIO BAR: BAR1 */
        rid = PCIR_BAR(1);
        sc->vmci_res1 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_MEMORY,
            &rid, RF_ACTIVE);
        if (sc->vmci_res1 == NULL) {
                VMCI_LOG_ERROR(LGPFX"Could not map: BAR1\n");
                return (ENXIO);
        }

        sc->vmci_iot1 = rman_get_bustag(sc->vmci_res1);
        sc->vmci_ioh1 = rman_get_bushandle(sc->vmci_res1);

        return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_unmap_bars --
 *
 *     Unmaps the VMCI PCI I/O and MMIO BARs.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_unmap_bars(struct vmci_softc *sc)
{
        int rid;

        if (sc->vmci_res0 != NULL) {
                rid = PCIR_BAR(0);
                bus_release_resource(sc->vmci_dev, SYS_RES_IOPORT, rid,
                    sc->vmci_res0);
                sc->vmci_res0 = NULL;
        }

        if (sc->vmci_res1 != NULL) {
                rid = PCIR_BAR(1);
                bus_release_resource(sc->vmci_dev, SYS_RES_MEMORY, rid,
                    sc->vmci_res1);
                sc->vmci_res1 = NULL;
        }
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_config_capabilities --
 *
 *     Check the VMCI device capabilities and configure the device accordingly.
 *
 * Results:
 *     0 if success, ENODEV otherwise.
 *
 * Side effects:
 *     Device capabilities are enabled.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_config_capabilities(struct vmci_softc *sc)
{
        unsigned long bitmap_PPN;
        int error;

        /*
         * Verify that the VMCI device supports the capabilities that we
         * need. Datagrams are necessary and notifications will be used
         * if the device supports it.
         */
        sc->capabilities = bus_space_read_4(sc->vmci_iot0, sc->vmci_ioh0,
            VMCI_CAPS_ADDR);

        if ((sc->capabilities & VMCI_CAPS_DATAGRAM) == 0) {
                VMCI_LOG_ERROR(LGPFX"VMCI device does not support "
                    "datagrams.\n");
                return (ENODEV);
        }

        if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) {
                sc->capabilities = VMCI_CAPS_DATAGRAM;
                error = vmci_dma_malloc(PAGE_SIZE, 1,
                    &sc->vmci_notifications_bitmap);
                if (error)
                        VMCI_LOG_ERROR(LGPFX"Failed to alloc memory for "
                            "notification bitmap.\n");
                else {
                        memset(sc->vmci_notifications_bitmap.dma_vaddr, 0,
                            PAGE_SIZE);
                        sc->capabilities |= VMCI_CAPS_NOTIFICATIONS;
                }
        } else
                sc->capabilities = VMCI_CAPS_DATAGRAM;

        /* Let the host know which capabilities we intend to use. */
        bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
            VMCI_CAPS_ADDR, sc->capabilities);

        /*
         * Register notification bitmap with device if that capability is
         * used.
         */
        if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) {
                bitmap_PPN =
                    sc->vmci_notifications_bitmap.dma_paddr >> PAGE_SHIFT;
                vmci_register_notification_bitmap(bitmap_PPN);
        }

        /* Check host capabilities. */
        if (!vmci_check_host_capabilities())
                return (ENODEV);

        return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_dmamap_cb --
 *
 *     Callback to receive mapping information resulting from the load of a
 *     bus_dmamap_t via bus_dmamap_load()
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
        bus_addr_t *baddr = arg;

        if (error == 0)
                *baddr = segs->ds_addr;
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_dma_malloc_int --
 *
 *     Internal function that allocates DMA memory.
 *
 * Results:
 *     0 if success.
 *     ENOMEM if insufficient memory.
 *     EINPROGRESS if mapping is deferred.
 *     EINVAL if the request was invalid.
 *
 * Side effects:
 *     DMA memory is allocated.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_dma_malloc_int(struct vmci_softc *sc, bus_size_t size, bus_size_t align,
    struct vmci_dma_alloc *dma)
{
        int error;

        bzero(dma, sizeof(struct vmci_dma_alloc));

        error = bus_dma_tag_create(bus_get_dma_tag(vmci_sc->vmci_dev),
            align, 0,           /* alignment, bounds */
            BUS_SPACE_MAXADDR,  /* lowaddr */
            BUS_SPACE_MAXADDR,  /* highaddr */
            NULL, NULL,         /* filter, filterarg */
            size,               /* maxsize */
            1,                  /* nsegments */
            size,               /* maxsegsize */
            BUS_DMA_ALLOCNOW,   /* flags */
            NULL,               /* lockfunc */
            NULL,               /* lockfuncarg */
            &dma->dma_tag);
        if (error) {
                VMCI_LOG_ERROR(LGPFX"bus_dma_tag_create failed: %d\n", error);
                goto fail;
        }

        error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
            BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map);
        if (error) {
                VMCI_LOG_ERROR(LGPFX"bus_dmamem_alloc failed: %d\n", error);
                goto fail;
        }

        error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
            size, vmci_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
        if (error) {
                VMCI_LOG_ERROR(LGPFX"bus_dmamap_load failed: %d\n", error);
                goto fail;
        }

        dma->dma_size = size;

fail:
        if (error)
                vmci_dma_free(dma);

        return (error);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_dma_malloc --
 *
 *     This function is a wrapper around vmci_dma_malloc_int for callers
 *     outside of this module. Since we only support a single VMCI device, this
 *     wrapper provides access to the device softc structure.
 *
 * Results:
 *     0 if success.
 *     ENOMEM if insufficient memory.
 *     EINPROGRESS if mapping is deferred.
 *     EINVAL if the request was invalid.
 *
 * Side effects:
 *     DMA memory is allocated.
 *
 *------------------------------------------------------------------------------
 */

int
vmci_dma_malloc(bus_size_t size, bus_size_t align, struct vmci_dma_alloc *dma)
{

        return (vmci_dma_malloc_int(vmci_sc, size, align, dma));
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_dma_free_int --
 *
 *     Internal function that frees DMA memory.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     Frees DMA memory.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_dma_free_int(struct vmci_softc *sc, struct vmci_dma_alloc *dma)
{

        if (dma->dma_tag != NULL) {
                if (dma->dma_paddr != 0) {
                        bus_dmamap_sync(dma->dma_tag, dma->dma_map,
                            BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
                        bus_dmamap_unload(dma->dma_tag, dma->dma_map);
                }

                if (dma->dma_vaddr != NULL)
                        bus_dmamem_free(dma->dma_tag, dma->dma_vaddr,
                            dma->dma_map);

                bus_dma_tag_destroy(dma->dma_tag);
        }
        bzero(dma, sizeof(struct vmci_dma_alloc));
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_dma_free --
 *
 *     This function is a wrapper around vmci_dma_free_int for callers outside
 *     of this module. Since we only support a single VMCI device, this wrapper
 *     provides access to the device softc structure.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     Frees DMA memory.
 *
 *------------------------------------------------------------------------------
 */

void
vmci_dma_free(struct vmci_dma_alloc *dma)
{

        vmci_dma_free_int(vmci_sc, dma);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_config_interrupts --
 *
 *     Configures and enables interrupts. Try to configure MSI-X. If this fails,
 *     try to configure MSI. If even this fails, try legacy interrupts.
 *
 * Results:
 *     0 if success.
 *     ENOMEM if insufficient memory.
 *     ENODEV if the device doesn't support interrupts.
 *     ENXIO if the device configuration failed.
 *
 * Side effects:
 *     Interrupts get enabled if successful.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_config_interrupts(struct vmci_softc *sc)
{
        int error;

        data_buffer = malloc(data_buffer_size, M_DEVBUF, M_ZERO | M_NOWAIT);
        if (data_buffer == NULL)
                return (ENOMEM);

        sc->vmci_intr_type = VMCI_INTR_TYPE_MSIX;
        error = vmci_config_interrupt(sc);
        if (error) {
                sc->vmci_intr_type = VMCI_INTR_TYPE_MSI;
                error = vmci_config_interrupt(sc);
        }
        if (error) {
                sc->vmci_intr_type = VMCI_INTR_TYPE_INTX;
                error = vmci_config_interrupt(sc);
        }
        if (error)
                return (error);

        /* Enable specific interrupt bits. */
        if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS)
                bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
                    VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM | VMCI_IMR_NOTIFICATION);
        else
                bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
                    VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM);

        /* Enable interrupts. */
        bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
            VMCI_CONTROL_ADDR, VMCI_CONTROL_INT_ENABLE);

        return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_config_interrupt --
 *
 *     Check the number of interrupts supported, allocate resources and setup
 *     interrupts.
 *
 * Results:
 *     0 if success.
 *     ENOMEM if insufficient memory.
 *     ENODEV if the device doesn't support interrupts.
 *     ENXIO if the device configuration failed.
 *
 * Side effects:
 *     Resources get allocated and interrupts get setup (but not enabled) if
 *     successful.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_config_interrupt(struct vmci_softc *sc)
{
        int error;

        error = vmci_check_intr_cnt(sc);
        if (error)
                return (error);

        error = vmci_allocate_interrupt_resources(sc);
        if (error)
                return (error);

        error = vmci_setup_interrupts(sc);
        if (error)
                return (error);

        return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_check_intr_cnt --
 *
 *     Check the number of interrupts supported by the device and ask PCI bus
 *     to allocate appropriate number of interrupts.
 *
 * Results:
 *     0 if success.
 *     ENODEV if the device doesn't support any interrupts.
 *     ENXIO if the device configuration failed.
 *
 * Side effects:
 *     Resources get allocated on success.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_check_intr_cnt(struct vmci_softc *sc)
{

        if (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) {
                sc->vmci_num_intr = 1;
                return (0);
        }

        /*
         * Make sure that the device supports the required number of MSI/MSI-X
         * messages. We try for 2 MSI-X messages but 1 is good too. We need at
         * least 1 MSI message.
         */
        sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ?
            pci_msix_count(sc->vmci_dev) : pci_msi_count(sc->vmci_dev);

        if (!sc->vmci_num_intr) {
                VMCI_LOG_ERROR(LGPFX"Device does not support any interrupt"
                    " messages");
                return (ENODEV);
        }

        sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ?
            VMCI_MAX_INTRS : 1;
        if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) {
                if (pci_alloc_msix(sc->vmci_dev, &sc->vmci_num_intr))
                        return (ENXIO);
        } else if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSI) {
                if (pci_alloc_msi(sc->vmci_dev, &sc->vmci_num_intr))
                        return (ENXIO);
        }

        return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_allocate_interrupt_resources --
 *
 *     Allocate resources necessary for interrupts.
 *
 * Results:
 *     0 if success, ENXIO otherwise.
 *
 * Side effects:
 *     Resources get allocated on success.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_allocate_interrupt_resources(struct vmci_softc *sc)
{
        struct resource *irq;
        int flags, i, rid;

        flags = RF_ACTIVE;
        flags |= (sc->vmci_num_intr == 1) ? RF_SHAREABLE : 0;
        rid = (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) ? 0 : 1;

        for (i = 0; i < sc->vmci_num_intr; i++, rid++) {
                irq = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IRQ, &rid,
                    flags);
                if (irq == NULL)
                        return (ENXIO);
                sc->vmci_intrs[i].vmci_irq = irq;
                sc->vmci_intrs[i].vmci_rid = rid;
        }

        return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_setup_interrupts --
 *
 *     Sets up the interrupts.
 *
 * Results:
 *     0 if success, appropriate error code from bus_setup_intr otherwise.
 *
 * Side effects:
 *     Interrupt handler gets attached.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_setup_interrupts(struct vmci_softc *sc)
{
        struct vmci_interrupt *intr;
        int error, flags;

        flags = INTR_TYPE_NET | INTR_MPSAFE;
        if (sc->vmci_num_intr > 1)
                flags |= INTR_EXCL;

        intr = &sc->vmci_intrs[0];
        error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags, NULL,
            vmci_interrupt, NULL, &intr->vmci_handler);
        if (error)
                return (error);

        if (sc->vmci_num_intr == 2) {
                bus_describe_intr(sc->vmci_dev, intr->vmci_irq,
                    intr->vmci_handler, "dg");
                intr = &sc->vmci_intrs[1];
                error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags,
                    NULL, vmci_interrupt_bm, NULL, &intr->vmci_handler);
                if (error)
                        return (error);
                bus_describe_intr(sc->vmci_dev, intr->vmci_irq,
                    intr->vmci_handler, "bm");
        }

        return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_interrupt --
 *
 *     Interrupt handler for legacy or MSI interrupt, or for first MSI-X
 *     interrupt (vector VMCI_INTR_DATAGRAM).
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_interrupt(void *arg)
{

        if (vmci_sc->vmci_num_intr == 2)
                taskqueue_enqueue(taskqueue_swi,
                    &vmci_sc->vmci_interrupt_dq_task);
        else {
                unsigned int icr;

                icr = inl(vmci_sc->vmci_ioaddr + VMCI_ICR_ADDR);
                if (icr == 0 || icr == 0xffffffff)
                        return;
                if (icr & VMCI_ICR_DATAGRAM) {
                        taskqueue_enqueue(taskqueue_swi,
                            &vmci_sc->vmci_interrupt_dq_task);
                        icr &= ~VMCI_ICR_DATAGRAM;
                }
                if (icr & VMCI_ICR_NOTIFICATION) {
                        taskqueue_enqueue(taskqueue_swi,
                            &vmci_sc->vmci_interrupt_bm_task);
                        icr &= ~VMCI_ICR_NOTIFICATION;
                }
                if (icr != 0)
                        VMCI_LOG_INFO(LGPFX"Ignoring unknown interrupt "
                            "cause");
        }
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_interrupt_bm --
 *
 *     Interrupt handler for MSI-X interrupt vector VMCI_INTR_NOTIFICATION,
 *     which is for the notification bitmap. Will only get called if we are
 *     using MSI-X with exclusive vectors.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_interrupt_bm(void *arg)
{

        ASSERT(vmci_sc->vmci_num_intr == 2);
        taskqueue_enqueue(taskqueue_swi, &vmci_sc->vmci_interrupt_bm_task);
}

/*
 *------------------------------------------------------------------------------
 *
 * dispatch_datagrams --
 *
 *     Reads and dispatches incoming datagrams.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     Reads data from the device.
 *
 *------------------------------------------------------------------------------
 */

static void
dispatch_datagrams(void *context, int data)
{

        if (data_buffer == NULL)
                VMCI_LOG_INFO(LGPFX"dispatch_datagrams(): no buffer "
                    "present");

        vmci_read_datagrams_from_port((vmci_io_handle) 0,
            vmci_sc->vmci_ioaddr + VMCI_DATA_IN_ADDR,
            data_buffer, data_buffer_size);
}

/*
 *------------------------------------------------------------------------------
 *
 * process_bitmap --
 *
 *     Scans the notification bitmap for raised flags, clears them and handles
 *     the notifications.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static void
process_bitmap(void *context, int data)
{

        if (vmci_sc->vmci_notifications_bitmap.dma_vaddr == NULL)
                VMCI_LOG_INFO(LGPFX"process_bitmaps(): no bitmap present");

        vmci_scan_notification_bitmap(
            vmci_sc->vmci_notifications_bitmap.dma_vaddr);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_dismantle_interrupts --
 *
 *     Releases resources, detaches the interrupt handler and drains the task
 *     queue.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     No more interrupts.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_dismantle_interrupts(struct vmci_softc *sc)
{
        struct vmci_interrupt *intr;
        int i;

        for (i = 0; i < sc->vmci_num_intr; i++) {
                intr = &sc->vmci_intrs[i];
                if (intr->vmci_handler != NULL) {
                        bus_teardown_intr(sc->vmci_dev, intr->vmci_irq,
                            intr->vmci_handler);
                        intr->vmci_handler = NULL;
                }
                if (intr->vmci_irq != NULL) {
                        bus_release_resource(sc->vmci_dev, SYS_RES_IRQ,
                            intr->vmci_rid, intr->vmci_irq);
                        intr->vmci_irq = NULL;
                        intr->vmci_rid = -1;
                }
        }

        if ((sc->vmci_intr_type != VMCI_INTR_TYPE_INTX) &&
            (sc->vmci_num_intr))
                pci_release_msi(sc->vmci_dev);

        taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_dq_task);
        taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_bm_task);

        if (data_buffer != NULL)
                free(data_buffer, M_DEVBUF);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_delayed_work_fn_cb --
 *
 *     Callback function that executes the queued up delayed work functions.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_delayed_work_fn_cb(void *context, int data)
{
        vmci_list(vmci_delayed_work_info) temp_list;

        vmci_list_init(&temp_list);

        /*
         * Swap vmci_delayed_work_infos list with the empty temp_list while
         * holding a lock. vmci_delayed_work_infos would then be an empty list
         * and temp_list would contain the elements from the original
         * vmci_delayed_work_infos. Finally, iterate through temp_list
         * executing the delayed callbacks.
         */

        mtx_lock(&vmci_sc->vmci_delayed_work_lock);
        vmci_list_swap(&temp_list, &vmci_sc->vmci_delayed_work_infos,
            vmci_delayed_work_info, entry);
        mtx_unlock(&vmci_sc->vmci_delayed_work_lock);

        while (!vmci_list_empty(&temp_list)) {
                struct vmci_delayed_work_info *delayed_work_info =
                    vmci_list_first(&temp_list);

                delayed_work_info->work_fn(delayed_work_info->data);

                vmci_list_remove(delayed_work_info, entry);
                vmci_free_kernel_mem(delayed_work_info,
                    sizeof(*delayed_work_info));
        }
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_schedule_delayed_work_fn --
 *
 *     Schedule the specified callback.
 *
 * Results:
 *     0 if success, error code otherwise.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

int
vmci_schedule_delayed_work_fn(vmci_work_fn *work_fn, void *data)
{
        struct vmci_delayed_work_info *delayed_work_info;

        delayed_work_info = vmci_alloc_kernel_mem(sizeof(*delayed_work_info),
            VMCI_MEMORY_ATOMIC);

        if (!delayed_work_info)
                return (VMCI_ERROR_NO_MEM);

        delayed_work_info->work_fn = work_fn;
        delayed_work_info->data = data;
        mtx_lock(&vmci_sc->vmci_delayed_work_lock);
        vmci_list_insert(&vmci_sc->vmci_delayed_work_infos,
            delayed_work_info, entry);
        mtx_unlock(&vmci_sc->vmci_delayed_work_lock);

        taskqueue_enqueue(taskqueue_thread,
            &vmci_sc->vmci_delayed_work_task);

        return (VMCI_SUCCESS);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_send_datagram --
 *
 *     VM to hypervisor call mechanism.
 *
 * Results:
 *     The result of the hypercall.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

int
vmci_send_datagram(struct vmci_datagram *dg)
{
        int result;

        if (dg == NULL)
                return (VMCI_ERROR_INVALID_ARGS);

        /*
         * Need to acquire spinlock on the device because
         * the datagram data may be spread over multiple pages and the monitor
         * may interleave device user rpc calls from multiple VCPUs. Acquiring
         * the spinlock precludes that possibility. Disabling interrupts to
         * avoid incoming datagrams during a "rep out" and possibly landing up
         * in this function.
         */
        mtx_lock_spin(&vmci_sc->vmci_spinlock);

        /*
         * Send the datagram and retrieve the return value from the result
         * register.
         */
        __asm__ __volatile__(
            "cld\n\t"
            "rep outsb\n\t"
            : /* No output. */
            : "d"(vmci_sc->vmci_ioaddr + VMCI_DATA_OUT_ADDR),
            "c"(VMCI_DG_SIZE(dg)), "S"(dg)
            );

        /*
         * XXX: Should read result high port as well when updating handlers to
         * return 64bit.
         */

        result = bus_space_read_4(vmci_sc->vmci_iot0,
            vmci_sc->vmci_ioh0, VMCI_RESULT_LOW_ADDR);
        mtx_unlock_spin(&vmci_sc->vmci_spinlock);

        return (result);
}