/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2017 Joyent, Inc.
 */


#include <sys/errno.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/ddi.h>
#include <sys/stat.h>
#include <sys/sunddi.h>
#include <sys/file.h>
#include <sys/open.h>
#include <sys/modctl.h>
#include <sys/ddi_impldefs.h>
#include <sys/sysmacros.h>
#include <sys/ddidevmap.h>
#include <sys/policy.h>

#include <sys/vmsystm.h>
#include <vm/hat_i86.h>
#include <vm/hat_pte.h>
#include <vm/seg_kmem.h>
#include <vm/seg_mf.h>

#include <xen/io/blkif_impl.h>
#include <xen/io/blk_common.h>
#include <xen/io/xpvtap.h>


static int xpvtap_open(dev_t *devp, int flag, int otyp, cred_t *cred);
static int xpvtap_close(dev_t devp, int flag, int otyp, cred_t *cred);
static int xpvtap_ioctl(dev_t dev, int cmd, intptr_t arg, int mode,
    cred_t *cred, int *rval);
static int xpvtap_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off,
    size_t len, size_t *maplen, uint_t model);
static int xpvtap_segmap(dev_t dev, off_t off, struct as *asp, caddr_t *addrp,
    off_t len, unsigned int prot, unsigned int maxprot, unsigned int flags,
    cred_t *cred_p);
static int xpvtap_chpoll(dev_t dev, short events, int anyyet, short *reventsp,
    struct pollhead **phpp);

static  struct cb_ops xpvtap_cb_ops = {
        xpvtap_open,            /* cb_open */
        xpvtap_close,           /* cb_close */
        nodev,                  /* cb_strategy */
        nodev,                  /* cb_print */
        nodev,                  /* cb_dump */
        nodev,                  /* cb_read */
        nodev,                  /* cb_write */
        xpvtap_ioctl,           /* cb_ioctl */
        xpvtap_devmap,          /* cb_devmap */
        nodev,                  /* cb_mmap */
        xpvtap_segmap,          /* cb_segmap */
        xpvtap_chpoll,          /* cb_chpoll */
        ddi_prop_op,            /* cb_prop_op */
        NULL,                   /* cb_stream */
        D_NEW | D_MP | D_64BIT | D_DEVMAP,      /* cb_flag */
        CB_REV
};

static int xpvtap_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
    void **result);
static int xpvtap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
static int xpvtap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);

static struct dev_ops xpvtap_dev_ops = {
        DEVO_REV,               /* devo_rev */
        0,                      /* devo_refcnt */
        xpvtap_getinfo,         /* devo_getinfo */
        nulldev,                /* devo_identify */
        nulldev,                /* devo_probe */
        xpvtap_attach,          /* devo_attach */
        xpvtap_detach,          /* devo_detach */
        nodev,                  /* devo_reset */
        &xpvtap_cb_ops,         /* devo_cb_ops */
        NULL,                   /* devo_bus_ops */
        NULL                    /* power */
};


static struct modldrv xpvtap_modldrv = {
        &mod_driverops,         /* Type of module.  This one is a driver */
        "xpvtap driver",        /* Name of the module. */
        &xpvtap_dev_ops,        /* driver ops */
};

static struct modlinkage xpvtap_modlinkage = {
        MODREV_1,
        (void *) &xpvtap_modldrv,
        NULL
};


void *xpvtap_statep;


static xpvtap_state_t *xpvtap_drv_init(int instance);
static void xpvtap_drv_fini(xpvtap_state_t *state);
static uint_t xpvtap_intr(caddr_t arg);

typedef void (*xpvtap_rs_cleanup_t)(xpvtap_state_t *state, uint_t rs);
static void xpvtap_rs_init(uint_t min_val, uint_t max_val,
    xpvtap_rs_hdl_t *handle);
static void xpvtap_rs_fini(xpvtap_rs_hdl_t *handle);
static int xpvtap_rs_alloc(xpvtap_rs_hdl_t handle, uint_t *rs);
static void xpvtap_rs_free(xpvtap_rs_hdl_t handle, uint_t rs);
static void xpvtap_rs_flush(xpvtap_rs_hdl_t handle,
    xpvtap_rs_cleanup_t callback, void *arg);

static int xpvtap_segmf_register(xpvtap_state_t *state);
static void xpvtap_segmf_unregister(struct as *as, void *arg, uint_t event);

static int xpvtap_user_init(xpvtap_state_t *state);
static void xpvtap_user_fini(xpvtap_state_t *state);
static int xpvtap_user_ring_init(xpvtap_state_t *state);
static void xpvtap_user_ring_fini(xpvtap_state_t *state);
static int xpvtap_user_thread_init(xpvtap_state_t *state);
static void xpvtap_user_thread_fini(xpvtap_state_t *state);
static void xpvtap_user_thread_start(caddr_t arg);
static void xpvtap_user_thread_stop(xpvtap_state_t *state);
static void xpvtap_user_thread(void *arg);

static void xpvtap_user_app_stop(caddr_t arg);

static int xpvtap_user_request_map(xpvtap_state_t *state, blkif_request_t *req,
    uint_t *uid);
static int xpvtap_user_request_push(xpvtap_state_t *state,
    blkif_request_t *req, uint_t uid);
static int xpvtap_user_response_get(xpvtap_state_t *state,
    blkif_response_t *resp, uint_t *uid);
static void xpvtap_user_request_unmap(xpvtap_state_t *state, uint_t uid);


/*
 * _init()
 */
int
_init(void)
{
        int e;

        e = ddi_soft_state_init(&xpvtap_statep, sizeof (xpvtap_state_t), 1);
        if (e != 0) {
                return (e);
        }

        e = mod_install(&xpvtap_modlinkage);
        if (e != 0) {
                ddi_soft_state_fini(&xpvtap_statep);
                return (e);
        }

        return (0);
}


/*
 * _info()
 */
int
_info(struct modinfo *modinfop)
{
        return (mod_info(&xpvtap_modlinkage, modinfop));
}


/*
 * _fini()
 */
int
_fini(void)
{
        int e;

        e = mod_remove(&xpvtap_modlinkage);
        if (e != 0) {
                return (e);
        }

        ddi_soft_state_fini(&xpvtap_statep);

        return (0);
}


/*
 * xpvtap_attach()
 */
static int
xpvtap_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
        blk_ringinit_args_t args;
        xpvtap_state_t *state;
        int instance;
        int e;


        switch (cmd) {
        case DDI_ATTACH:
                break;

        case DDI_RESUME:
                return (DDI_SUCCESS);

        default:
                return (DDI_FAILURE);
        }

        /* initialize our state info */
        instance = ddi_get_instance(dip);
        state = xpvtap_drv_init(instance);
        if (state == NULL) {
                return (DDI_FAILURE);
        }
        state->bt_dip = dip;

        /* Initialize the guest ring */
        args.ar_dip = state->bt_dip;
        args.ar_intr = xpvtap_intr;
        args.ar_intr_arg = (caddr_t)state;
        args.ar_ringup = xpvtap_user_thread_start;
        args.ar_ringup_arg = (caddr_t)state;
        args.ar_ringdown = xpvtap_user_app_stop;
        args.ar_ringdown_arg = (caddr_t)state;
        e = blk_ring_init(&args, &state->bt_guest_ring);
        if (e != DDI_SUCCESS) {
                goto attachfail_ringinit;
        }

        /* create the minor node (for ioctl/mmap) */
        e = ddi_create_minor_node(dip, "xpvtap", S_IFCHR, instance,
            DDI_PSEUDO, 0);
        if (e != DDI_SUCCESS) {
                goto attachfail_minor_node;
        }

        /* Report that driver was loaded */
        ddi_report_dev(dip);

        return (DDI_SUCCESS);

attachfail_minor_node:
        blk_ring_fini(&state->bt_guest_ring);
attachfail_ringinit:
        xpvtap_drv_fini(state);
        return (DDI_FAILURE);
}


/*
 * xpvtap_detach()
 */
static int
xpvtap_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
        xpvtap_state_t *state;
        int instance;


        instance = ddi_get_instance(dip);
        state = ddi_get_soft_state(xpvtap_statep, instance);
        if (state == NULL) {
                return (DDI_FAILURE);
        }

        switch (cmd) {
        case DDI_DETACH:
                break;

        case DDI_SUSPEND:
        default:
                return (DDI_FAILURE);
        }

        xpvtap_user_thread_stop(state);
        blk_ring_fini(&state->bt_guest_ring);
        xpvtap_drv_fini(state);
        ddi_remove_minor_node(dip, NULL);

        return (DDI_SUCCESS);
}


/*
 * xpvtap_getinfo()
 */
/*ARGSUSED*/
static int
xpvtap_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
{
        xpvtap_state_t *state;
        int instance;
        dev_t dev;
        int e;


        dev = (dev_t)arg;
        instance = getminor(dev);

        switch (cmd) {
        case DDI_INFO_DEVT2DEVINFO:
                state = ddi_get_soft_state(xpvtap_statep, instance);
                if (state == NULL) {
                        return (DDI_FAILURE);
                }
                *result = (void *)state->bt_dip;
                e = DDI_SUCCESS;
                break;

        case DDI_INFO_DEVT2INSTANCE:
                *result = (void *)(uintptr_t)instance;
                e = DDI_SUCCESS;
                break;

        default:
                e = DDI_FAILURE;
                break;
        }

        return (e);
}


/*
 * xpvtap_open()
 */
/*ARGSUSED*/
static int
xpvtap_open(dev_t *devp, int flag, int otyp, cred_t *cred)
{
        xpvtap_state_t *state;
        int instance;


        if (secpolicy_xvm_control(cred)) {
                return (EPERM);
        }

        instance = getminor(*devp);
        state = ddi_get_soft_state(xpvtap_statep, instance);
        if (state == NULL) {
                return (ENXIO);
        }

        /* we should only be opened once */
        mutex_enter(&state->bt_open.bo_mutex);
        if (state->bt_open.bo_opened) {
                mutex_exit(&state->bt_open.bo_mutex);
                return (EBUSY);
        }
        state->bt_open.bo_opened = B_TRUE;
        mutex_exit(&state->bt_open.bo_mutex);

        /*
         * save the apps address space. need it for mapping/unmapping grefs
         * since will be doing it in a separate kernel thread.
         */
        state->bt_map.um_as = curproc->p_as;

        return (0);
}


/*
 * xpvtap_close()
 */
/*ARGSUSED*/
static int
xpvtap_close(dev_t devp, int flag, int otyp, cred_t *cred)
{
        xpvtap_state_t *state;
        int instance;


        instance = getminor(devp);
        state = ddi_get_soft_state(xpvtap_statep, instance);
        if (state == NULL) {
                return (ENXIO);
        }

        /*
         * wake thread so it can cleanup and wait for it to exit so we can
         * be sure it's not in the middle of processing a request/response.
         */
        mutex_enter(&state->bt_thread.ut_mutex);
        state->bt_thread.ut_wake = B_TRUE;
        state->bt_thread.ut_exit = B_TRUE;
        cv_signal(&state->bt_thread.ut_wake_cv);
        if (!state->bt_thread.ut_exit_done) {
                cv_wait(&state->bt_thread.ut_exit_done_cv,
                    &state->bt_thread.ut_mutex);
        }
        ASSERT(state->bt_thread.ut_exit_done);
        mutex_exit(&state->bt_thread.ut_mutex);

        state->bt_map.um_as = NULL;
        state->bt_map.um_guest_pages = NULL;

        /*
         * when the ring is brought down, a userland hotplug script is run
         * which tries to bring the userland app down. We'll wait for a bit
         * for the user app to exit. Notify the thread waiting that the app
         * has closed the driver.
         */
        mutex_enter(&state->bt_open.bo_mutex);
        ASSERT(state->bt_open.bo_opened);
        state->bt_open.bo_opened = B_FALSE;
        cv_signal(&state->bt_open.bo_exit_cv);
        mutex_exit(&state->bt_open.bo_mutex);

        return (0);
}


/*
 * xpvtap_ioctl()
 */
/*ARGSUSED*/
static int
xpvtap_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred,
    int *rval)
{
        xpvtap_state_t *state;
        int instance;


        if (secpolicy_xvm_control(cred)) {
                return (EPERM);
        }

        instance = getminor(dev);
        if (instance == -1) {
                return (EBADF);
        }

        state = ddi_get_soft_state(xpvtap_statep, instance);
        if (state == NULL) {
                return (EBADF);
        }

        switch (cmd) {
        case XPVTAP_IOCTL_RESP_PUSH:
                /*
                 * wake thread, thread handles guest requests and user app
                 * responses.
                 */
                mutex_enter(&state->bt_thread.ut_mutex);
                state->bt_thread.ut_wake = B_TRUE;
                cv_signal(&state->bt_thread.ut_wake_cv);
                mutex_exit(&state->bt_thread.ut_mutex);
                break;

        default:
                cmn_err(CE_WARN, "ioctl(%d) not supported\n", cmd);
                return (ENXIO);
        }

        return (0);
}


/*
 * xpvtap_segmap()
 */
/*ARGSUSED*/
static int
xpvtap_segmap(dev_t dev, off_t off, struct as *asp, caddr_t *addrp,
    off_t len, unsigned int prot, unsigned int maxprot, unsigned int flags,
    cred_t *cred_p)
{
        struct segmf_crargs a;
        xpvtap_state_t *state;
        int instance;
        int e;


        if (secpolicy_xvm_control(cred_p)) {
                return (EPERM);
        }

        instance = getminor(dev);
        state = ddi_get_soft_state(xpvtap_statep, instance);
        if (state == NULL) {
                return (EBADF);
        }

        /* the user app should be doing a MAP_SHARED mapping */
        if ((flags & MAP_TYPE) != MAP_SHARED) {
                return (EINVAL);
        }

        /*
         * if this is the user ring (offset = 0), devmap it (which ends up in
         * xpvtap_devmap). devmap will alloc and map the ring into the
         * app's VA space.
         */
        if (off == 0) {
                e = devmap_setup(dev, (offset_t)off, asp, addrp, (size_t)len,
                    prot, maxprot, flags, cred_p);
                return (e);
        }

        /* this should be the mmap for the gref pages (offset = PAGESIZE) */
        if (off != PAGESIZE) {
                return (EINVAL);
        }

        /* make sure we get the size we're expecting */
        if (len != XPVTAP_GREF_BUFSIZE) {
                return (EINVAL);
        }

        /*
         * reserve user app VA space for the gref pages and use segmf to
         * manage the backing store for the physical memory. segmf will
         * map in/out the grefs and fault them in/out.
         */
        ASSERT(asp == state->bt_map.um_as);
        as_rangelock(asp);
        if ((flags & MAP_FIXED) == 0) {
                map_addr(addrp, len, 0, 0, flags);
                if (*addrp == NULL) {
                        as_rangeunlock(asp);
                        return (ENOMEM);
                }
        } else {
                /* User specified address */
                (void) as_unmap(asp, *addrp, len);
        }
        a.dev = dev;
        a.prot = (uchar_t)prot;
        a.maxprot = (uchar_t)maxprot;
        e = as_map(asp, *addrp, len, segmf_create, &a);
        if (e != 0) {
                as_rangeunlock(asp);
                return (e);
        }
        as_rangeunlock(asp);

        /*
         * Stash user base address, and compute address where the request
         * array will end up.
         */
        state->bt_map.um_guest_pages = (caddr_t)*addrp;
        state->bt_map.um_guest_size = (size_t)len;

        /* register an as callback so we can cleanup when the app goes away */
        e = as_add_callback(asp, xpvtap_segmf_unregister, state,
            AS_UNMAP_EVENT, *addrp, len, KM_SLEEP);
        if (e != 0) {
                (void) as_unmap(asp, *addrp, len);
                return (EINVAL);
        }

        /* wake thread to see if there are requests already queued up */
        mutex_enter(&state->bt_thread.ut_mutex);
        state->bt_thread.ut_wake = B_TRUE;
        cv_signal(&state->bt_thread.ut_wake_cv);
        mutex_exit(&state->bt_thread.ut_mutex);

        return (0);
}


/*
 * xpvtap_devmap()
 */
/*ARGSUSED*/
static int
xpvtap_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len,
    size_t *maplen, uint_t model)
{
        xpvtap_user_ring_t *usring;
        xpvtap_state_t *state;
        int instance;
        int e;


        instance = getminor(dev);
        state = ddi_get_soft_state(xpvtap_statep, instance);
        if (state == NULL) {
                return (EBADF);
        }

        /* we should only get here if the offset was == 0 */
        if (off != 0) {
                return (EINVAL);
        }

        /* we should only be mapping in one page */
        if (len != PAGESIZE) {
                return (EINVAL);
        }

        /*
         * we already allocated the user ring during driver attach, all we
         * need to do is map it into the user app's VA.
         */
        usring = &state->bt_user_ring;
        e = devmap_umem_setup(dhp, state->bt_dip, NULL, usring->ur_cookie, 0,
            PAGESIZE, PROT_ALL, DEVMAP_DEFAULTS, NULL);
        if (e < 0) {
                return (e);
        }

        /* return the size to compete the devmap */
        *maplen = PAGESIZE;

        return (0);
}


/*
 * xpvtap_chpoll()
 */
static int
xpvtap_chpoll(dev_t dev, short events, int anyyet, short *reventsp,
    struct pollhead **phpp)
{
        xpvtap_user_ring_t *usring;
        xpvtap_state_t *state;
        int instance;


        instance = getminor(dev);
        if (instance == -1) {
                return (EBADF);
        }
        state = ddi_get_soft_state(xpvtap_statep, instance);
        if (state == NULL) {
                return (EBADF);
        }

        if (((events & (POLLIN | POLLRDNORM)) == 0) && !anyyet) {
                return (EINVAL);
        }

        /*
         * if we pushed requests on the user ring since the last poll, wakeup
         * the user app
         */
        *reventsp = 0;
        usring = &state->bt_user_ring;
        if (usring->ur_prod_polled != usring->ur_ring.req_prod_pvt) {

                /*
                 * XXX - is this faster here or xpvtap_user_request_push??
                 * prelim data says here.  Because less membars or because
                 * user thread will spin in poll requests before getting to
                 * responses?
                 */
                RING_PUSH_REQUESTS(&usring->ur_ring);

                usring->ur_prod_polled = usring->ur_ring.sring->req_prod;
                *reventsp =  POLLIN | POLLRDNORM;
        }

        if ((*reventsp == 0 && !anyyet) || (events & POLLET)) {
                *phpp = &state->bt_pollhead;
        }

        return (0);
}


/*
 * xpvtap_drv_init()
 */
static xpvtap_state_t *
xpvtap_drv_init(int instance)
{
        xpvtap_state_t *state;
        int e;


        e = ddi_soft_state_zalloc(xpvtap_statep, instance);
        if (e != DDI_SUCCESS) {
                return (NULL);
        }
        state = ddi_get_soft_state(xpvtap_statep, instance);
        if (state == NULL) {
                goto drvinitfail_get_soft_state;
        }

        state->bt_instance = instance;
        mutex_init(&state->bt_open.bo_mutex, NULL, MUTEX_DRIVER, NULL);
        cv_init(&state->bt_open.bo_exit_cv, NULL, CV_DRIVER, NULL);
        state->bt_open.bo_opened = B_FALSE;
        state->bt_map.um_registered = B_FALSE;

        /* initialize user ring, thread, mapping state */
        e = xpvtap_user_init(state);
        if (e != DDI_SUCCESS) {
                goto drvinitfail_userinit;
        }

        return (state);

drvinitfail_userinit:
        cv_destroy(&state->bt_open.bo_exit_cv);
        mutex_destroy(&state->bt_open.bo_mutex);
drvinitfail_get_soft_state:
        (void) ddi_soft_state_free(xpvtap_statep, instance);
        return (NULL);
}


/*
 * xpvtap_drv_fini()
 */
static void
xpvtap_drv_fini(xpvtap_state_t *state)
{
        xpvtap_user_fini(state);
        cv_destroy(&state->bt_open.bo_exit_cv);
        mutex_destroy(&state->bt_open.bo_mutex);
        (void) ddi_soft_state_free(xpvtap_statep, state->bt_instance);
}


/*
 * xpvtap_intr()
 *    this routine will be called when we have a request on the guest ring.
 */
static uint_t
xpvtap_intr(caddr_t arg)
{
        xpvtap_state_t *state;


        state = (xpvtap_state_t *)arg;

        /* wake thread, thread handles guest requests and user app responses */
        mutex_enter(&state->bt_thread.ut_mutex);
        state->bt_thread.ut_wake = B_TRUE;
        cv_signal(&state->bt_thread.ut_wake_cv);
        mutex_exit(&state->bt_thread.ut_mutex);

        return (DDI_INTR_CLAIMED);
}


/*
 * xpvtap_segmf_register()
 */
static int
xpvtap_segmf_register(xpvtap_state_t *state)
{
        struct seg *seg;
        uint64_t pte_ma;
        struct as *as;
        caddr_t uaddr;
        uint_t pgcnt;
        int i;


        as = state->bt_map.um_as;
        pgcnt = btopr(state->bt_map.um_guest_size);
        uaddr = state->bt_map.um_guest_pages;

        if (pgcnt == 0) {
                return (DDI_FAILURE);
        }

        AS_LOCK_ENTER(as, RW_READER);

        seg = as_findseg(as, state->bt_map.um_guest_pages, 0);
        if ((seg == NULL) || ((uaddr + state->bt_map.um_guest_size) >
            (seg->s_base + seg->s_size))) {
                AS_LOCK_EXIT(as);
                return (DDI_FAILURE);
        }

        /*
         * lock down the htables so the HAT can't steal them. Register the
         * PTE MA's for each gref page with seg_mf so we can do user space
         * gref mappings.
         */
        for (i = 0; i < pgcnt; i++) {
                hat_prepare_mapping(as->a_hat, uaddr, &pte_ma);
                hat_devload(as->a_hat, uaddr, PAGESIZE, (pfn_t)0,
                    PROT_READ | PROT_WRITE | PROT_USER | HAT_UNORDERED_OK,
                    HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
                hat_release_mapping(as->a_hat, uaddr);
                segmf_add_gref_pte(seg, uaddr, pte_ma);
                uaddr += PAGESIZE;
        }

        state->bt_map.um_registered = B_TRUE;

        AS_LOCK_EXIT(as);

        return (DDI_SUCCESS);
}


/*
 * xpvtap_segmf_unregister()
 *    as_callback routine
 */
/*ARGSUSED*/
static void
xpvtap_segmf_unregister(struct as *as, void *arg, uint_t event)
{
        xpvtap_state_t *state;
        caddr_t uaddr;
        uint_t pgcnt;
        int i;


        state = (xpvtap_state_t *)arg;
        if (!state->bt_map.um_registered) {
                /* remove the callback (which is this routine) */
                (void) as_delete_callback(as, arg);
                return;
        }

        pgcnt = btopr(state->bt_map.um_guest_size);
        uaddr = state->bt_map.um_guest_pages;

        /* unmap any outstanding req's grefs */
        xpvtap_rs_flush(state->bt_map.um_rs, xpvtap_user_request_unmap, state);

        /* Unlock the gref pages */
        for (i = 0; i < pgcnt; i++) {
                AS_LOCK_ENTER(as, RW_WRITER);
                hat_prepare_mapping(as->a_hat, uaddr, NULL);
                hat_unload(as->a_hat, uaddr, PAGESIZE, HAT_UNLOAD_UNLOCK);
                hat_release_mapping(as->a_hat, uaddr);
                AS_LOCK_EXIT(as);
                uaddr += PAGESIZE;
        }

        /* remove the callback (which is this routine) */
        (void) as_delete_callback(as, arg);

        state->bt_map.um_registered = B_FALSE;
}


/*
 * xpvtap_user_init()
 */
static int
xpvtap_user_init(xpvtap_state_t *state)
{
        xpvtap_user_map_t *map;
        int e;


        map = &state->bt_map;

        /* Setup the ring between the driver and user app */
        e = xpvtap_user_ring_init(state);
        if (e != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        /*
         * the user ring can handle BLKIF_RING_SIZE outstanding requests. This
         * is the same number of requests as the guest ring. Initialize the
         * state we use to track request IDs to the user app. These IDs will
         * also identify which group of gref pages correspond with the
         * request.
         */
        xpvtap_rs_init(0, (BLKIF_RING_SIZE - 1), &map->um_rs);

        /*
         * allocate the space to store a copy of each outstanding requests. We
         * will need to reference the ID and the number of segments when we
         * get the response from the user app.
         */
        map->um_outstanding_reqs = kmem_zalloc(
            sizeof (*map->um_outstanding_reqs) * BLKIF_RING_SIZE,
            KM_SLEEP);

        /*
         * initialize the thread we use to process guest requests and user
         * responses.
         */
        e = xpvtap_user_thread_init(state);
        if (e != DDI_SUCCESS) {
                goto userinitfail_user_thread_init;
        }

        return (DDI_SUCCESS);

userinitfail_user_thread_init:
        xpvtap_rs_fini(&map->um_rs);
        kmem_free(map->um_outstanding_reqs,
            sizeof (*map->um_outstanding_reqs) * BLKIF_RING_SIZE);
        xpvtap_user_ring_fini(state);
        return (DDI_FAILURE);
}


/*
 * xpvtap_user_ring_init()
 */
static int
xpvtap_user_ring_init(xpvtap_state_t *state)
{
        xpvtap_user_ring_t *usring;


        usring = &state->bt_user_ring;

        /* alocate and initialize the page for the shared user ring */
        usring->ur_sring = (blkif_sring_t *)ddi_umem_alloc(PAGESIZE,
            DDI_UMEM_SLEEP, &usring->ur_cookie);
        SHARED_RING_INIT(usring->ur_sring);
        FRONT_RING_INIT(&usring->ur_ring, usring->ur_sring, PAGESIZE);
        usring->ur_prod_polled = 0;

        return (DDI_SUCCESS);
}


/*
 * xpvtap_user_thread_init()
 */
static int
xpvtap_user_thread_init(xpvtap_state_t *state)
{
        xpvtap_user_thread_t *thread;
        char taskqname[32];


        thread = &state->bt_thread;

        mutex_init(&thread->ut_mutex, NULL, MUTEX_DRIVER, NULL);
        cv_init(&thread->ut_wake_cv, NULL, CV_DRIVER, NULL);
        cv_init(&thread->ut_exit_done_cv, NULL, CV_DRIVER, NULL);
        thread->ut_wake = B_FALSE;
        thread->ut_exit = B_FALSE;
        thread->ut_exit_done = B_TRUE;

        /* create but don't start the user thread */
        (void) sprintf(taskqname, "xvptap_%d", state->bt_instance);
        thread->ut_taskq = ddi_taskq_create(state->bt_dip, taskqname, 1,
            TASKQ_DEFAULTPRI, 0);
        if (thread->ut_taskq == NULL) {
                goto userinitthrfail_taskq_create;
        }

        return (DDI_SUCCESS);

userinitthrfail_taskq_create:
        cv_destroy(&thread->ut_exit_done_cv);
        cv_destroy(&thread->ut_wake_cv);
        mutex_destroy(&thread->ut_mutex);

        return (DDI_FAILURE);
}


/*
 * xpvtap_user_thread_start()
 */
static void
xpvtap_user_thread_start(caddr_t arg)
{
        xpvtap_user_thread_t *thread;
        xpvtap_state_t *state;
        int e;


        state = (xpvtap_state_t *)arg;
        thread = &state->bt_thread;

        /* start the user thread */
        thread->ut_exit_done = B_FALSE;
        e = ddi_taskq_dispatch(thread->ut_taskq, xpvtap_user_thread, state,
            DDI_SLEEP);
        if (e != DDI_SUCCESS) {
                thread->ut_exit_done = B_TRUE;
                cmn_err(CE_WARN, "Unable to start user thread\n");
        }
}


/*
 * xpvtap_user_thread_stop()
 */
static void
xpvtap_user_thread_stop(xpvtap_state_t *state)
{
        /* wake thread so it can exit */
        mutex_enter(&state->bt_thread.ut_mutex);
        state->bt_thread.ut_wake = B_TRUE;
        state->bt_thread.ut_exit = B_TRUE;
        cv_signal(&state->bt_thread.ut_wake_cv);
        if (!state->bt_thread.ut_exit_done) {
                cv_wait(&state->bt_thread.ut_exit_done_cv,
                    &state->bt_thread.ut_mutex);
        }
        mutex_exit(&state->bt_thread.ut_mutex);
        ASSERT(state->bt_thread.ut_exit_done);
}


/*
 * xpvtap_user_fini()
 */
static void
xpvtap_user_fini(xpvtap_state_t *state)
{
        xpvtap_user_map_t *map;


        map = &state->bt_map;

        xpvtap_user_thread_fini(state);
        xpvtap_rs_fini(&map->um_rs);
        kmem_free(map->um_outstanding_reqs,
            sizeof (*map->um_outstanding_reqs) * BLKIF_RING_SIZE);
        xpvtap_user_ring_fini(state);
}


/*
 * xpvtap_user_ring_fini()
 */
static void
xpvtap_user_ring_fini(xpvtap_state_t *state)
{
        ddi_umem_free(state->bt_user_ring.ur_cookie);
}


/*
 * xpvtap_user_thread_fini()
 */
static void
xpvtap_user_thread_fini(xpvtap_state_t *state)
{
        ddi_taskq_destroy(state->bt_thread.ut_taskq);
        cv_destroy(&state->bt_thread.ut_exit_done_cv);
        cv_destroy(&state->bt_thread.ut_wake_cv);
        mutex_destroy(&state->bt_thread.ut_mutex);
}


/*
 * xpvtap_user_thread()
 */
static void
xpvtap_user_thread(void *arg)
{
        xpvtap_user_thread_t *thread;
        blkif_response_t resp;
        xpvtap_state_t *state;
        blkif_request_t req;
        boolean_t b;
        uint_t uid;
        int e;


        state = (xpvtap_state_t *)arg;
        thread = &state->bt_thread;

xpvtap_thread_start:
        /* See if we are supposed to exit */
        mutex_enter(&thread->ut_mutex);
        if (thread->ut_exit) {
                thread->ut_exit_done = B_TRUE;
                cv_signal(&state->bt_thread.ut_exit_done_cv);
                mutex_exit(&thread->ut_mutex);
                return;
        }

        /*
         * if we aren't supposed to be awake, wait until someone wakes us.
         * when we wake up, check for a kill or someone telling us to exit.
         */
        if (!thread->ut_wake) {
                e = cv_wait_sig(&thread->ut_wake_cv, &thread->ut_mutex);
                if ((e == 0) || (thread->ut_exit)) {
                        thread->ut_exit = B_TRUE;
                        mutex_exit(&thread->ut_mutex);
                        goto xpvtap_thread_start;
                }
        }

        /* if someone didn't wake us, go back to the start of the thread */
        if (!thread->ut_wake) {
                mutex_exit(&thread->ut_mutex);
                goto xpvtap_thread_start;
        }

        /* we are awake */
        thread->ut_wake = B_FALSE;
        mutex_exit(&thread->ut_mutex);

        /* process requests from the guest */
        do {
                /*
                 * check for requests from the guest. if we don't have any,
                 * break out of the loop.
                 */
                e = blk_ring_request_get(state->bt_guest_ring, &req);
                if (e == B_FALSE) {
                        break;
                }

                /* we got a request, map the grefs into the user app's VA */
                e = xpvtap_user_request_map(state, &req, &uid);
                if (e != DDI_SUCCESS) {
                        /*
                         * If we couldn't map the request (e.g. user app hasn't
                         * opened the device yet), requeue it and try again
                         * later
                         */
                        blk_ring_request_requeue(state->bt_guest_ring);
                        break;
                }

                /* push the request to the user app */
                e = xpvtap_user_request_push(state, &req, uid);
                if (e != DDI_SUCCESS) {
                        resp.id = req.id;
                        resp.operation = req.operation;
                        resp.status = BLKIF_RSP_ERROR;
                        blk_ring_response_put(state->bt_guest_ring, &resp);
                }
        } while (!thread->ut_exit);

        /* process reponses from the user app */
        do {
                /*
                 * check for responses from the user app. if we don't have any,
                 * break out of the loop.
                 */
                b = xpvtap_user_response_get(state, &resp, &uid);
                if (b != B_TRUE) {
                        break;
                }

                /*
                 * if we got a response, unmap the grefs from the matching
                 * request.
                 */
                xpvtap_user_request_unmap(state, uid);

                /* push the response to the guest */
                blk_ring_response_put(state->bt_guest_ring, &resp);
        } while (!thread->ut_exit);

        goto xpvtap_thread_start;
}


/*
 * xpvtap_user_request_map()
 */
static int
xpvtap_user_request_map(xpvtap_state_t *state, blkif_request_t *req,
    uint_t *uid)
{
        grant_ref_t gref[BLKIF_MAX_SEGMENTS_PER_REQUEST];
        struct seg *seg;
        struct as *as;
        domid_t domid;
        caddr_t uaddr;
        uint_t flags;
        int i;
        int e;


        domid = xvdi_get_oeid(state->bt_dip);

        as = state->bt_map.um_as;
        if ((as == NULL) || (state->bt_map.um_guest_pages == NULL)) {
                return (DDI_FAILURE);
        }

        /* has to happen after segmap returns */
        if (!state->bt_map.um_registered) {
                /* register the pte's with segmf */
                e = xpvtap_segmf_register(state);
                if (e != DDI_SUCCESS) {
                        return (DDI_FAILURE);
                }
        }

        /* alloc an ID for the user ring */
        e = xpvtap_rs_alloc(state->bt_map.um_rs, uid);
        if (e != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        /* if we don't have any segments to map, we're done */
        if ((req->operation == BLKIF_OP_WRITE_BARRIER) ||
            (req->operation == BLKIF_OP_FLUSH_DISKCACHE) ||
            (req->nr_segments == 0)) {
                return (DDI_SUCCESS);
        }

        /* get the apps gref address */
        uaddr = XPVTAP_GREF_REQADDR(state->bt_map.um_guest_pages, *uid);

        AS_LOCK_ENTER(as, RW_READER);
        seg = as_findseg(as, state->bt_map.um_guest_pages, 0);
        if ((seg == NULL) || ((uaddr + mmu_ptob(req->nr_segments)) >
            (seg->s_base + seg->s_size))) {
                AS_LOCK_EXIT(as);
                return (DDI_FAILURE);
        }

        /* if we are reading from disk, we are writing into memory */
        flags = 0;
        if (req->operation == BLKIF_OP_READ) {
                flags |= SEGMF_GREF_WR;
        }

        /* Load the grefs into seg_mf */
        for (i = 0; i < req->nr_segments; i++) {
                gref[i] = req->seg[i].gref;
        }
        (void) segmf_add_grefs(seg, uaddr, flags, gref, req->nr_segments,
            domid);

        AS_LOCK_EXIT(as);

        return (DDI_SUCCESS);
}


/*
 * xpvtap_user_request_push()
 */
static int
xpvtap_user_request_push(xpvtap_state_t *state, blkif_request_t *req,
    uint_t uid)
{
        blkif_request_t *outstanding_req;
        blkif_front_ring_t *uring;
        blkif_request_t *target;
        xpvtap_user_map_t *map;


        uring = &state->bt_user_ring.ur_ring;
        map = &state->bt_map;

        target = RING_GET_REQUEST(uring, uring->req_prod_pvt);

        /*
         * Save request from the frontend. used for ID mapping and unmap
         * on response/cleanup
         */
        outstanding_req = &map->um_outstanding_reqs[uid];
        bcopy(req, outstanding_req, sizeof (*outstanding_req));

        /* put the request on the user ring */
        bcopy(req, target, sizeof (*req));
        target->id = (uint64_t)uid;
        uring->req_prod_pvt++;

        pollwakeup(&state->bt_pollhead, POLLIN | POLLRDNORM);

        return (DDI_SUCCESS);
}


static void
xpvtap_user_request_unmap(xpvtap_state_t *state, uint_t uid)
{
        blkif_request_t *req;
        struct seg *seg;
        struct as *as;
        caddr_t uaddr;
        int e;


        as = state->bt_map.um_as;
        if (as == NULL) {
                return;
        }

        /* get a copy of the original request */
        req = &state->bt_map.um_outstanding_reqs[uid];

        /* unmap the grefs for this request */
        if ((req->operation != BLKIF_OP_WRITE_BARRIER) &&
            (req->operation != BLKIF_OP_FLUSH_DISKCACHE) &&
            (req->nr_segments != 0)) {
                uaddr = XPVTAP_GREF_REQADDR(state->bt_map.um_guest_pages, uid);
                AS_LOCK_ENTER(as, RW_READER);
                seg = as_findseg(as, state->bt_map.um_guest_pages, 0);
                if ((seg == NULL) || ((uaddr + mmu_ptob(req->nr_segments)) >
                    (seg->s_base + seg->s_size))) {
                        AS_LOCK_EXIT(as);
                        xpvtap_rs_free(state->bt_map.um_rs, uid);
                        return;
                }

                e = segmf_release_grefs(seg, uaddr, req->nr_segments);
                if (e != 0) {
                        cmn_err(CE_WARN, "unable to release grefs");
                }

                AS_LOCK_EXIT(as);
        }

        /* free up the user ring id */
        xpvtap_rs_free(state->bt_map.um_rs, uid);
}


static int
xpvtap_user_response_get(xpvtap_state_t *state, blkif_response_t *resp,
    uint_t *uid)
{
        blkif_front_ring_t *uring;
        blkif_response_t *target;


        uring = &state->bt_user_ring.ur_ring;

        if (!RING_HAS_UNCONSUMED_RESPONSES(uring)) {
                return (B_FALSE);
        }

        target = NULL;
        target = RING_GET_RESPONSE(uring, uring->rsp_cons);
        if (target == NULL) {
                return (B_FALSE);
        }

        /* copy out the user app response */
        bcopy(target, resp, sizeof (*resp));
        uring->rsp_cons++;

        /* restore the quests id from the original request */
        *uid = (uint_t)resp->id;
        resp->id = state->bt_map.um_outstanding_reqs[*uid].id;

        return (B_TRUE);
}


/*
 * xpvtap_user_app_stop()
 */
static void xpvtap_user_app_stop(caddr_t arg)
{
        xpvtap_state_t *state;
        clock_t rc;

        state = (xpvtap_state_t *)arg;

        /*
         * Give the app 10 secs to exit. If it doesn't exit, it's not a serious
         * problem, we just won't auto-detach the driver.
         */
        mutex_enter(&state->bt_open.bo_mutex);
        if (state->bt_open.bo_opened) {
                rc = cv_reltimedwait(&state->bt_open.bo_exit_cv,
                    &state->bt_open.bo_mutex, drv_usectohz(10000000),
                    TR_CLOCK_TICK);
                if (rc <= 0) {
                        cmn_err(CE_NOTE, "!user process still has driver open, "
                            "deferring detach\n");
                }
        }
        mutex_exit(&state->bt_open.bo_mutex);
}


/*
 * xpvtap_rs_init()
 *    Initialize the resource structure. init() returns a handle to be used
 *    for the rest of the resource functions. This code is written assuming
 *    that min_val will be close to 0. Therefore, we will allocate the free
 *    buffer only taking max_val into account.
 */
static void
xpvtap_rs_init(uint_t min_val, uint_t max_val, xpvtap_rs_hdl_t *handle)
{
        xpvtap_rs_t *rstruct;
        uint_t array_size;
        uint_t index;


        ASSERT(handle != NULL);
        ASSERT(min_val < max_val);

        /* alloc space for resource structure */
        rstruct = kmem_alloc(sizeof (xpvtap_rs_t), KM_SLEEP);

        /*
         * Test to see if the max value is 64-bit aligned. If so, we don't need
         * to allocate an extra 64-bit word. alloc space for free buffer
         * (8 bytes per uint64_t).
         */
        if ((max_val & 0x3F) == 0) {
                rstruct->rs_free_size = (max_val >> 6) * 8;
        } else {
                rstruct->rs_free_size = ((max_val >> 6) + 1) * 8;
        }
        rstruct->rs_free = kmem_alloc(rstruct->rs_free_size, KM_SLEEP);

        /* Initialize resource structure */
        rstruct->rs_min = min_val;
        rstruct->rs_last = min_val;
        rstruct->rs_max = max_val;
        mutex_init(&rstruct->rs_mutex, NULL, MUTEX_DRIVER, NULL);
        rstruct->rs_flushing = B_FALSE;

        /* Mark all resources as free */
        array_size = rstruct->rs_free_size >> 3;
        for (index = 0; index < array_size; index++) {
                rstruct->rs_free[index] = (uint64_t)0xFFFFFFFFFFFFFFFF;
        }

        /* setup handle which is returned from this function */
        *handle = rstruct;
}


/*
 * xpvtap_rs_fini()
 *    Frees up the space allocated in init().  Notice that a pointer to the
 *    handle is used for the parameter.  fini() will set the handle to NULL
 *    before returning.
 */
static void
xpvtap_rs_fini(xpvtap_rs_hdl_t *handle)
{
        xpvtap_rs_t *rstruct;


        ASSERT(handle != NULL);

        rstruct = (xpvtap_rs_t *)*handle;

        mutex_destroy(&rstruct->rs_mutex);
        kmem_free(rstruct->rs_free, rstruct->rs_free_size);
        kmem_free(rstruct, sizeof (xpvtap_rs_t));

        /* set handle to null.  This helps catch bugs. */
        *handle = NULL;
}


/*
 * xpvtap_rs_alloc()
 *    alloc a resource. If alloc fails, we are out of resources.
 */
static int
xpvtap_rs_alloc(xpvtap_rs_hdl_t handle, uint_t *resource)
{
        xpvtap_rs_t *rstruct;
        uint_t array_idx;
        uint64_t free;
        uint_t index;
        uint_t last;
        uint_t min;
        uint_t max;


        ASSERT(handle != NULL);
        ASSERT(resource != NULL);

        rstruct = (xpvtap_rs_t *)handle;

        mutex_enter(&rstruct->rs_mutex);
        min = rstruct->rs_min;
        max = rstruct->rs_max;

        /*
         * Find a free resource. This will return out of the loop once it finds
         * a free resource. There are a total of 'max'-'min'+1 resources.
         * Performs a round robin allocation.
         */
        for (index = min; index <= max; index++) {

                array_idx = rstruct->rs_last >> 6;
                free = rstruct->rs_free[array_idx];
                last = rstruct->rs_last & 0x3F;

                /* if the next resource to check is free */
                if ((free & ((uint64_t)1 << last)) != 0) {
                        /* we are using this resource */
                        *resource = rstruct->rs_last;

                        /* take it out of the free list */
                        rstruct->rs_free[array_idx] &= ~((uint64_t)1 << last);

                        /*
                         * increment the last count so we start checking the
                         * next resource on the next alloc().  Note the rollover
                         * at 'max'+1.
                         */
                        rstruct->rs_last++;
                        if (rstruct->rs_last > max) {
                                rstruct->rs_last = rstruct->rs_min;
                        }

                        /* unlock the resource structure */
                        mutex_exit(&rstruct->rs_mutex);

                        return (DDI_SUCCESS);
                }

                /*
                 * This resource is not free, lets go to the next one. Note the
                 * rollover at 'max'.
                 */
                rstruct->rs_last++;
                if (rstruct->rs_last > max) {
                        rstruct->rs_last = rstruct->rs_min;
                }
        }

        mutex_exit(&rstruct->rs_mutex);

        return (DDI_FAILURE);
}


/*
 * xpvtap_rs_free()
 *    Free the previously alloc'd resource.  Once a resource has been free'd,
 *    it can be used again when alloc is called.
 */
static void
xpvtap_rs_free(xpvtap_rs_hdl_t handle, uint_t resource)
{
        xpvtap_rs_t *rstruct;
        uint_t array_idx;
        uint_t offset;


        ASSERT(handle != NULL);

        rstruct = (xpvtap_rs_t *)handle;
        ASSERT(resource >= rstruct->rs_min);
        ASSERT(resource <= rstruct->rs_max);

        if (!rstruct->rs_flushing) {
                mutex_enter(&rstruct->rs_mutex);
        }

        /* Put the resource back in the free list */
        array_idx = resource >> 6;
        offset = resource & 0x3F;
        rstruct->rs_free[array_idx] |= ((uint64_t)1 << offset);

        if (!rstruct->rs_flushing) {
                mutex_exit(&rstruct->rs_mutex);
        }
}


/*
 * xpvtap_rs_flush()
 */
static void
xpvtap_rs_flush(xpvtap_rs_hdl_t handle, xpvtap_rs_cleanup_t callback,
    void *arg)
{
        xpvtap_rs_t *rstruct;
        uint_t array_idx;
        uint64_t free;
        uint_t index;
        uint_t last;
        uint_t min;
        uint_t max;


        ASSERT(handle != NULL);

        rstruct = (xpvtap_rs_t *)handle;

        mutex_enter(&rstruct->rs_mutex);
        min = rstruct->rs_min;
        max = rstruct->rs_max;

        rstruct->rs_flushing = B_TRUE;

        /*
         * for all resources not free, call the callback routine to clean it
         * up.
         */
        for (index = min; index <= max; index++) {

                array_idx = rstruct->rs_last >> 6;
                free = rstruct->rs_free[array_idx];
                last = rstruct->rs_last & 0x3F;

                /* if the next resource to check is not free */
                if ((free & ((uint64_t)1 << last)) == 0) {
                        /* call the callback to cleanup */
                        (*callback)(arg, rstruct->rs_last);

                        /* put it back in the free list */
                        rstruct->rs_free[array_idx] |= ((uint64_t)1 << last);
                }

                /* go to the next one. Note the rollover at 'max' */
                rstruct->rs_last++;
                if (rstruct->rs_last > max) {
                        rstruct->rs_last = rstruct->rs_min;
                }
        }

        mutex_exit(&rstruct->rs_mutex);
}