/*      $NetBSD: xengnt.c,v 1.41 2023/02/25 00:35:52 riastradh Exp $      */

/*
 * Copyright (c) 2006 Manuel Bouyer.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: xengnt.c,v 1.41 2023/02/25 00:35:52 riastradh Exp $");

#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kmem.h>
#include <sys/queue.h>
#include <sys/extent.h>
#include <sys/kernel.h>
#include <sys/mutex.h>
#include <uvm/uvm.h>

#include <xen/hypervisor.h>
#include <xen/xen.h>
#include <xen/granttables.h>

#include "opt_xen.h"

/* #define XENDEBUG */
#ifdef XENDEBUG
#define DPRINTF(x) printf x
#else
#define DPRINTF(x)
#endif

/* External tools reserve first few grant table entries. */
#define NR_RESERVED_ENTRIES 8

/* current supported version */
int gnt_v = 0;
#define GNT_ISV1 (gnt_v == 1)
#define GNT_ISV2 (gnt_v == 2)
/* Current number of frames making up the grant table */
int gnt_nr_grant_frames;
/* Maximum number of frames that can make up the grant table */
int gnt_max_grant_frames;

/* table of free grant entries */
grant_ref_t *gnt_entries;
/* last free entry */
int last_gnt_entry;
/* empty entry in the list */
#define XENGNT_NO_ENTRY 0xffffffff

/* VM address of the grant table */
#define NR_GRANT_ENTRIES_PER_PAGE_V1 (PAGE_SIZE / sizeof(grant_entry_v1_t))
#define NR_GRANT_ENTRIES_PER_PAGE_V2 (PAGE_SIZE / sizeof(grant_entry_v2_t))
#define NR_GRANT_ENTRIES_PER_PAGE \
    ((gnt_v == 1) ? NR_GRANT_ENTRIES_PER_PAGE_V1 : NR_GRANT_ENTRIES_PER_PAGE_V2)
#define NR_GRANT_STATUS_PER_PAGE (PAGE_SIZE / sizeof(grant_status_t))

union {
        grant_entry_v1_t *gntt_v1;
        grant_entry_v2_t *gntt_v2;
        void *gntt;
} grant_table;

/* Number of grant status frames (v2 only)*/
int gnt_status_frames;

grant_status_t *grant_status;
kmutex_t grant_lock;

static grant_ref_t xengnt_get_entry(void);
static void xengnt_free_entry(grant_ref_t);
static int xengnt_more_entries(void);
static int xengnt_map_status(void);
static bool xengnt_finish_init(void);

void
xengnt_init(void)
{
        struct gnttab_query_size query;
        int rc;
        int nr_grant_entries;
        int i;

        /* first try to see which version we support */
        struct gnttab_set_version gntversion;
        gnt_v = gntversion.version = 2;
        rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gntversion, 1);
        if (rc < 0 || gntversion.version != 2) {
                aprint_debug("GNTTABOP_set_version 2 failed (%d), "
                    "fall back to version 1\n", rc);
                gnt_v = 1;
        }

        query.dom = DOMID_SELF;
        rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
        if ((rc < 0) || (query.status != GNTST_okay))
                gnt_max_grant_frames = 4; /* Legacy max number of frames */
        else
                gnt_max_grant_frames = query.max_nr_frames;

        /*
         * Always allocate max number of grant frames, never expand in runtime
         */
        gnt_nr_grant_frames = gnt_max_grant_frames;

        nr_grant_entries =
            gnt_max_grant_frames * NR_GRANT_ENTRIES_PER_PAGE;

        grant_table.gntt = (void *)uvm_km_alloc(kernel_map,
            gnt_max_grant_frames * PAGE_SIZE, 0, UVM_KMF_VAONLY);
        if (grant_table.gntt == NULL)
                panic("xengnt_init() table no VM space");

        gnt_entries = kmem_alloc((nr_grant_entries + 1) * sizeof(grant_ref_t),
            KM_SLEEP);
        for (i = 0; i <= nr_grant_entries; i++)
                gnt_entries[i] = XENGNT_NO_ENTRY;

        if (GNT_ISV2) {
                gnt_status_frames =
                    round_page(nr_grant_entries * sizeof(grant_status_t)) / PAGE_SIZE;
                grant_status = (void *)uvm_km_alloc(kernel_map,
                    gnt_status_frames * PAGE_SIZE, 0, UVM_KMF_VAONLY);
                if (grant_status == NULL)
                        panic("xengnt_init() status no VM space");
        }

        mutex_init(&grant_lock, MUTEX_DEFAULT, IPL_VM);

        xengnt_finish_init();
}

/*
 * Resume grant table state
 */
bool
xengnt_resume(void)
{
        int rc;

        struct gnttab_set_version gntversion;
        KASSERT(gnt_v == 1 || gnt_v == 2);
        gntversion.version = gnt_v;
        rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gntversion, 1);

        if (GNT_ISV2) {
                if (rc < 0 || gntversion.version != 2) {
                        panic("GNTTABOP_set_version 2 failed %d", rc);
                }
        } else {
                if (rc == 0 && gntversion.version != 1) {
                        panic("GNTTABOP_set_version 1 failed");
                }
        }

        return xengnt_finish_init();
}

static bool
xengnt_finish_init(void)
{
        int previous_nr_grant_frames = gnt_nr_grant_frames;

        last_gnt_entry = 0;
        gnt_nr_grant_frames = 0;

        mutex_enter(&grant_lock);
        while (gnt_nr_grant_frames < previous_nr_grant_frames) {
                if (xengnt_more_entries() != 0)
                        panic("xengnt_resume: can't restore grant frames");
        }
        if (GNT_ISV2)
                xengnt_map_status();
        mutex_exit(&grant_lock);
        return true;
}

/*
 * Suspend grant table state
 */
bool
xengnt_suspend(void) {

        int i;

        mutex_enter(&grant_lock);
        KASSERT(gnt_entries[last_gnt_entry] == XENGNT_NO_ENTRY);

        for (i = 0; i < last_gnt_entry; i++) {
                /* invalidate all grant entries (necessary for resume) */
                gnt_entries[i] = XENGNT_NO_ENTRY;
        }

        /* Remove virtual => machine mapping for grant table */
        pmap_kremove((vaddr_t)grant_table.gntt, gnt_nr_grant_frames * PAGE_SIZE);

        if (GNT_ISV2) {
                /* Remove virtual => machine mapping for status table */
                pmap_kremove((vaddr_t)grant_status, gnt_status_frames * PAGE_SIZE);
        }

        pmap_update(pmap_kernel());
        mutex_exit(&grant_lock);
        return true;
}

/*
 * Get status frames and enter them into the VA space.
 */
static int
xengnt_map_status(void)
{
        uint64_t *pages;
        size_t sz;
        KASSERT(mutex_owned(&grant_lock));
        KASSERT(GNT_ISV2);

        sz = gnt_status_frames * sizeof(*pages);
        pages = kmem_alloc(sz, KM_NOSLEEP);
        if (pages == NULL)
                return ENOMEM;

#ifdef XENPV
        gnttab_get_status_frames_t getstatus;
        int err;

        getstatus.dom = DOMID_SELF;
        getstatus.nr_frames = gnt_status_frames;
        set_xen_guest_handle(getstatus.frame_list, pages);

        /*
         * get the status frames, and return the list of their virtual
         * addresses in 'pages'
         */
        if ((err = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
            &getstatus, 1)) != 0)
                panic("%s: get_status_frames failed: %d", __func__, err);
        if (getstatus.status != GNTST_okay) {
                aprint_error("%s: get_status_frames returned %d\n",
                    __func__, getstatus.status);
                kmem_free(pages, sz);
                return ENOMEM;
        }
#else /* XENPV */
        for (int i = 0; i < gnt_status_frames; i++) {
                struct vm_page *pg;
                struct xen_add_to_physmap xmap;

                pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO);
                pages[i] = atop(uvm_vm_page_to_phys(pg));

                xmap.domid = DOMID_SELF;
                xmap.space = XENMAPSPACE_grant_table;
                xmap.idx = i | XENMAPIDX_grant_table_status;
                xmap.gpfn = pages[i];

                if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xmap) < 0)
                        panic("%s: Unable to add grant tables\n", __func__);
        }
#endif /* XENPV */
        /*
         * map between status_table addresses and the machine addresses of
         * the status table frames
         */
        for (int i = 0; i < gnt_status_frames; i++) {
                pmap_kenter_ma(((vaddr_t)grant_status) + i * PAGE_SIZE,
                    ((paddr_t)pages[i]) << PAGE_SHIFT,
                    VM_PROT_WRITE, 0);
        }
        pmap_update(pmap_kernel());

        kmem_free(pages, sz);
        return 0;
}

/*
 * Add another page to the grant table
 * Returns 0 on success, ENOMEM on failure
 */
static int
xengnt_more_entries(void)
{
        gnttab_setup_table_t setup;
        u_long *pages;
        int nframes_new = gnt_nr_grant_frames + 1;
        int i, start_gnt;
        size_t sz;
        KASSERT(mutex_owned(&grant_lock));

        if (gnt_nr_grant_frames == gnt_max_grant_frames)
                return ENOMEM;

        sz = nframes_new * sizeof(*pages);
        pages = kmem_alloc(sz, KM_NOSLEEP);
        if (pages == NULL)
                return ENOMEM;

        if (xen_feature(XENFEAT_auto_translated_physmap)) {
                /*
                 * Note: Although we allocate space for the entire
                 * table, in this mode we only update one entry at a
                 * time.
                 */
                struct vm_page *pg;
                struct xen_add_to_physmap xmap;

                pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO);
                pages[gnt_nr_grant_frames] = atop(uvm_vm_page_to_phys(pg));

                xmap.domid = DOMID_SELF;
                xmap.space = XENMAPSPACE_grant_table;
                xmap.idx = gnt_nr_grant_frames;
                xmap.gpfn = pages[gnt_nr_grant_frames];

                if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xmap) < 0)
                        panic("%s: Unable to add grant frames\n", __func__);

        } else {
                setup.dom = DOMID_SELF;
                setup.nr_frames = nframes_new;
                set_xen_guest_handle(setup.frame_list, pages);

                /*
                 * setup the grant table, made of nframes_new frames
                 * and return the list of their virtual addresses
                 * in 'pages'
                 */
                if (HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1) != 0)
                        panic("%s: setup table failed", __func__);
                if (setup.status != GNTST_okay) {
                        aprint_error("%s: setup table returned %d\n",
                            __func__, setup.status);
                        kmem_free(pages, sz);
                        return ENOMEM;
                }
        }

        DPRINTF(("xengnt_more_entries: map 0x%lx -> %p\n",
            pages[gnt_nr_grant_frames],
            (char *)grant_table + gnt_nr_grant_frames * PAGE_SIZE));

        /*
         * map between grant_table addresses and the machine addresses of
         * the grant table frames
         */
        pmap_kenter_ma(((vaddr_t)grant_table.gntt) + gnt_nr_grant_frames * PAGE_SIZE,
            ((paddr_t)pages[gnt_nr_grant_frames]) << PAGE_SHIFT,
            VM_PROT_WRITE, 0);
        pmap_update(pmap_kernel());

        /*
         * add the grant entries associated to the last grant table frame
         * and mark them as free. Prevent using the first grants (from 0 to 8)
         * since they are used by the tools.
         */
        start_gnt = (gnt_nr_grant_frames * NR_GRANT_ENTRIES_PER_PAGE) <
                    (NR_RESERVED_ENTRIES + 1) ?
                    (NR_RESERVED_ENTRIES + 1) :
                    (gnt_nr_grant_frames * NR_GRANT_ENTRIES_PER_PAGE);
        for (i = start_gnt;
            i < nframes_new * NR_GRANT_ENTRIES_PER_PAGE;
            i++) {
                KASSERT(gnt_entries[last_gnt_entry] == XENGNT_NO_ENTRY);
                gnt_entries[last_gnt_entry] = i;
                last_gnt_entry++;
        }
        gnt_nr_grant_frames = nframes_new;
        kmem_free(pages, sz);
        return 0;
}

/*
 * Returns a reference to the first free entry in grant table
 */
static grant_ref_t
xengnt_get_entry(void)
{
        grant_ref_t entry;
        static struct timeval xengnt_nonmemtime;
        static const struct timeval xengnt_nonmemintvl = {5,0};

        KASSERT(mutex_owned(&grant_lock));

        if (__predict_false(last_gnt_entry == 0)) {
                if (ratecheck(&xengnt_nonmemtime, &xengnt_nonmemintvl))
                        printf("xengnt_get_entry: out of grant "
                            "table entries\n");
                return XENGNT_NO_ENTRY;
        }
        KASSERT(gnt_entries[last_gnt_entry] == XENGNT_NO_ENTRY);
        last_gnt_entry--;
        entry = gnt_entries[last_gnt_entry];
        gnt_entries[last_gnt_entry] = XENGNT_NO_ENTRY;
        KASSERT(entry != XENGNT_NO_ENTRY && entry > NR_RESERVED_ENTRIES);
        KASSERT(last_gnt_entry >= 0);
        KASSERT(last_gnt_entry <= gnt_max_grant_frames * NR_GRANT_ENTRIES_PER_PAGE);
        return entry;
}

/*
 * Mark the grant table entry as free
 */
static void
xengnt_free_entry(grant_ref_t entry)
{
        mutex_enter(&grant_lock);
        KASSERT(entry > NR_RESERVED_ENTRIES);
        KASSERT(gnt_entries[last_gnt_entry] == XENGNT_NO_ENTRY);
        KASSERT(last_gnt_entry >= 0);
        KASSERT(last_gnt_entry <= gnt_max_grant_frames * NR_GRANT_ENTRIES_PER_PAGE);
        gnt_entries[last_gnt_entry] = entry;
        last_gnt_entry++;
        mutex_exit(&grant_lock);
}

int
xengnt_grant_access(domid_t dom, paddr_t ma, int ro, grant_ref_t *entryp)
{
        mutex_enter(&grant_lock);

        *entryp = xengnt_get_entry();
        if (__predict_false(*entryp == XENGNT_NO_ENTRY)) {
                mutex_exit(&grant_lock);
                return ENOMEM;
        }

        if (GNT_ISV2) {
                grant_table.gntt_v2[*entryp].full_page.frame = ma >> PAGE_SHIFT;
                grant_table.gntt_v2[*entryp].hdr.domid = dom;
                /*
                 * ensure that the above values reach global visibility
                 * before permitting frame's access (done when we set flags)
                 */
                xen_wmb();
                grant_table.gntt_v2[*entryp].hdr.flags =
                    GTF_permit_access | (ro ? GTF_readonly : 0);
        } else {
                grant_table.gntt_v1[*entryp].frame = ma >> PAGE_SHIFT;
                grant_table.gntt_v1[*entryp].domid = dom;
                /*
                * ensure that the above values reach global visibility
                * before permitting frame's access (done when we set flags)
                */
                xen_wmb();
                grant_table.gntt_v1[*entryp].flags =
                   GTF_permit_access | (ro ? GTF_readonly : 0);
        }
        mutex_exit(&grant_lock);
        return 0;
}

static inline uint16_t
xen_atomic_cmpxchg16(volatile uint16_t *ptr, uint16_t  val, uint16_t newval)
{
        unsigned long result;

        __asm volatile(__LOCK_PREFIX
           "cmpxchgw %w1,%2"
           :"=a" (result)
           :"q"(newval), "m" (*ptr), "0" (val)
           :"memory");

        return result;
}

void
xengnt_revoke_access(grant_ref_t entry)
{
        if (GNT_ISV2) {
                grant_table.gntt_v2[entry].hdr.flags = 0;
                xen_mb();       /* Concurrent access by hypervisor */

                if (__predict_false(
                    (grant_status[entry] & (GTF_reading|GTF_writing)) != 0)) {
                        printf("xengnt_revoke_access(%u): still in use\n",
                            entry);
                } else {

                        /*
                         * The read of grant_status needs to have acquire
                         * semantics.
                         * Reads already have that on x86, so need only protect
                         * against compiler reordering. May need full barrier
                         * on other architectures.
                         */
                        __insn_barrier();
                }
        } else {
                uint16_t flags, nflags;

                nflags = grant_table.gntt_v1[entry].flags;

                do {
                       if ((flags = nflags) & (GTF_reading|GTF_writing))
                               panic("xengnt_revoke_access: still in use");
                       nflags = xen_atomic_cmpxchg16(
                            &grant_table.gntt_v1[entry].flags, flags, 0);
                } while (nflags != flags);

        }
        xengnt_free_entry(entry);
}

int
xengnt_status(grant_ref_t entry)
{
        if (GNT_ISV2)
                return grant_status[entry] & (GTF_reading|GTF_writing);
        else
                return (grant_table.gntt_v1[entry].flags & (GTF_reading|GTF_writing));
}