/*      $NetBSD: iommu.c,v 1.119 2023/12/20 05:33:58 thorpej Exp $      */

/*
 * Copyright (c) 1999, 2000 Matthew R. Green
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * Copyright (c) 2001, 2002 Eduardo Horvath
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * UltraSPARC IOMMU support; used by both the sbus and pci code.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: iommu.c,v 1.119 2023/12/20 05:33:58 thorpej Exp $");

#include "opt_ddb.h"

#include <sys/param.h>
#include <sys/extent.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/proc.h>

#include <uvm/uvm.h>

#include <sys/bus.h>
#include <sparc64/dev/iommureg.h>
#include <sparc64/dev/iommuvar.h>

#include <machine/autoconf.h>
#include <machine/cpu.h>
#include <machine/hypervisor.h>

#ifdef DEBUG
#define IDB_BUSDMA      0x1
#define IDB_IOMMU       0x2
#define IDB_INFO        0x4
#define IDB_SYNC        0x8
int iommudebug = 0x0;
#define DPRINTF(l, s)   do { if (iommudebug & l) printf s; } while (0)
#define IOTTE_DEBUG(n)  (n)
#else
#define DPRINTF(l, s)
#define IOTTE_DEBUG(n)  0
#endif

#define iommu_strbuf_flush(i, v) do {                                   \
        if ((i)->sb_flush)                                              \
                bus_space_write_8((i)->sb_is->is_bustag, (i)->sb_sb,    \
                        STRBUFREG(strbuf_pgflush), (v));                \
        } while (0)

static  int iommu_strbuf_flush_done(struct strbuf_ctl *);
static  void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
                bus_size_t, int);
static void iommu_enter_sun4u(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags);
static void iommu_enter_sun4v(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags);
static void iommu_remove_sun4u(struct iommu_state *is, vaddr_t va, size_t len);
static void iommu_remove_sun4v(struct iommu_state *is, vaddr_t va, size_t len);

/*
 * initialise the UltraSPARC IOMMU (SBUS or PCI):
 *      - allocate and setup the iotsb.
 *      - enable the IOMMU
 *      - initialise the streaming buffers (if they exist)
 *      - create a private DVMA map.
 */
void
iommu_init(char *name, struct iommu_state *is, int tsbsize, uint32_t iovabase)
{
        psize_t size;
        vaddr_t va;
        paddr_t pa;
        struct vm_page *pg;
        struct pglist pglist;

        DPRINTF(IDB_INFO, ("iommu_init: tsbsize %x iovabase %x\n", tsbsize, iovabase));
        
        /*
         * Setup the iommu.
         *
         * The sun4u iommu is part of the SBUS or PCI controller so we will
         * deal with it here..
         *
         * For sysio and psycho/psycho+ the IOMMU address space always ends at
         * 0xffffe000, but the starting address depends on the size of the
         * map.  The map size is 1024 * 2 ^ is->is_tsbsize entries, where each
         * entry is 8 bytes.  The start of the map can be calculated by
         * (0xffffe000 << (8 + is->is_tsbsize)).
         *
         * But sabre and hummingbird use a different scheme that seems to
         * be hard-wired, so we read the start and size from the PROM and
         * just use those values.
         */
        if (strncmp(name, "pyro", 4) == 0) {
                is->is_cr = IOMMUREG_READ(is, iommu_cr);
                is->is_cr &= ~IOMMUCR_FIRE_BE;
                is->is_cr |= (IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN |
                    IOMMUCR_FIRE_TE);
        } else 
                is->is_cr = IOMMUCR_EN;
        is->is_tsbsize = tsbsize;
        if (iovabase == -1) {
                is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
                is->is_dvmaend = IOTSB_VEND - 1;
        } else {
                is->is_dvmabase = iovabase;
                is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1;
        }

        /*
         * Allocate memory for I/O pagetables.  They need to be physically
         * contiguous.
         */

        size = PAGE_SIZE << is->is_tsbsize;
        if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1,
                (paddr_t)PAGE_SIZE, (paddr_t)0, &pglist, 1, 0) != 0)
                panic("iommu_init: no memory");

        va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY);
        if (va == 0)
                panic("iommu_init: no memory");
        is->is_tsb = (int64_t *)va;

        is->is_ptsb = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist));

        /* Map the pages */
        TAILQ_FOREACH(pg, &pglist, pageq.queue) {
                pa = VM_PAGE_TO_PHYS(pg);
                pmap_kenter_pa(va, pa | PMAP_NVC,
                    VM_PROT_READ | VM_PROT_WRITE, 0);
                va += PAGE_SIZE;
        }
        pmap_update(pmap_kernel());
        memset(is->is_tsb, 0, size);

#ifdef DEBUG
        if (iommudebug & IDB_INFO)
        {
                /* Probe the iommu */
                if (!CPU_ISSUN4V) {
                        printf("iommu cr=%llx tsb=%llx\n",
                            (unsigned long long)bus_space_read_8(is->is_bustag,
                                is->is_iommu,
                                offsetof(struct iommureg, iommu_cr)),
                            (unsigned long long)bus_space_read_8(is->is_bustag,
                                is->is_iommu,
                                offsetof(struct iommureg, iommu_tsb)));
                        printf("TSB base %p phys %llx\n", (void *)is->is_tsb,
                            (unsigned long long)is->is_ptsb);
                        delay(1000000); /* 1 s */
                }
        }
#endif

        /*
         * Now all the hardware's working we need to allocate a dvma map.
         */
        aprint_debug("DVMA map: %x to %x\n",
                (unsigned int)is->is_dvmabase,
                (unsigned int)is->is_dvmaend);
        aprint_debug("IOTSB: %llx to %llx\n",
                (unsigned long long)is->is_ptsb,
                (unsigned long long)(is->is_ptsb + size - 1));
        is->is_dvmamap = vmem_create(name,
                                     is->is_dvmabase,
                                     (is->is_dvmaend + 1) - is->is_dvmabase,
                                     PAGE_SIZE,         /* quantum */
                                     NULL,              /* importfn */
                                     NULL,              /* releasefn */
                                     NULL,              /* source */
                                     0,                 /* qcache_max */
                                     VM_SLEEP,
                                     IPL_VM);
        KASSERT(is->is_dvmamap != NULL);

        /*
         * Set the TSB size.  The relevant bits were moved to the TSB
         * base register in the PCIe host bridges.
         */
        if (is->is_flags & IOMMU_TSBSIZE_IN_PTSB)
                is->is_ptsb |= is->is_tsbsize;
        else
                is->is_cr |= (is->is_tsbsize << 16);

        /*
         * now actually start up the IOMMU
         */
        iommu_reset(is);
}

/*
 * Streaming buffers don't exist on the UltraSPARC IIi; we should have
 * detected that already and disabled them.  If not, we will notice that
 * they aren't there when the STRBUF_EN bit does not remain.
 */
void
iommu_reset(struct iommu_state *is)
{
        int i;
        struct strbuf_ctl *sb;

        if (CPU_ISSUN4V)
                return;
        
        IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);

        /* Enable IOMMU in diagnostic mode */
        IOMMUREG_WRITE(is, iommu_cr, is->is_cr|IOMMUCR_DE);

        for (i = 0; i < 2; i++) {
                if ((sb = is->is_sb[i])) {

                        /* Enable diagnostics mode? */
                        bus_space_write_8(is->is_bustag, is->is_sb[i]->sb_sb,
                                STRBUFREG(strbuf_ctl), STRBUF_EN);

                        membar_Lookaside();

                        /* No streaming buffers? Disable them */
                        if (bus_space_read_8(is->is_bustag,
                                is->is_sb[i]->sb_sb,
                                STRBUFREG(strbuf_ctl)) == 0) {
                                is->is_sb[i]->sb_flush = NULL;
                        } else {

                                /*
                                 * locate the pa of the flush buffer.
                                 */
                                if (pmap_extract(pmap_kernel(),
                                     (vaddr_t)is->is_sb[i]->sb_flush,
                                     &is->is_sb[i]->sb_flushpa) == FALSE)
                                        is->is_sb[i]->sb_flush = NULL;
                        }
                }
        }

        if (is->is_flags & IOMMU_FLUSH_CACHE)
                IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL);
}

/*
 * Here are the iommu control routines.
 */

static void
iommu_enter(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags)
{
        DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx flags %x\n",
            va, (long)pa, flags));
        if (!CPU_ISSUN4V)
                iommu_enter_sun4u(sb, va, pa, flags);
        else
                iommu_enter_sun4v(sb, va, pa, flags);
}


void
iommu_enter_sun4u(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags)
{
        struct iommu_state *is = sb->sb_is;
        int strbuf = (flags & BUS_DMA_STREAMING);
        int64_t tte;

#ifdef DIAGNOSTIC
        if (va < is->is_dvmabase || va > is->is_dvmaend)
                panic("iommu_enter: va %#lx not in DVMA space", va);
#endif

        /* Is the streamcache flush really needed? */
        if (sb->sb_flush)
                iommu_strbuf_flush(sb, va);
        else
                /* If we can't flush the strbuf don't enable it. */
                strbuf = 0;

        tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE),
                !(flags & BUS_DMA_NOCACHE), (strbuf));
#ifdef DEBUG
        tte |= (flags & 0xff000LL)<<(4*8);
#endif

        is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte;
        bus_space_write_8(is->is_bustag, is->is_iommu,
                IOMMUREG(iommu_flush), va);
        DPRINTF(IDB_IOMMU, ("iommu_enter: slot %d va %lx pa %lx "
                "TSB[%lx]@%p=%lx\n", (int)IOTSBSLOT(va,is->is_tsbsize),
                va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize),
                (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)],
                (u_long)tte));
}

void
iommu_enter_sun4v(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags)
{
        struct iommu_state *is = sb->sb_is;
        u_int64_t tsbid = IOTSBSLOT(va, is->is_tsbsize);
        paddr_t page_list[1], addr;
        u_int64_t attr, nmapped;
        int err;

#ifdef DIAGNOSTIC
        if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
                panic("viommu_enter: va %#lx not in DVMA space", va);
#endif

        attr = PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE;
        if (flags & BUS_DMA_READ)
                attr &= ~PCI_MAP_ATTR_READ;
        if (flags & BUS_DMA_WRITE)
                attr &= ~PCI_MAP_ATTR_WRITE;

        page_list[0] = trunc_page(pa);
        if (!pmap_extract(pmap_kernel(), (vaddr_t)page_list, &addr))
                panic("viommu_enter: pmap_extract failed");
        err = hv_pci_iommu_map(is->is_devhandle, tsbid, 1, attr,
            addr, &nmapped);
        if (err != H_EOK || nmapped != 1)
                panic("hv_pci_iommu_map: err=%d, nmapped=%lu", err, (long unsigned int)nmapped);
}

/*
 * Find the value of a DVMA address (debug routine).
 */
paddr_t
iommu_extract(struct iommu_state *is, vaddr_t dva)
{
        int64_t tte = 0;

        if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
                tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];

        if ((tte & IOTTE_V) == 0)
                return ((paddr_t)-1L);
        return (tte & IOTTE_PAMASK);
}

/*
 * iommu_remove: removes mappings created by iommu_enter
 *
 * Only demap from IOMMU if flag is set.
 *
 * XXX: this function needs better internal error checking.
 */

static void
iommu_remove(struct iommu_state *is, vaddr_t va, size_t len)
{
        DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx len %zu\n", va, len));
        if (!CPU_ISSUN4V)
                iommu_remove_sun4u(is, va, len);
        else
                iommu_remove_sun4v(is, va, len);
}

void
iommu_remove_sun4u(struct iommu_state *is, vaddr_t va, size_t len)
{

        int slot;

#ifdef DIAGNOSTIC
        if (va < is->is_dvmabase || va > is->is_dvmaend)
                panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
        if ((long)(va + len) < (long)va)
                panic("iommu_remove: va 0x%lx + len 0x%lx wraps",
                      (long) va, (long) len);
        if (len & ~0xfffffff)
                panic("iommu_remove: ridiculous len 0x%lx", (u_long)len);
#endif

        va = trunc_page(va);
        DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n",
                va, (u_long)IOTSBSLOT(va, is->is_tsbsize),
                &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]));
        while (len > 0) {
                DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d "
                        "for va %p size %lx\n",
                        (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va,
                        (u_long)len));
                if (len <= PAGE_SIZE)
                        len = 0;
                else
                        len -= PAGE_SIZE;

#if 0
                /*
                 * XXX Zero-ing the entry would not require RMW
                 *
                 * Disabling valid bit while a page is used by a device
                 * causes an uncorrectable DMA error.
                 * Workaround to avoid an uncorrectable DMA error is
                 * eliminating the next line, but the page is mapped
                 * until the next iommu_enter call.
                 */
                is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V;
                membar_StoreStore();
#endif
                IOMMUREG_WRITE(is, iommu_flush, va);

                /* Flush cache if necessary. */
                slot = IOTSBSLOT(trunc_page(va), is->is_tsbsize);
                if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
                    (len == 0 || (slot % 8) == 7))
                        IOMMUREG_WRITE(is, iommu_cache_flush,
                            is->is_ptsb + slot * 8);

                va += PAGE_SIZE;
        }
}

void
iommu_remove_sun4v(struct iommu_state *is, vaddr_t va, size_t len)
{
        u_int64_t tsbid = IOTSBSLOT(va, is->is_tsbsize);
        u_int64_t ndemapped;
        int err;

#ifdef DIAGNOSTIC
        if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
                panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
        if (va != trunc_page(va)) {
                printf("iommu_remove: unaligned va: %lx\n", va);
                va = trunc_page(va);
        }
#endif

        err = hv_pci_iommu_demap(is->is_devhandle, tsbid, 1, &ndemapped);
        if (err != H_EOK || ndemapped != 1)
                panic("hv_pci_iommu_unmap: err=%d", err);
}

static int
iommu_strbuf_flush_done(struct strbuf_ctl *sb)
{
        struct iommu_state *is = sb->sb_is;
        struct timeval cur, flushtimeout;

#define BUMPTIME(t, usec) { \
        register volatile struct timeval *tp = (t); \
        register long us; \
 \
        tp->tv_usec = us = tp->tv_usec + (usec); \
        if (us >= 1000000) { \
                tp->tv_usec = us - 1000000; \
                tp->tv_sec++; \
        } \
}

        if (!sb->sb_flush)
                return (0);

        /*
         * Streaming buffer flushes:
         *
         *   1 Tell strbuf to flush by storing va to strbuf_pgflush.  If
         *     we're not on a cache line boundary (64-bits):
         *   2 Store 0 in flag
         *   3 Store pointer to flag in flushsync
         *   4 wait till flushsync becomes 0x1
         *
         * If it takes more than .5 sec, something
         * went wrong.
         */

        *sb->sb_flush = 0;
        bus_space_write_8(is->is_bustag, sb->sb_sb,
                STRBUFREG(strbuf_flushsync), sb->sb_flushpa);

        microtime(&flushtimeout);
        cur = flushtimeout;
        BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */

        DPRINTF(IDB_IOMMU, ("%s: flush = %lx at va = %lx pa = %lx now="
                "%"PRIx64":%"PRIx32" until = %"PRIx64":%"PRIx32"\n", __func__,
                (long)*sb->sb_flush, (long)sb->sb_flush, (long)sb->sb_flushpa,
                cur.tv_sec, cur.tv_usec,
                flushtimeout.tv_sec, flushtimeout.tv_usec));

        /* Bypass non-coherent D$ */
        while ((!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) &&
               timercmp(&cur, &flushtimeout, <=))
                microtime(&cur);

#ifdef DIAGNOSTIC
        if (!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) {
                printf("%s: flush timeout %p, at %p\n", __func__,
                        (void *)(u_long)*sb->sb_flush,
                        (void *)(u_long)sb->sb_flushpa); /* panic? */
#ifdef DDB
                Debugger();
#endif
        }
#endif
        DPRINTF(IDB_IOMMU, ("%s: flushed\n", __func__));
        return (*sb->sb_flush);
}

/*
 * IOMMU DVMA operations, common to SBUS and PCI.
 */
int
iommu_dvmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
        bus_size_t buflen, struct proc *p, int flags)
{
        struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
        struct iommu_state *is = sb->sb_is;
        int err, needsflush;
        bus_size_t sgsize;
        paddr_t curaddr;
        u_long sgstart, sgend, bmask;
        vmem_addr_t dvmaddr;
        bus_size_t align, boundary, len;
        vaddr_t vaddr = (vaddr_t)buf;
        int seg;
        struct pmap *pmap;
        int slot;

        if (map->dm_nsegs) {
                /* Already in use?? */
#ifdef DIAGNOSTIC
                printf("iommu_dvmamap_load: map still in use\n");
#endif
                bus_dmamap_unload(t, map);
        }

        /*
         * Make sure that on error condition we return "no valid mappings".
         */
        map->dm_nsegs = 0;
        KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);

        if (buflen > map->_dm_size) {
                DPRINTF(IDB_BUSDMA,
                    ("iommu_dvmamap_load(): error %d > %d -- "
                     "map size exceeded!\n", (int)buflen, (int)map->_dm_size));
                return (EINVAL);
        }

        sgsize = round_page(buflen + ((int)vaddr & PGOFSET));

        /*
         * A boundary presented to bus_dmamem_alloc() takes precedence
         * over boundary in the map.
         */
        if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0)
                boundary = map->_dm_boundary;
        align = uimax(map->dm_segs[0]._ds_align, PAGE_SIZE);

        /*
         * If our segment size is larger than the boundary we need to
         * split the transfer up int little pieces ourselves.
         */
        KASSERT(is->is_dvmamap != NULL);
        err = vmem_xalloc(is->is_dvmamap, sgsize,
                          align,                /* alignment */
                          0,                    /* phase */
                          (sgsize > boundary) ? 0 : boundary,
                          VMEM_ADDR_MIN,        /* minaddr */
                          VMEM_ADDR_MAX,        /* maxaddr */
                          VM_NOSLEEP | VM_BESTFIT,
                          &dvmaddr);

#ifdef DEBUG
        if (err || (dvmaddr == (u_long)-1)) {
                printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n",
                    (int)sgsize, flags);
#ifdef DDB
                Debugger();
#endif
        }
#endif
        if (err != 0)
                return (err);

        if (dvmaddr == (u_long)-1)
                return (ENOMEM);

        /* Set the active DVMA map */
        map->_dm_dvmastart = dvmaddr;
        map->_dm_dvmasize = sgsize;

        /*
         * Now split the DVMA range into segments, not crossing
         * the boundary.
         */
        seg = 0;
        sgstart = dvmaddr + (vaddr & PGOFSET);
        sgend = sgstart + buflen - 1;
        map->dm_segs[seg].ds_addr = sgstart;
        DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary - 1 %lx "
            "~(boundary - 1) %lx\n", (long)boundary, (long)(boundary - 1),
            (long)~(boundary - 1)));
        bmask = ~(boundary - 1);
        while ((sgstart & bmask) != (sgend & bmask) ||
               sgend - sgstart + 1 > map->dm_maxsegsz) {
                /* Oops. We crossed a boundary or large seg. Split the xfer. */
                len = map->dm_maxsegsz;
                if ((sgstart & bmask) != (sgend & bmask))
                        len = uimin(len, boundary - (sgstart & (boundary - 1)));
                map->dm_segs[seg].ds_len = len;
                DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
                    "seg %d start %lx size %lx\n", seg,
                    (long)map->dm_segs[seg].ds_addr,
                    (long)map->dm_segs[seg].ds_len));
                if (++seg >= map->_dm_segcnt) {
                        /* Too many segments.  Fail the operation. */
                        DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
                            "too many segments %d\n", seg));
                        vmem_xfree(is->is_dvmamap, dvmaddr, sgsize);
                        map->_dm_dvmastart = 0;
                        map->_dm_dvmasize = 0;
                        return (EFBIG);
                }
                sgstart += len;
                map->dm_segs[seg].ds_addr = sgstart;
        }
        map->dm_segs[seg].ds_len = sgend - sgstart + 1;
        DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
            "seg %d start %lx size %lx\n", seg,
            (long)map->dm_segs[seg].ds_addr, (long)map->dm_segs[seg].ds_len));
        map->dm_nsegs = seg + 1;
        map->dm_mapsize = buflen;

        if (p != NULL)
                pmap = p->p_vmspace->vm_map.pmap;
        else
                pmap = pmap_kernel();

        needsflush = 0;
        for (; buflen > 0; ) {

                /*
                 * Get the physical address for this page.
                 */
                if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) {
#ifdef DIAGNOSTIC
                        printf("iommu_dvmamap_load: pmap_extract failed %lx\n", vaddr);
#endif
                        bus_dmamap_unload(t, map);
                        return (-1);
                }

                /*
                 * Compute the segment size, and adjust counts.
                 */
                sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
                if (buflen < sgsize)
                        sgsize = buflen;

                DPRINTF(IDB_BUSDMA,
                    ("iommu_dvmamap_load: map %p loading va %p "
                    "dva %lx at pa %lx\n",
                    map, (void *)vaddr, (long)dvmaddr,
                    (long)trunc_page(curaddr)));
                iommu_enter(sb, trunc_page(dvmaddr), trunc_page(curaddr),
                    flags | IOTTE_DEBUG(0x4000));
                needsflush = 1;

                vaddr += sgsize;
                buflen -= sgsize;

                /* Flush cache if necessary. */
                slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize);
                if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
                    (buflen <= 0 || (slot % 8) == 7))
                        IOMMUREG_WRITE(is, iommu_cache_flush,
                            is->is_ptsb + slot * 8);

                dvmaddr += PAGE_SIZE;
        }
        if (needsflush)
                iommu_strbuf_flush_done(sb);
#ifdef DIAGNOSTIC
        for (seg = 0; seg < map->dm_nsegs; seg++) {
                if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
                        map->dm_segs[seg].ds_addr > is->is_dvmaend) {
                        printf("seg %d dvmaddr %lx out of range %x - %x\n",
                            seg, (long)map->dm_segs[seg].ds_addr,
                            is->is_dvmabase, is->is_dvmaend);
#ifdef DDB
                        Debugger();
#endif
                }
        }
#endif
        return (0);
}


void
iommu_dvmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
{
        struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
        struct iommu_state *is = sb->sb_is;

        /* Flush the iommu */
        if (!map->_dm_dvmastart)
                panic("%s: error dvmastart is zero!\n", __func__);

        if (is->is_flags & IOMMU_SYNC_BEFORE_UNMAP) {

                /* Flush the caches */
                bus_dmamap_unload(t->_parent, map);

                iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize);

        } else {

                iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize);

                /* Flush the caches */
                bus_dmamap_unload(t->_parent, map);
        }

        vmem_xfree(is->is_dvmamap, map->_dm_dvmastart, map->_dm_dvmasize);
        map->_dm_dvmastart = 0;
        map->_dm_dvmasize = 0;

        /* Clear the map */
}


int
iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
        bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
{
        struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
        struct iommu_state *is = sb->sb_is;
        struct vm_page *pg;
        int i, j;
        int left;
        int err, needsflush;
        bus_size_t sgsize;
        paddr_t pa;
        bus_size_t boundary, align;
        u_long dvmaddr, sgstart, sgend, bmask;
        struct pglist *pglist;
        const int pagesz = PAGE_SIZE;
        int slot;
#ifdef DEBUG
        int npg = 0;
#endif

        if (map->dm_nsegs) {
                /* Already in use?? */
#ifdef DIAGNOSTIC
                printf("iommu_dvmamap_load_raw: map still in use\n");
#endif
                bus_dmamap_unload(t, map);
        }

        /*
         * A boundary presented to bus_dmamem_alloc() takes precedence
         * over boundary in the map.
         */
        if ((boundary = segs[0]._ds_boundary) == 0)
                boundary = map->_dm_boundary;

        align = uimax(segs[0]._ds_align, pagesz);

        /*
         * Make sure that on error condition we return "no valid mappings".
         */
        map->dm_nsegs = 0;
        /* Count up the total number of pages we need */
        pa = trunc_page(segs[0].ds_addr);
        sgsize = 0;
        left = size;
        for (i = 0; left > 0 && i < nsegs; i++) {
                if (round_page(pa) != round_page(segs[i].ds_addr))
                        sgsize = round_page(sgsize) +
                            (segs[i].ds_addr & PGOFSET);
                sgsize += uimin(left, segs[i].ds_len);
                left -= segs[i].ds_len;
                pa = segs[i].ds_addr + segs[i].ds_len;
        }
        sgsize = round_page(sgsize);

        /*
         * If our segment size is larger than the boundary we need to
         * split the transfer up into little pieces ourselves.
         */
        const vm_flag_t vmflags = VM_BESTFIT |
            ((flags & BUS_DMA_NOWAIT) ? VM_NOSLEEP : VM_SLEEP);

        err = vmem_xalloc(is->is_dvmamap, sgsize,
                          align,                /* alignment */
                          0,                    /* phase */
                          (sgsize > boundary) ? 0 : boundary,
                          VMEM_ADDR_MIN,        /* minaddr */
                          VMEM_ADDR_MAX,        /* maxaddr */
                          vmflags,
                          &dvmaddr);
        if (err != 0)
                return (err);

#ifdef DEBUG
        if (dvmaddr == (u_long)-1)
        {
                printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n",
                    (int)sgsize, flags);
#ifdef DDB
                Debugger();
#endif
        }
#endif
        if (dvmaddr == (u_long)-1)
                return (ENOMEM);

        /* Set the active DVMA map */
        map->_dm_dvmastart = dvmaddr;
        map->_dm_dvmasize = sgsize;

        bmask = ~(boundary - 1);
        if ((pglist = segs[0]._ds_mlist) == NULL) {
                u_long prev_va = 0UL, last_va = dvmaddr;
                paddr_t prev_pa = 0;
                int end = 0, offset;
                bus_size_t len = size;

                /*
                 * This segs is made up of individual physical
                 *  segments, probably by _bus_dmamap_load_uio() or
                 * _bus_dmamap_load_mbuf().  Ignore the mlist and
                 * load each one individually.
                 */
                j = 0;
                needsflush = 0;
                for (i = 0; i < nsegs ; i++) {

                        pa = segs[i].ds_addr;
                        offset = (pa & PGOFSET);
                        pa = trunc_page(pa);
                        dvmaddr = trunc_page(dvmaddr);
                        left = uimin(len, segs[i].ds_len);

                        DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: converting "
                                "physseg %d start %lx size %lx\n", i,
                                (long)segs[i].ds_addr, (long)segs[i].ds_len));

                        if ((pa == prev_pa) &&
                                ((offset != 0) || (end != offset))) {
                                /* We can re-use this mapping */
                                dvmaddr = prev_va;
                        }

                        sgstart = dvmaddr + offset;
                        sgend = sgstart + left - 1;

                        /* Are the segments virtually adjacent? */
                        if ((j > 0) && (end == offset) &&
                            ((offset == 0) || (pa == prev_pa)) &&
                            (map->dm_segs[j-1].ds_len + left <=
                             map->dm_maxsegsz)) {
                                /* Just append to the previous segment. */
                                map->dm_segs[--j].ds_len += left;
                                /* Restore sgstart for boundary check */
                                sgstart = map->dm_segs[j].ds_addr;
                                DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
                                        "appending seg %d start %lx size %lx\n", j,
                                        (long)map->dm_segs[j].ds_addr,
                                        (long)map->dm_segs[j].ds_len));
                        } else {
                                if (j >= map->_dm_segcnt) {
                                        iommu_remove(is, map->_dm_dvmastart,
                                            last_va - map->_dm_dvmastart);
                                        goto fail;
                                }
                                map->dm_segs[j].ds_addr = sgstart;
                                map->dm_segs[j].ds_len = left;
                                DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
                                        "seg %d start %lx size %lx\n", j,
                                        (long)map->dm_segs[j].ds_addr,
                                        (long)map->dm_segs[j].ds_len));
                        }
                        end = (offset + left) & PGOFSET;

                        /* Check for boundary issues */
                        while ((sgstart & bmask) != (sgend & bmask)) {
                                /* Need a new segment. */
                                map->dm_segs[j].ds_len =
                                        boundary - (sgstart & (boundary - 1));
                                DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
                                        "seg %d start %lx size %lx\n", j,
                                        (long)map->dm_segs[j].ds_addr,
                                        (long)map->dm_segs[j].ds_len));
                                if (++j >= map->_dm_segcnt) {
                                        iommu_remove(is, map->_dm_dvmastart,
                                            last_va - map->_dm_dvmastart);
                                        goto fail;
                                }
                                sgstart += map->dm_segs[j-1].ds_len;
                                map->dm_segs[j].ds_addr = sgstart;
                                map->dm_segs[j].ds_len = sgend - sgstart + 1;
                        }

                        if (sgsize == 0)
                                panic("iommu_dmamap_load_raw: size botch");

                        /* Now map a series of pages. */
                        while (dvmaddr <= sgend) {
                                DPRINTF(IDB_BUSDMA,
                                        ("iommu_dvmamap_load_raw: map %p "
                                                "loading va %lx at pa %lx\n",
                                                map, (long)dvmaddr,
                                                (long)(pa)));
                                /* Enter it if we haven't before. */
                                if (prev_va != dvmaddr) {
                                        iommu_enter(sb, prev_va = dvmaddr,
                                            prev_pa = pa,
                                            flags | IOTTE_DEBUG(++npg << 12));
                                        needsflush = 1;

                                        /* Flush cache if necessary. */
                                        slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize);
                                        if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
                                            ((dvmaddr + pagesz) > sgend || (slot % 8) == 7))
                                                IOMMUREG_WRITE(is, iommu_cache_flush,
                                                    is->is_ptsb + slot * 8);
                                }

                                dvmaddr += pagesz;
                                pa += pagesz;
                                last_va = dvmaddr;
                        }

                        len -= left;
                        ++j;
                }
                if (needsflush)
                        iommu_strbuf_flush_done(sb);

                map->dm_mapsize = size;
                map->dm_nsegs = j;
#ifdef DIAGNOSTIC
                { int seg;
        for (seg = 0; seg < map->dm_nsegs; seg++) {
                if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
                    map->dm_segs[seg].ds_addr > is->is_dvmaend) {
                        printf("seg %d dvmaddr %lx out of range %x - %x\n",
                                seg, (long)map->dm_segs[seg].ds_addr,
                                is->is_dvmabase, is->is_dvmaend);
#ifdef DDB
                        Debugger();
#endif
                }
        }
                }
#endif
                return (0);
        }

        /*
         * This was allocated with bus_dmamem_alloc.
         * The pages are on a `pglist'.
         */
        i = 0;
        sgstart = dvmaddr;
        sgend = sgstart + size - 1;
        map->dm_segs[i].ds_addr = sgstart;
        while ((sgstart & bmask) != (sgend & bmask)) {
                /* Oops.  We crossed a boundary.  Split the xfer. */
                map->dm_segs[i].ds_len = boundary - (sgstart & (boundary - 1));
                DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
                        "seg %d start %lx size %lx\n", i,
                        (long)map->dm_segs[i].ds_addr,
                        (long)map->dm_segs[i].ds_len));
                if (++i >= map->_dm_segcnt) {
                        /* Too many segments.  Fail the operation. */
                        goto fail;
                }
                sgstart += map->dm_segs[i-1].ds_len;
                map->dm_segs[i].ds_addr = sgstart;
        }
        DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
                        "seg %d start %lx size %lx\n", i,
                        (long)map->dm_segs[i].ds_addr, (long)map->dm_segs[i].ds_len));
        map->dm_segs[i].ds_len = sgend - sgstart + 1;

        needsflush = 0;
        TAILQ_FOREACH(pg, pglist, pageq.queue) {
                if (sgsize == 0)
                        panic("iommu_dmamap_load_raw: size botch");
                pa = VM_PAGE_TO_PHYS(pg);

                DPRINTF(IDB_BUSDMA,
                    ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n",
                    map, (long)dvmaddr, (long)(pa)));
                iommu_enter(sb, dvmaddr, pa, flags | IOTTE_DEBUG(0x8000));
                needsflush = 1;

                sgsize -= pagesz;

                /* Flush cache if necessary. */
                slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize);
                if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
                    (sgsize == 0 || (slot % 8) == 7))
                        IOMMUREG_WRITE(is, iommu_cache_flush,
                            is->is_ptsb + slot * 8);

                dvmaddr += pagesz;
        }
        if (needsflush)
                iommu_strbuf_flush_done(sb);
        map->dm_mapsize = size;
        map->dm_nsegs = i+1;
#ifdef DIAGNOSTIC
        { int seg;
        for (seg = 0; seg < map->dm_nsegs; seg++) {
                if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
                        map->dm_segs[seg].ds_addr > is->is_dvmaend) {
                        printf("seg %d dvmaddr %lx out of range %x - %x\n",
                                seg, (long)map->dm_segs[seg].ds_addr,
                                is->is_dvmabase, is->is_dvmaend);
#ifdef DDB
                        Debugger();
#endif
                }
        }
        }
#endif
        return (0);

fail:
        vmem_xfree(is->is_dvmamap, map->_dm_dvmastart, sgsize);
        map->_dm_dvmastart = 0;
        map->_dm_dvmasize = 0;
        return (EFBIG);
}


/*
 * Flush an individual dma segment, returns non-zero if the streaming buffers
 * need flushing afterwards.
 */
static int
iommu_dvmamap_sync_range(struct strbuf_ctl *sb, vaddr_t va, bus_size_t len)
{
        vaddr_t vaend;
        struct iommu_state *is = sb->sb_is;

#ifdef DIAGNOSTIC
        if (va < is->is_dvmabase || va > is->is_dvmaend)
                panic("invalid va: %llx", (long long)va);
#endif

        if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) {
                DPRINTF(IDB_SYNC, 
                        ("iommu_dvmamap_sync_range: attempting to flush "
                         "non-streaming entry\n"));
                return (0);
        }

        vaend = round_page(va + len) - 1;
        va = trunc_page(va);

#ifdef DIAGNOSTIC
        if (va < is->is_dvmabase || vaend > is->is_dvmaend)
                panic("invalid va range: %llx to %llx (%x to %x)",
                    (long long)va, (long long)vaend,
                    is->is_dvmabase,
                    is->is_dvmaend);
#endif

        for ( ; va <= vaend; va += PAGE_SIZE) {
                DPRINTF(IDB_SYNC,
                    ("iommu_dvmamap_sync_range: flushing va %p\n",
                    (void *)(u_long)va));
                iommu_strbuf_flush(sb, va);
        }

        return (1);
}

static void
_iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
        bus_size_t len, int ops)
{
        struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
        bus_size_t count;
        int i, needsflush = 0;

        if (!sb->sb_flush)
                return;

        for (i = 0; i < map->dm_nsegs; i++) {
                if (offset < map->dm_segs[i].ds_len)
                        break;
                offset -= map->dm_segs[i].ds_len;
        }

        if (i == map->dm_nsegs)
                panic("%s: segment too short %llu", __func__,
                    (unsigned long long)offset);

        if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) {
                /* Nothing to do */;
        }

        if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) {

                for (; len > 0 && i < map->dm_nsegs; i++) {
                        count = MIN(map->dm_segs[i].ds_len - offset, len);
                        if (count > 0 && 
                            iommu_dvmamap_sync_range(sb,
                                map->dm_segs[i].ds_addr + offset, count))
                                needsflush = 1;
                        offset = 0;
                        len -= count;
                }
#ifdef DIAGNOSTIC
                if (i == map->dm_nsegs && len > 0)
                        panic("%s: leftover %llu", __func__,
                            (unsigned long long)len);
#endif

                if (needsflush)
                        iommu_strbuf_flush_done(sb);
        }
}

void
iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
        bus_size_t len, int ops)
{

        /* If len is 0, then there is nothing to do */
        if (len == 0)
                return;

        if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) {
                /* Flush the CPU then the IOMMU */
                bus_dmamap_sync(t->_parent, map, offset, len, ops);
                _iommu_dvmamap_sync(t, map, offset, len, ops);
        }
        if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)) {
                /* Flush the IOMMU then the CPU */
                _iommu_dvmamap_sync(t, map, offset, len, ops);
                bus_dmamap_sync(t->_parent, map, offset, len, ops);
        }
}

int
iommu_dvmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
        bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
        int flags)
{

        DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx "
           "segp %p flags %d\n", (unsigned long long)size,
           (unsigned long long)alignment, (unsigned long long)boundary,
           segs, flags));
        return (bus_dmamem_alloc(t->_parent, size, alignment, boundary,
            segs, nsegs, rsegs, flags|BUS_DMA_DVMA));
}

void
iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs)
{

        DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n",
            segs, nsegs));
        bus_dmamem_free(t->_parent, segs, nsegs);
}

/*
 * Map the DVMA mappings into the kernel pmap.
 * Check the flags to see whether we're streaming or coherent.
 */
int
iommu_dvmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs,
        size_t size, void **kvap, int flags)
{
        struct vm_page *pg;
        vaddr_t va;
        bus_addr_t addr;
        struct pglist *pglist;
        int cbit;
        const uvm_flag_t kmflags =
            (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0;

        DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n",
            segs, nsegs, size));

        /*
         * Allocate some space in the kernel map, and then map these pages
         * into this space.
         */
        size = round_page(size);
        va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags);
        if (va == 0)
                return (ENOMEM);

        *kvap = (void *)va;

        /*
         * digest flags:
         */
        cbit = 0;
        if (flags & BUS_DMA_COHERENT)   /* Disable vcache */
                cbit |= PMAP_NVC;
        if (flags & BUS_DMA_NOCACHE)    /* side effects */
                cbit |= PMAP_NC;

        /*
         * Now take this and map it into the CPU.
         */
        pglist = segs[0]._ds_mlist;
        TAILQ_FOREACH(pg, pglist, pageq.queue) {
#ifdef DIAGNOSTIC
                if (size == 0)
                        panic("iommu_dvmamem_map: size botch");
#endif
                addr = VM_PAGE_TO_PHYS(pg);
                DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: "
                    "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit));
                pmap_kenter_pa(va, addr | cbit,
                    VM_PROT_READ | VM_PROT_WRITE, 0);
                va += PAGE_SIZE;
                size -= PAGE_SIZE;
        }
        pmap_update(pmap_kernel());
        return (0);
}

/*
 * Unmap DVMA mappings from kernel
 */
void
iommu_dvmamem_unmap(bus_dma_tag_t t, void *kva, size_t size)
{

        DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n",
            kva, size));

#ifdef DIAGNOSTIC
        if ((u_long)kva & PGOFSET)
                panic("iommu_dvmamem_unmap");
#endif

        size = round_page(size);
        pmap_kremove((vaddr_t)kva, size);
        pmap_update(pmap_kernel());
        uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY);
}