/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#include <sys/sunddi.h>
#include <sys/sunndi.h>
#include <sys/acpi/acpi.h>
#include <sys/acpica.h>
#include <sys/amd_iommu.h>
#include <sys/bootconf.h>
#include <sys/sysmacros.h>
#include <sys/ddidmareq.h>

#include "amd_iommu_impl.h"
#include "amd_iommu_acpi.h"
#include "amd_iommu_page_tables.h"

ddi_dma_attr_t amd_iommu_pgtable_dma_attr = {
        DMA_ATTR_V0,
        0U,                             /* dma_attr_addr_lo */
        0xffffffffffffffffULL,          /* dma_attr_addr_hi */
        0xffffffffU,                    /* dma_attr_count_max */
        (uint64_t)4096,                 /* dma_attr_align */
        1,                              /* dma_attr_burstsizes */
        64,                             /* dma_attr_minxfer */
        0xffffffffU,                    /* dma_attr_maxxfer */
        0xffffffffU,                    /* dma_attr_seg */
        1,                              /* dma_attr_sgllen, variable */
        64,                             /* dma_attr_granular */
        0                               /* dma_attr_flags */
};

static amd_iommu_domain_t **amd_iommu_domain_table;

static struct {
        int f_count;
        amd_iommu_page_table_t *f_list;
} amd_iommu_pgtable_freelist;
int amd_iommu_no_pgtable_freelist;

/*ARGSUSED*/
static int
amd_iommu_get_src_bdf(amd_iommu_t *iommu, int32_t bdf, int32_t *src_bdfp)
{
        amd_iommu_acpi_ivhd_t *hinfop;

        hinfop = amd_iommu_lookup_ivhd(bdf);
        if (hinfop == NULL) {
                if (bdf == -1) {
                        *src_bdfp = bdf;
                } else {
                        cmn_err(CE_WARN, "No IVHD entry for 0x%x", bdf);
                        return (DDI_FAILURE);
                }
        } else if (hinfop->ach_src_deviceid == -1) {
                *src_bdfp = bdf;
        } else {
                *src_bdfp = hinfop->ach_src_deviceid;
        }

        return (DDI_SUCCESS);
}

/*ARGSUSED*/
static int
amd_iommu_get_domain(amd_iommu_t *iommu, dev_info_t *rdip, int alias,
    uint16_t deviceid, domain_id_t *domainid, const char *path)
{
        const char *f = "amd_iommu_get_domain";

        *domainid = AMD_IOMMU_INVALID_DOMAIN;

        ASSERT(strcmp(ddi_driver_name(rdip), "agpgart") != 0);

        switch (deviceid) {
                case AMD_IOMMU_INVALID_DOMAIN:
                case AMD_IOMMU_IDENTITY_DOMAIN:
                case AMD_IOMMU_PASSTHRU_DOMAIN:
                case AMD_IOMMU_SYS_DOMAIN:
                        *domainid = AMD_IOMMU_SYS_DOMAIN;
                        break;
                default:
                        *domainid = deviceid;
                        break;
        }

        if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
                cmn_err(CE_NOTE, "%s: domainid for %s = %d",
                    f, path, *domainid);
        }

        return (DDI_SUCCESS);
}

static uint16_t
hash_domain(domain_id_t domainid)
{
        return (domainid % AMD_IOMMU_DOMAIN_HASH_SZ);
}

/*ARGSUSED*/
void
amd_iommu_init_page_tables(amd_iommu_t *iommu)
{
        amd_iommu_domain_table = kmem_zalloc(
            sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ, KM_SLEEP);
}

/*ARGSUSED*/
void
amd_iommu_fini_page_tables(amd_iommu_t *iommu)
{
        if (amd_iommu_domain_table) {
                kmem_free(amd_iommu_domain_table,
                    sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ);
                amd_iommu_domain_table = NULL;
        }
}

static amd_iommu_domain_t *
amd_iommu_lookup_domain(amd_iommu_t *iommu, domain_id_t domainid,
    map_type_t type, int km_flags)
{
        uint16_t idx;
        amd_iommu_domain_t *dp;
        char name[AMD_IOMMU_VMEM_NAMELEN+1];

        ASSERT(amd_iommu_domain_table);

        idx = hash_domain(domainid);

        for (dp = amd_iommu_domain_table[idx]; dp; dp = dp->d_next) {
                if (dp->d_domainid == domainid)
                        return (dp);
        }

        ASSERT(type != AMD_IOMMU_INVALID_MAP);

        dp = kmem_zalloc(sizeof (*dp), km_flags);
        if (dp == NULL)
                return (NULL);
        dp->d_domainid = domainid;
        dp->d_pgtable_root_4K = 0;      /* make this explicit */

        if (type == AMD_IOMMU_VMEM_MAP) {
                uint64_t base;
                uint64_t size;
                (void) snprintf(name, sizeof (name), "dvma_idx%d_domain%d",
                    iommu->aiomt_idx, domainid);
                base = MMU_PAGESIZE;
                size = AMD_IOMMU_SIZE_4G - MMU_PAGESIZE;
                dp->d_vmem = vmem_create(name, (void *)(uintptr_t)base, size,
                    MMU_PAGESIZE, NULL, NULL, NULL, 0,
                    km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
                if (dp->d_vmem == NULL) {
                        kmem_free(dp, sizeof (*dp));
                        return (NULL);
                }
        } else {
                dp->d_vmem = NULL;
        }

        dp->d_next = amd_iommu_domain_table[idx];
        dp->d_prev = NULL;
        amd_iommu_domain_table[idx] = dp;
        if (dp->d_next)
                dp->d_next->d_prev = dp;
        dp->d_ref = 0;


        return (dp);
}

static void
amd_iommu_teardown_domain(amd_iommu_t *iommu, amd_iommu_domain_t *dp)
{
        uint16_t idx;
        int flags;
        amd_iommu_cmdargs_t cmdargs = {0};
        domain_id_t domainid = dp->d_domainid;
        const char *f = "amd_iommu_teardown_domain";

        ASSERT(dp->d_ref == 0);

        idx = hash_domain(dp->d_domainid);

        if (dp->d_prev == NULL)
                amd_iommu_domain_table[idx] = dp->d_next;
        else
                dp->d_prev->d_next = dp->d_next;

        if (dp->d_next)
                dp->d_next->d_prev = dp->d_prev;

        if (dp->d_vmem != NULL) {
                vmem_destroy(dp->d_vmem);
                dp->d_vmem = NULL;
        }

        kmem_free(dp, sizeof (*dp));

        cmdargs.ca_domainid = (uint16_t)domainid;
        cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
        flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
            AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;

        if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
            &cmdargs, flags, 0) != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: idx=%d: domainid=%d"
                    "Failed to invalidate domain in IOMMU HW cache",
                    f, iommu->aiomt_idx, cmdargs.ca_domainid);
        }
}

static int
amd_iommu_get_deviceid(amd_iommu_t *iommu, dev_info_t *rdip, int32_t *deviceid,
    int *aliasp, const char *path)
{
        int bus = -1;
        int device = -1;
        int func = -1;
        uint16_t bdf;
        int32_t src_bdf;
        dev_info_t *idip = iommu->aiomt_dip;
        const char *driver = ddi_driver_name(idip);
        int instance = ddi_get_instance(idip);
        dev_info_t *pci_dip;
        const char *f = "amd_iommu_get_deviceid";

        /* be conservative. Always assume an alias */
        *aliasp = 1;
        *deviceid = 0;

        /* Check for special special devices (rdip == NULL) */
        if (rdip == NULL) {
                if (amd_iommu_get_src_bdf(iommu, -1, &src_bdf) != DDI_SUCCESS) {
                        cmn_err(CE_WARN,
                            "%s: %s%d: idx=%d, failed to get SRC BDF "
                            "for special-device",
                            f, driver, instance, iommu->aiomt_idx);
                        return (DDI_DMA_NOMAPPING);
                }
                *deviceid = src_bdf;
                *aliasp = 1;
                return (DDI_SUCCESS);
        }

        if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
                cmn_err(CE_NOTE, "%s: attempting to get deviceid for %s",
                    f, path);
        }

        pci_dip = amd_iommu_pci_dip(rdip, path);
        if (pci_dip == NULL) {
                cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get PCI dip "
                    "for rdip=%p, path = %s",
                    f, driver, instance, iommu->aiomt_idx, (void *)rdip,
                    path);
                return (DDI_DMA_NOMAPPING);
        }

        if (acpica_get_bdf(pci_dip, &bus, &device, &func) != DDI_SUCCESS) {
                ndi_rele_devi(pci_dip);
                cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get BDF for "
                    "PCI dip (%p). rdip path = %s",
                    f, driver, instance, iommu->aiomt_idx,
                    (void *)pci_dip, path);
                return (DDI_DMA_NOMAPPING);
        }

        ndi_rele_devi(pci_dip);

        if (bus > UINT8_MAX || bus < 0 ||
            device > UINT8_MAX || device < 0 ||
            func > UINT8_MAX || func < 0) {
                cmn_err(CE_WARN, "%s: %s%d:  idx=%d, invalid BDF(%d,%d,%d) "
                    "for PCI dip (%p). rdip path = %s", f, driver, instance,
                    iommu->aiomt_idx,
                    bus, device, func,
                    (void *)pci_dip, path);
                return (DDI_DMA_NOMAPPING);
        }

        bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) | (uint8_t)func;

        if (amd_iommu_get_src_bdf(iommu, bdf, &src_bdf) != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get SRC BDF "
                    "for PCI dip (%p) rdip path = %s.",
                    f, driver, instance, iommu->aiomt_idx, (void *)pci_dip,
                    path);
                return (DDI_DMA_NOMAPPING);
        }

        if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
                cmn_err(CE_NOTE, "%s: Deviceid = %u for path = %s",
                    f, src_bdf, path);
        }

        *deviceid = src_bdf;
        *aliasp = (src_bdf != bdf);

        return (DDI_SUCCESS);
}

/*ARGSUSED*/
static int
init_devtbl(amd_iommu_t *iommu, uint64_t *devtbl_entry, domain_id_t domainid,
    amd_iommu_domain_t *dp)
{
        uint64_t entry[4] = {0};
        int i;

        /* If already passthru, don't touch */
        if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 0 &&
            AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) {
                return (0);
        }

        if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 1 &&
            AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 1) {

                ASSERT(dp->d_pgtable_root_4K ==
                    AMD_IOMMU_REG_GET64(&(devtbl_entry[0]),
                    AMD_IOMMU_DEVTBL_ROOT_PGTBL));

                ASSERT(dp->d_domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]),
                    AMD_IOMMU_DEVTBL_DOMAINID));

                return (0);
        }

        /* New devtbl entry for this domain. Bump up the domain ref-count */
        dp->d_ref++;

        entry[3] = 0;
        entry[2] = 0;
        AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_EX, 1);
        AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SD, 0);
        AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_CACHE, 0);
        AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_IOCTL, 1);
        AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SA, 0);
        AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SE, 1);
        AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_DOMAINID,
            (uint16_t)domainid);
        AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IW, 1);
        AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IR, 1);
        AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL,
            dp->d_pgtable_root_4K);
        AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_PG_MODE,
            AMD_IOMMU_PGTABLE_MAXLEVEL);
        AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_TV,
            domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1);
        AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_V,
            domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1);

        for (i = 1; i < 4; i++) {
                devtbl_entry[i] = entry[i];
        }
        devtbl_entry[0] = entry[0];

        /* we did an actual init */
        return (1);
}

void
amd_iommu_set_passthru(amd_iommu_t *iommu, dev_info_t *rdip)
{
        int32_t deviceid;
        int alias;
        uint64_t *devtbl_entry;
        amd_iommu_cmdargs_t cmdargs = {0};
        char *path;
        int pathfree;
        int V;
        int TV;
        int instance;
        const char *driver;
        const char *f = "amd_iommu_set_passthru";

        if (rdip) {
                driver = ddi_driver_name(rdip);
                instance = ddi_get_instance(rdip);
        } else {
                driver = "special-device";
                instance = 0;
        }

        path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
        if (path) {
                if (rdip)
                        (void) ddi_pathname(rdip, path);
                else
                        (void) strcpy(path, "special-device");
                pathfree = 1;
        } else {
                pathfree = 0;
                path = "<path-mem-alloc-failed>";
        }

        if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
            != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
                    "Failed to get device ID for device %s.", f, driver,
                    instance,
                    iommu->aiomt_idx, (void *)rdip, path);
                goto out;
        }

        /* No deviceid */
        if (deviceid == -1) {
                goto out;
        }

        if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
            iommu->aiomt_devtbl_sz) {
                cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
                    "for rdip (%p) exceeds device table size (%u), path=%s",
                    f, driver,
                    instance, iommu->aiomt_idx, deviceid, (void *)rdip,
                    iommu->aiomt_devtbl_sz, path);
                goto out;
        }

        /*LINTED*/
        devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
            [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];

        V = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V);
        TV = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV);

        /* Already passthru */
        if (V == 0 && TV == 0) {
                goto out;
        }

        /* Existing translations */
        if (V == 1 && TV == 1) {
                goto out;
        }

        /* Invalid setting */
        if (V == 0 && TV == 1) {
                goto out;
        }

        AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 0);

        cmdargs.ca_deviceid = (uint16_t)deviceid;
        (void) amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
            &cmdargs, 0, 0);

out:
        if (pathfree)
                kmem_free(path, MAXPATHLEN);
}

static int
amd_iommu_set_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
    domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp,
    const char *path)
{
        uint64_t *devtbl_entry;
        amd_iommu_cmdargs_t cmdargs = {0};
        int error, flags;
        dev_info_t *idip = iommu->aiomt_dip;
        const char *driver = ddi_driver_name(idip);
        int instance = ddi_get_instance(idip);
        const char *f = "amd_iommu_set_devtbl_entry";

        if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
                cmn_err(CE_NOTE, "%s: attempting to set devtbl entry for %s",
                    f, path);
        }

        if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
            iommu->aiomt_devtbl_sz) {
                cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
                    "for rdip (%p) exceeds device table size (%u), path=%s",
                    f, driver,
                    instance, iommu->aiomt_idx, deviceid, (void *)rdip,
                    iommu->aiomt_devtbl_sz, path);
                return (DDI_DMA_NOMAPPING);
        }

        /*LINTED*/
        devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
            [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];

        if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
                cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s",
                    f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);
        }

        /*
         * Flush internal caches, need to do this if we came up from
         * fast boot
         */
        cmdargs.ca_deviceid = deviceid;
        error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
            &cmdargs, 0, 0);
        if (error != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: idx=%d: deviceid=%d"
                    "Failed to invalidate domain in IOMMU HW cache",
                    f, iommu->aiomt_idx, deviceid);
                return (error);
        }

        cmdargs.ca_domainid = (uint16_t)domainid;
        cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
        flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
            AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;

        error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
            &cmdargs, flags, 0);
        if (error != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: idx=%d: domainid=%d"
                    "Failed to invalidate translations in IOMMU HW cache",
                    f, iommu->aiomt_idx, cmdargs.ca_domainid);
                return (error);
        }

        /* Initialize device table entry */
        if (init_devtbl(iommu, devtbl_entry, domainid, dp)) {
                cmdargs.ca_deviceid = deviceid;
                error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
                    &cmdargs, 0, 0);
        }

        return (error);
}

int
amd_iommu_clear_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
    domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp,
    int *domain_freed, char *path)
{
        uint64_t *devtbl_entry;
        int error = DDI_SUCCESS;
        amd_iommu_cmdargs_t cmdargs = {0};
        const char *driver = ddi_driver_name(iommu->aiomt_dip);
        int instance = ddi_get_instance(iommu->aiomt_dip);
        const char *f = "amd_iommu_clear_devtbl_entry";

        if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
                cmn_err(CE_NOTE, "%s: attempting to clear devtbl entry for "
                    "domainid = %d, deviceid = %u, path = %s",
                    f, domainid, deviceid, path);
        }

        if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
            iommu->aiomt_devtbl_sz) {
                cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
                    "for rdip (%p) exceeds device table size (%u), path = %s",
                    f, driver, instance,
                    iommu->aiomt_idx, deviceid, (void *)rdip,
                    iommu->aiomt_devtbl_sz, path);
                return (DDI_FAILURE);
        }

        /*LINTED*/
        devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
            [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];

        if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
                cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s",
                    f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);
        }

        if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) {
                /* Nothing to do */
                return (DDI_SUCCESS);
        }

        ASSERT(dp->d_pgtable_root_4K == AMD_IOMMU_REG_GET64(&(devtbl_entry[0]),
            AMD_IOMMU_DEVTBL_ROOT_PGTBL));

        ASSERT(domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]),
            AMD_IOMMU_DEVTBL_DOMAINID));

        AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV, 0);
        AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL, 0);
        AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 1);

        SYNC_FORDEV(iommu->aiomt_dmahdl);

        dp->d_ref--;
        ASSERT(dp->d_ref >= 0);

        if (dp->d_ref == 0) {
                *domain_freed = 1;
        }

        cmdargs.ca_deviceid = deviceid;
        error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
            &cmdargs, 0, 0);
        if (error != DDI_SUCCESS)
                error = DDI_FAILURE;

        return (error);
}

int
amd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t *ampt)
{
        ampt->ampt_hash = kmem_zalloc(sizeof (amd_iommu_page_table_t *) *
            AMD_IOMMU_PGTABLE_HASH_SZ, KM_SLEEP);
        return (DDI_SUCCESS);
}

void
amd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t *ampt)
{
        kmem_free(ampt->ampt_hash,
            sizeof (amd_iommu_page_table_t *) * AMD_IOMMU_PGTABLE_HASH_SZ);
        ampt->ampt_hash = NULL;
}

static uint32_t
pt_hashfn(uint64_t pa_4K)
{
        return (pa_4K % AMD_IOMMU_PGTABLE_HASH_SZ);
}

static void
amd_iommu_insert_pgtable_hash(amd_iommu_page_table_t *pt)
{
        uint64_t pa_4K = ((uint64_t)pt->pt_cookie.dmac_cookie_addr) >> 12;
        uint32_t idx = pt_hashfn(pa_4K);

        ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);

        mutex_enter(&amd_iommu_page_table_hash.ampt_lock);

        pt->pt_next = amd_iommu_page_table_hash.ampt_hash[idx];
        pt->pt_prev = NULL;
        amd_iommu_page_table_hash.ampt_hash[idx] = pt;
        if (pt->pt_next)
                pt->pt_next->pt_prev = pt;

        mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
}

static void
amd_iommu_remove_pgtable_hash(amd_iommu_page_table_t *pt)
{
        uint64_t pa_4K = (pt->pt_cookie.dmac_cookie_addr >> 12);
        uint32_t idx = pt_hashfn(pa_4K);

        ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);

        mutex_enter(&amd_iommu_page_table_hash.ampt_lock);

        if (pt->pt_next)
                pt->pt_next->pt_prev = pt->pt_prev;

        if (pt->pt_prev)
                pt->pt_prev->pt_next = pt->pt_next;
        else
                amd_iommu_page_table_hash.ampt_hash[idx] = pt->pt_next;

        pt->pt_next = NULL;
        pt->pt_prev = NULL;

        mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
}

static amd_iommu_page_table_t *
amd_iommu_lookup_pgtable_hash(domain_id_t domainid, uint64_t pgtable_pa_4K)
{
        amd_iommu_page_table_t *pt;
        uint32_t idx = pt_hashfn(pgtable_pa_4K);

        mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
        pt = amd_iommu_page_table_hash.ampt_hash[idx];
        for (; pt; pt = pt->pt_next) {
                if (domainid != pt->pt_domainid)
                        continue;
                ASSERT((pt->pt_cookie.dmac_cookie_addr &
                    AMD_IOMMU_PGTABLE_ALIGN) == 0);
                if ((pt->pt_cookie.dmac_cookie_addr >> 12) == pgtable_pa_4K) {
                        break;
                }
        }
        mutex_exit(&amd_iommu_page_table_hash.ampt_lock);

        return (pt);
}

/*ARGSUSED*/
static amd_iommu_page_table_t *
amd_iommu_lookup_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *ppt,
    amd_iommu_domain_t *dp, int level, uint16_t index)
{
        uint64_t *pdtep;
        uint64_t pgtable_pa_4K;

        ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL);
        ASSERT(dp);

        if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
                ASSERT(ppt == NULL);
                ASSERT(index == 0);
                pgtable_pa_4K = dp->d_pgtable_root_4K;
        } else {
                ASSERT(ppt);
                pdtep = &(ppt->pt_pgtblva[index]);
                if (AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_PR) == 0) {
                        if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
                                cmn_err(CE_NOTE, "Skipping PR=0 pdte: 0x%"
                                    PRIx64, *pdtep);
                        }
                        return (NULL);
                }
                pgtable_pa_4K = AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_ADDR);
        }

        return (amd_iommu_lookup_pgtable_hash(dp->d_domainid, pgtable_pa_4K));
}

static amd_iommu_page_table_t *
amd_iommu_alloc_from_freelist(void)
{
        int i;
        uint64_t *pte_array;
        amd_iommu_page_table_t *pt;

        if (amd_iommu_no_pgtable_freelist == 1)
                return (NULL);

        if (amd_iommu_pgtable_freelist.f_count == 0)
                return (NULL);

        pt = amd_iommu_pgtable_freelist.f_list;
        amd_iommu_pgtable_freelist.f_list = pt->pt_next;
        amd_iommu_pgtable_freelist.f_count--;

        pte_array = pt->pt_pgtblva;
        for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) {
                ASSERT(pt->pt_pte_ref[i] == 0);
                ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]),
                    AMD_IOMMU_PTDE_PR)  == 0);
        }

        return (pt);
}

static int
amd_iommu_alloc_pgtable(amd_iommu_t *iommu, domain_id_t domainid,
    const char *path, amd_iommu_page_table_t **ptp, int km_flags)
{
        int err;
        uint_t ncookies;
        amd_iommu_page_table_t *pt;
        dev_info_t *idip = iommu->aiomt_dip;
        const char *driver = ddi_driver_name(idip);
        int instance = ddi_get_instance(idip);
        const char *f = "amd_iommu_alloc_pgtable";

        *ptp = NULL;

        pt = amd_iommu_alloc_from_freelist();
        if (pt)
                goto init_pgtable;

        pt = kmem_zalloc(sizeof (amd_iommu_page_table_t), km_flags);
        if (pt == NULL)
                return (DDI_DMA_NORESOURCES);

        /*
         * Each page table is 4K in size
         */
        pt->pt_mem_reqsz = AMD_IOMMU_PGTABLE_SZ;

        /*
         * Alloc a DMA handle. Use the IOMMU dip as we want this DMA
         * to *not* enter the IOMMU - no recursive entrance.
         */
        err = ddi_dma_alloc_handle(idip, &amd_iommu_pgtable_dma_attr,
            km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
            NULL, &pt->pt_dma_hdl);
        if (err != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path = %s. "
                    "Cannot alloc DMA handle for IO Page Table",
                    f, driver, instance, domainid, path);
                kmem_free(pt, sizeof (amd_iommu_page_table_t));
                return (err == DDI_DMA_NORESOURCES ? err : DDI_DMA_NOMAPPING);
        }

        /*
         * Alloc memory for IO Page Table.
         * XXX remove size_t cast kludge
         */
        err = ddi_dma_mem_alloc(pt->pt_dma_hdl, pt->pt_mem_reqsz,
            &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED,
            km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
            NULL, (caddr_t *)&pt->pt_pgtblva,
            (size_t *)&pt->pt_mem_realsz, &pt->pt_mem_hdl);
        if (err != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. "
                    "Cannot allocate DMA memory for IO Page table",
                    f, driver, instance, domainid, path);
                ddi_dma_free_handle(&pt->pt_dma_hdl);
                kmem_free(pt, sizeof (amd_iommu_page_table_t));
                return (DDI_DMA_NORESOURCES);
        }

        /*
         * The Page table DMA VA must be 4K aligned and
         * size >= than requested memory.
         *
         */
        ASSERT(((uint64_t)(uintptr_t)pt->pt_pgtblva & AMD_IOMMU_PGTABLE_ALIGN)
            == 0);
        ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz);

        /*
         * Now bind the handle
         */
        err = ddi_dma_addr_bind_handle(pt->pt_dma_hdl, NULL,
            (caddr_t)pt->pt_pgtblva, pt->pt_mem_realsz,
            DDI_DMA_READ | DDI_DMA_CONSISTENT,
            km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
            NULL, &pt->pt_cookie, &ncookies);
        if (err != DDI_DMA_MAPPED) {
                cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. "
                    "Cannot bind memory for DMA to IO Page Tables. "
                    "bufrealsz=%p",
                    f, driver, instance, domainid, path,
                    (void *)(uintptr_t)pt->pt_mem_realsz);
                ddi_dma_mem_free(&pt->pt_mem_hdl);
                ddi_dma_free_handle(&pt->pt_dma_hdl);
                kmem_free(pt, sizeof (amd_iommu_page_table_t));
                return (err == DDI_DMA_PARTIAL_MAP ? DDI_DMA_NOMAPPING :
                    err);
        }

        /*
         * We assume the DMA engine on the IOMMU is capable of handling the
         * whole page table in a single cookie. If not and multiple cookies
         * are needed we fail.
         */
        if (ncookies != 1) {
                cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path=%s "
                    "Cannot handle multiple "
                    "cookies for DMA to IO page Table, #cookies=%u",
                    f, driver, instance, domainid, path, ncookies);
                (void) ddi_dma_unbind_handle(pt->pt_dma_hdl);
                ddi_dma_mem_free(&pt->pt_mem_hdl);
                ddi_dma_free_handle(&pt->pt_dma_hdl);
                kmem_free(pt, sizeof (amd_iommu_page_table_t));
                return (DDI_DMA_NOMAPPING);
        }

init_pgtable:
        /*
         * The address in the cookie must be 4K aligned and >= table size
         */
        ASSERT(pt->pt_cookie.dmac_cookie_addr != (uintptr_t)NULL);
        ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
        ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_realsz);
        ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_reqsz);
        ASSERT(pt->pt_mem_reqsz >= AMD_IOMMU_PGTABLE_SIZE);
        ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz);
        ASSERT(pt->pt_pgtblva);

        pt->pt_domainid = AMD_IOMMU_INVALID_DOMAIN;
        pt->pt_level = 0x7;
        pt->pt_index = 0;
        pt->pt_ref = 0;
        pt->pt_next = NULL;
        pt->pt_prev = NULL;
        pt->pt_parent = NULL;

        bzero(pt->pt_pgtblva, pt->pt_mem_realsz);
        SYNC_FORDEV(pt->pt_dma_hdl);

        amd_iommu_insert_pgtable_hash(pt);

        *ptp = pt;

        return (DDI_SUCCESS);
}

static int
amd_iommu_move_to_freelist(amd_iommu_page_table_t *pt)
{
        if (amd_iommu_no_pgtable_freelist == 1)
                return (DDI_FAILURE);

        if (amd_iommu_pgtable_freelist.f_count ==
            AMD_IOMMU_PGTABLE_FREELIST_MAX)
                return (DDI_FAILURE);

        pt->pt_next = amd_iommu_pgtable_freelist.f_list;
        amd_iommu_pgtable_freelist.f_list = pt;
        amd_iommu_pgtable_freelist.f_count++;

        return (DDI_SUCCESS);
}

static void
amd_iommu_free_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *pt)
{
        int i;
        uint64_t *pte_array;
        dev_info_t *dip = iommu->aiomt_dip;
        int instance = ddi_get_instance(dip);
        const char *driver = ddi_driver_name(dip);
        const char *f = "amd_iommu_free_pgtable";

        ASSERT(pt->pt_ref == 0);

        amd_iommu_remove_pgtable_hash(pt);

        pte_array = pt->pt_pgtblva;
        for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) {
                ASSERT(pt->pt_pte_ref[i] == 0);
                ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]),
                    AMD_IOMMU_PTDE_PR)  == 0);
        }

        if (amd_iommu_move_to_freelist(pt) == DDI_SUCCESS)
                return;

        /* Unbind the handle */
        if (ddi_dma_unbind_handle(pt->pt_dma_hdl) != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: %s%d: idx=%d, domainid=%d. "
                    "Failed to unbind handle: %p for IOMMU Page Table",
                    f, driver, instance, iommu->aiomt_idx, pt->pt_domainid,
                    (void *)pt->pt_dma_hdl);
        }
        /* Free the table memory allocated for DMA */
        ddi_dma_mem_free(&pt->pt_mem_hdl);

        /* Free the DMA handle */
        ddi_dma_free_handle(&pt->pt_dma_hdl);

        kmem_free(pt, sizeof (amd_iommu_page_table_t));

}

static int
init_pde(amd_iommu_page_table_t *ppt, amd_iommu_page_table_t *pt)
{
        uint64_t *pdep = &(ppt->pt_pgtblva[pt->pt_index]);
        uint64_t next_pgtable_pa_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12;

        /* nothing to set. PDE is already set */
        if (AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1) {
                ASSERT(PT_REF_VALID(ppt));
                ASSERT(PT_REF_VALID(pt));
                ASSERT(ppt->pt_pte_ref[pt->pt_index] == 0);
                ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_ADDR)
                    == next_pgtable_pa_4K);
                return (DDI_SUCCESS);
        }

        ppt->pt_ref++;
        ASSERT(PT_REF_VALID(ppt));

        /* Page Directories are always RW */
        AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IW, 1);
        AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IR, 1);
        AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_ADDR,
            next_pgtable_pa_4K);
        pt->pt_parent = ppt;
        AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_NXT_LVL,
            pt->pt_level);
        ppt->pt_pte_ref[pt->pt_index] = 0;
        AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_PR, 1);
        SYNC_FORDEV(ppt->pt_dma_hdl);
        ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1);

        return (DDI_SUCCESS);
}

static int
init_pte(amd_iommu_page_table_t *pt, uint64_t pa, uint16_t index,
    struct ddi_dma_req *dmareq)
{
        uint64_t *ptep = &(pt->pt_pgtblva[index]);
        uint64_t pa_4K = pa >> 12;
        int R;
        int W;

        /* nothing to set if PTE is already set */
        if (AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1) {
                /*
                 * Adjust current permissions
                 * DDI_DMA_WRITE means direction of DMA is MEM -> I/O
                 * so that requires Memory READ permissions i.e. sense
                 * is inverted.
                 * Note: either or both of DD_DMA_READ/WRITE may be set
                 */
                if (amd_iommu_no_RW_perms == 0) {
                        R = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IR);
                        W = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IW);
                        if (R == 0 && ((dmareq->dmar_flags & DDI_DMA_WRITE) ||
                            (dmareq->dmar_flags & DDI_DMA_RDWR))) {
                                AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
                        }
                        if (W  == 0 && ((dmareq->dmar_flags & DDI_DMA_READ) ||
                            (dmareq->dmar_flags & DDI_DMA_RDWR))) {
                                AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
                        }
                }
                ASSERT(PT_REF_VALID(pt));
                pt->pt_pte_ref[index]++;
                ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR)
                    == pa_4K);
                return (DDI_SUCCESS);
        }

        pt->pt_ref++;
        ASSERT(PT_REF_VALID(pt));

        /* see comment above about inverting sense of RD/WR */
        if (amd_iommu_no_RW_perms == 0) {
                AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 0);
                AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 0);
                if (dmareq->dmar_flags & DDI_DMA_RDWR) {
                        AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
                        AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
                } else {
                        if (dmareq->dmar_flags & DDI_DMA_WRITE) {
                                AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
                        }
                        if (dmareq->dmar_flags & DDI_DMA_READ) {
                                AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
                        }
                }
        } else {
                AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
                AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
        }

        /* TODO what is correct for FC and U */
        AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_FC, 0);
        AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_U, 0);
        AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_ADDR, pa_4K);
        AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_NXT_LVL, 0);
        ASSERT(pt->pt_pte_ref[index] == 0);
        pt->pt_pte_ref[index] = 1;
        AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_PR, 1);
        SYNC_FORDEV(pt->pt_dma_hdl);
        ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1);

        return (DDI_SUCCESS);
}


static void
init_pt(amd_iommu_page_table_t *pt, amd_iommu_domain_t *dp,
    int level, uint16_t index)
{
        ASSERT(dp);

        if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
                dp->d_pgtable_root_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12;
        } else {
                ASSERT(level >= 1 && level < AMD_IOMMU_PGTABLE_MAXLEVEL);
        }

        pt->pt_domainid = dp->d_domainid;
        pt->pt_level = level;
        pt->pt_index = index;
}

static int
amd_iommu_setup_1_pgtable(amd_iommu_t *iommu, dev_info_t *rdip,
    struct ddi_dma_req *dmareq,
    domain_id_t domainid, amd_iommu_domain_t *dp,
    amd_iommu_page_table_t *ppt,
    uint16_t index, int level, uint64_t va, uint64_t pa,
    amd_iommu_page_table_t **ptp,  uint16_t *next_idxp, const char *path,
    int km_flags)
{
        int error;
        amd_iommu_page_table_t *pt;
        const char *driver = ddi_driver_name(rdip);
        int instance = ddi_get_instance(rdip);
        const char *f = "amd_iommu_setup_1_pgtable";

        *ptp = NULL;
        *next_idxp = 0;
        error = DDI_SUCCESS;

        ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL);

        ASSERT(dp);
        if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
                ASSERT(ppt == NULL);
                ASSERT(index == 0);
        } else {
                ASSERT(ppt);
        }

        /* Check if page table is already allocated */
        if (pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index)) {
                ASSERT(pt->pt_domainid == domainid);
                ASSERT(pt->pt_level == level);
                ASSERT(pt->pt_index == index);
                goto out;
        }

        if ((error = amd_iommu_alloc_pgtable(iommu, domainid, path, &pt,
            km_flags)) != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: %s%d: idx = %u, domainid = %d, va = %p "
                    "path = %s", f, driver, instance, iommu->aiomt_idx,
                    domainid, (void *)(uintptr_t)va, path);
                return (error);
        }

        ASSERT(dp->d_domainid == domainid);

        init_pt(pt, dp, level, index);

out:
        if (level != AMD_IOMMU_PGTABLE_MAXLEVEL) {
                error = init_pde(ppt, pt);
        }

        if (level == 1) {
                ASSERT(error == DDI_SUCCESS);
                error = init_pte(pt, pa, AMD_IOMMU_VA_BITS(va, level), dmareq);
        } else {
                *next_idxp = AMD_IOMMU_VA_BITS(va, level);
                *ptp = pt;
        }

        return (error);
}

typedef enum {
        PDTE_NOT_TORN = 0x1,
        PDTE_TORN_DOWN = 0x2,
        PGTABLE_TORN_DOWN = 0x4
} pdte_tear_t;

static pdte_tear_t
amd_iommu_teardown_pdte(amd_iommu_t *iommu,
    amd_iommu_page_table_t *pt, int index)
{
        uint8_t next_level;
        pdte_tear_t retval;
        uint64_t *ptdep = &(pt->pt_pgtblva[index]);

        next_level = AMD_IOMMU_REG_GET64(ptdep,
            AMD_IOMMU_PTDE_NXT_LVL);

        if (AMD_IOMMU_REG_GET64(ptdep, AMD_IOMMU_PTDE_PR) == 1) {
                if (pt->pt_level == 1) {
                        ASSERT(next_level == 0);
                        /* PTE */
                        pt->pt_pte_ref[index]--;
                        if (pt->pt_pte_ref[index] != 0) {
                                return (PDTE_NOT_TORN);
                        }
                } else {
                        ASSERT(next_level != 0 && next_level != 7);
                }
                ASSERT(pt->pt_pte_ref[index] == 0);
                ASSERT(PT_REF_VALID(pt));

                AMD_IOMMU_REG_SET64(ptdep, AMD_IOMMU_PTDE_PR, 0);
                SYNC_FORDEV(pt->pt_dma_hdl);
                ASSERT(AMD_IOMMU_REG_GET64(ptdep,
                    AMD_IOMMU_PTDE_PR) == 0);
                pt->pt_ref--;
                ASSERT(PT_REF_VALID(pt));
                retval = PDTE_TORN_DOWN;
        } else {
                ASSERT(0);
                ASSERT(pt->pt_pte_ref[index] == 0);
                ASSERT(PT_REF_VALID(pt));
                retval = PDTE_NOT_TORN;
        }

        if (pt->pt_ref == 0) {
                amd_iommu_free_pgtable(iommu, pt);
                return (PGTABLE_TORN_DOWN);
        }

        return (retval);
}

static int
amd_iommu_create_pgtables(amd_iommu_t *iommu, dev_info_t *rdip,
    struct ddi_dma_req *dmareq, uint64_t va,
    uint64_t pa, uint16_t deviceid, domain_id_t domainid,
    amd_iommu_domain_t *dp, const char *path, int km_flags)
{
        int level;
        uint16_t index;
        uint16_t next_idx;
        amd_iommu_page_table_t *pt;
        amd_iommu_page_table_t *ppt;
        int error;
        const char *driver = ddi_driver_name(rdip);
        int instance = ddi_get_instance(rdip);
        const char *f = "amd_iommu_create_pgtables";

        if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
                cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, "
                    "deviceid = %u, va = %p, pa = %p, path = %s",
                    f, driver, instance,
                    iommu->aiomt_idx, domainid, deviceid,
                    (void *)(uintptr_t)va,
                    (void *)(uintptr_t)pa, path);
        }

        if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) {
                /* No need for pagetables. Just set up device table entry */
                goto passthru;
        }

        index = 0;
        ppt = NULL;
        for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0;
            level--, pt = NULL, next_idx = 0) {
                if ((error = amd_iommu_setup_1_pgtable(iommu, rdip, dmareq,
                    domainid, dp, ppt, index, level, va, pa, &pt,
                    &next_idx, path, km_flags)) != DDI_SUCCESS) {
                        cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, "
                            "deviceid=%u, va= %p, pa = %p, Failed to setup "
                            "page table(s) at level = %d, path = %s.",
                            f, driver, instance, iommu->aiomt_idx,
                            domainid, deviceid, (void *)(uintptr_t)va,
                            (void *)(uintptr_t)pa, level, path);
                        return (error);
                }

                if (level > 1) {
                        ASSERT(pt);
                        ASSERT(pt->pt_domainid == domainid);
                        ppt = pt;
                        index = next_idx;
                } else {
                        ASSERT(level == 1);
                        ASSERT(pt == NULL);
                        ASSERT(next_idx == 0);
                        ppt = NULL;
                        index = 0;
                }
        }

passthru:
        if ((error = amd_iommu_set_devtbl_entry(iommu, rdip, domainid, deviceid,
            dp, path)) != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, deviceid=%u, "
                    "domainid=%d."
                    "Failed to set device table entry for path %s.",
                    f, driver, instance,
                    iommu->aiomt_idx, (void *)rdip, deviceid, domainid, path);
                return (error);
        }

        SYNC_FORDEV(iommu->aiomt_dmahdl);

        return (DDI_SUCCESS);
}

static int
amd_iommu_destroy_pgtables(amd_iommu_t *iommu, dev_info_t *rdip,
    uint64_t pageva, uint16_t deviceid, domain_id_t domainid,
    amd_iommu_domain_t *dp, map_type_t type, int *domain_freed, char *path)
{
        int level;
        int flags;
        amd_iommu_cmdargs_t cmdargs = {0};
        uint16_t index;
        uint16_t prev_index;
        amd_iommu_page_table_t *pt;
        amd_iommu_page_table_t *ppt;
        pdte_tear_t retval;
        int tear_level;
        int invalidate_pte;
        int invalidate_pde;
        int error = DDI_FAILURE;
        const char *driver = ddi_driver_name(iommu->aiomt_dip);
        int instance = ddi_get_instance(iommu->aiomt_dip);
        const char *f = "amd_iommu_destroy_pgtables";

        tear_level = -1;
        if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
                cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, "
                    "deviceid = %u, va = %p, path = %s",
                    f, driver, instance,
                    iommu->aiomt_idx, domainid, deviceid,
                    (void *)(uintptr_t)pageva, path);
        }

        if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) {
                /*
                 * there are no pagetables for the passthru domain.
                 * Just the device table entry
                 */
                error = DDI_SUCCESS;
                goto passthru;
        }

        ppt = NULL;
        index = 0;
        for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0; level--) {
                pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index);
                if (pt) {
                        ppt = pt;
                        index = AMD_IOMMU_VA_BITS(pageva, level);
                        continue;
                }
                break;
        }

        if (level == 0) {
                uint64_t *ptep;
                uint64_t pa_4K;

                ASSERT(pt);
                ASSERT(pt == ppt);
                ASSERT(pt->pt_domainid == dp->d_domainid);

                ptep = &(pt->pt_pgtblva[index]);

                pa_4K = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR);
                if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
                        ASSERT(pageva == (pa_4K << MMU_PAGESHIFT));
                }
        }

        invalidate_pde = 0;
        invalidate_pte = 0;
        for (++level; level <= AMD_IOMMU_PGTABLE_MAXLEVEL; level++) {
                prev_index = pt->pt_index;
                ppt = pt->pt_parent;
                retval = amd_iommu_teardown_pdte(iommu, pt, index);
                switch (retval) {
                        case PDTE_NOT_TORN:
                                goto invalidate;
                        case PDTE_TORN_DOWN:
                                invalidate_pte = 1;
                                goto invalidate;
                        case PGTABLE_TORN_DOWN:
                                invalidate_pte = 1;
                                invalidate_pde = 1;
                                tear_level = level;
                                break;
                }
                index = prev_index;
                pt = ppt;
        }

invalidate:
        /*
         * Now teardown the IOMMU HW caches if applicable
         */
        if (invalidate_pte) {
                cmdargs.ca_domainid = (uint16_t)domainid;
                if (amd_iommu_pageva_inval_all) {
                        cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
                        flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
                            AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
                } else if (invalidate_pde) {
                        cmdargs.ca_addr =
                            (uintptr_t)AMD_IOMMU_VA_INVAL(pageva, tear_level);
                        flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
                            AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
                } else {
                        cmdargs.ca_addr = (uintptr_t)pageva;
                        flags = 0;
                }
                if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
                    &cmdargs, flags, 0) != DDI_SUCCESS) {
                        cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, "
                            "rdip=%p. Failed to invalidate IOMMU HW cache "
                            "for %s", f, driver, instance,
                            iommu->aiomt_idx, domainid, (void *)rdip, path);
                        error = DDI_FAILURE;
                        goto out;
                }
        }

passthru:
        if (tear_level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
                error = amd_iommu_clear_devtbl_entry(iommu, rdip, domainid,
                    deviceid, dp, domain_freed, path);
        } else {
                error = DDI_SUCCESS;
        }

out:
        SYNC_FORDEV(iommu->aiomt_dmahdl);

        return (error);
}

static int
cvt_bind_error(int error)
{
        switch (error) {
        case DDI_DMA_MAPPED:
        case DDI_DMA_PARTIAL_MAP:
        case DDI_DMA_NORESOURCES:
        case DDI_DMA_NOMAPPING:
                break;
        default:
                cmn_err(CE_PANIC, "Unsupported error code: %d", error);
                /*NOTREACHED*/
        }
        return (error);
}

int
amd_iommu_map_pa2va(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
    struct ddi_dma_req *dmareq, uint64_t start_pa, uint64_t pa_sz,
    map_type_t type, uint64_t *start_vap, int km_flags)
{
        pfn_t pfn_start;
        pfn_t pfn_end;
        pfn_t pfn;
        int alias;
        int32_t deviceid;
        domain_id_t domainid;
        amd_iommu_domain_t *dp;
        uint64_t end_pa;
        uint64_t start_va;
        uint64_t end_va;
        uint64_t pg_start;
        uint64_t pg_end;
        uint64_t pg;
        uint64_t va_sz;
        char *path;
        int error = DDI_DMA_NOMAPPING;
        const char *driver = ddi_driver_name(iommu->aiomt_dip);
        int instance = ddi_get_instance(iommu->aiomt_dip);
        const char *f = "amd_iommu_map_pa2va";

        ASSERT(pa_sz != 0);

        *start_vap = 0;

        ASSERT(rdip);

        path = kmem_alloc(MAXPATHLEN, km_flags);
        if (path == NULL) {
                error = DDI_DMA_NORESOURCES;
                goto out;
        }
        (void) ddi_pathname(rdip, path);

        /*
         * First get deviceid
         */
        if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
            != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
                    "Failed to get device ID for %s.", f, driver, instance,
                    iommu->aiomt_idx, (void *)rdip, path);
                error = DDI_DMA_NOMAPPING;
                goto out;
        }

        /*
         * Next get the domain for this rdip
         */
        if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path)
            != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. "
                    "Failed to get domain.", f, driver, instance,
                    iommu->aiomt_idx, (void *)rdip, path);
                error = DDI_DMA_NOMAPPING;
                goto out;
        }

        dp = amd_iommu_lookup_domain(iommu, domainid, type, km_flags);
        if (dp == NULL) {
                cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. "
                    "Failed to get device ID for %s.", f, driver, instance,
                    iommu->aiomt_idx, domainid, (void *)rdip, path);
                error = DDI_DMA_NORESOURCES;
                goto out;
        }

        ASSERT(dp->d_domainid == domainid);

        pfn_start = start_pa >> MMU_PAGESHIFT;

        if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
                cmn_err(CE_NOTE, "pa = %p, pfn_new = %p, pfn_start = %p, "
                    "pgshift = %d",
                    (void *)(uintptr_t)start_pa,
                    (void *)(uintptr_t)(start_pa >> MMU_PAGESHIFT),
                    (void *)(uintptr_t)pfn_start, MMU_PAGESHIFT);
        }

        end_pa = start_pa + pa_sz - 1;
        pfn_end = end_pa >> MMU_PAGESHIFT;

        if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
                start_va = start_pa;
                end_va = end_pa;
                va_sz = pa_sz;
                *start_vap = start_va;
        } else {
                va_sz = mmu_ptob(pfn_end - pfn_start + 1);
                start_va = (uintptr_t)vmem_xalloc(dp->d_vmem, va_sz,
                    MAX(attrp->dma_attr_align, MMU_PAGESIZE),
                    0,
                    attrp->dma_attr_seg + 1,
                    (void *)(uintptr_t)attrp->dma_attr_addr_lo,
                    (void *)(uintptr_t)MIN((attrp->dma_attr_addr_hi + 1),
                    AMD_IOMMU_SIZE_4G), /* XXX rollover */
                    km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
                if (start_va == 0) {
                        cmn_err(CE_WARN, "%s: No VA resources",
                            amd_iommu_modname);
                        error = DDI_DMA_NORESOURCES;
                        goto out;
                }
                ASSERT((start_va & MMU_PAGEOFFSET) == 0);
                end_va = start_va + va_sz - 1;
                *start_vap = start_va + (start_pa & MMU_PAGEOFFSET);
        }

        pg_start = start_va >> MMU_PAGESHIFT;
        pg_end = end_va >> MMU_PAGESHIFT;

        pg = pg_start;
        for (pfn = pfn_start; pfn <= pfn_end; pfn++, pg++) {

                if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
                        cmn_err(CE_NOTE, "%s: attempting to create page tables "
                            "for pfn = %p, va = %p, path = %s",
                            f, (void *)(uintptr_t)(pfn << MMU_PAGESHIFT),
                            (void *)(uintptr_t)(pg << MMU_PAGESHIFT), path);

                }

                if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
                        ASSERT(pfn == pg);
                }

                if ((error = amd_iommu_create_pgtables(iommu, rdip, dmareq,
                    pg << MMU_PAGESHIFT,
                    pfn << MMU_PAGESHIFT, deviceid, domainid, dp, path,
                    km_flags)) != DDI_SUCCESS) {
                        cmn_err(CE_WARN, "Failed to create_pgtables");
                        goto out;
                }

                if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
                        cmn_err(CE_NOTE, "%s: successfully created page tables "
                            "for pfn = %p, vapg = %p, path = %s",
                            f, (void *)(uintptr_t)pfn,
                            (void *)(uintptr_t)pg, path);
                }

        }
        ASSERT(pg == pg_end + 1);


        if (amd_iommu_debug & AMD_IOMMU_DEBUG_PA2VA) {
                cmn_err(CE_NOTE, "pa=%p, va=%p",
                    (void *)(uintptr_t)start_pa,
                    (void *)(uintptr_t)(*start_vap));
        }
        error = DDI_DMA_MAPPED;

out:
        kmem_free(path, MAXPATHLEN);
        return (cvt_bind_error(error));
}

int
amd_iommu_unmap_va(amd_iommu_t *iommu, dev_info_t *rdip, uint64_t start_va,
    uint64_t va_sz, map_type_t type)
{
        uint64_t end_va;
        uint64_t pg_start;
        uint64_t pg_end;
        uint64_t pg;
        uint64_t actual_sz;
        char *path;
        int pathfree;
        int alias;
        int32_t deviceid;
        domain_id_t domainid;
        amd_iommu_domain_t *dp;
        int error;
        int domain_freed;
        const char *driver = ddi_driver_name(iommu->aiomt_dip);
        int instance = ddi_get_instance(iommu->aiomt_dip);
        const char *f = "amd_iommu_unmap_va";

        if (amd_iommu_no_unmap)
                return (DDI_SUCCESS);

        path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
        if (path) {
                (void) ddi_pathname(rdip, path);
                pathfree = 1;
        } else {
                pathfree = 0;
                path = "<path-mem-alloc-failed>";
        }

        /*
         * First get deviceid
         */
        if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
            != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
                    "Failed to get device ID for %s.", f, driver, instance,
                    iommu->aiomt_idx, (void *)rdip, path);
                error = DDI_FAILURE;
                goto out;
        }

        /*
         * Next get the domain for this rdip
         */
        if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path)
            != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. "
                    "Failed to get domain.", f, driver, instance,
                    iommu->aiomt_idx, (void *)rdip, path);
                error = DDI_FAILURE;
                goto out;
        }

        /* should never result in domain allocation/vmem_create */
        dp = amd_iommu_lookup_domain(iommu, domainid, AMD_IOMMU_INVALID_MAP,
            KM_NOSLEEP);
        if (dp == NULL) {
                cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. "
                    "Failed to get device ID for %s.", f, driver, instance,
                    iommu->aiomt_idx, domainid, (void *)rdip, path);
                error = DDI_FAILURE;
                goto out;
        }

        ASSERT(dp->d_domainid == domainid);

        pg_start = start_va >> MMU_PAGESHIFT;
        end_va = start_va + va_sz - 1;
        pg_end = end_va >> MMU_PAGESHIFT;
        actual_sz = (pg_end - pg_start + 1) << MMU_PAGESHIFT;

        domain_freed = 0;
        for (pg = pg_start; pg <= pg_end; pg++) {
                domain_freed = 0;
                if (amd_iommu_destroy_pgtables(iommu, rdip,
                    pg << MMU_PAGESHIFT, deviceid, domainid, dp, type,
                    &domain_freed, path) != DDI_SUCCESS) {
                        error = DDI_FAILURE;
                        goto out;
                }
                if (domain_freed) {
                        ASSERT(pg == pg_end);
                        break;
                }
        }

        /*
         * vmem_xalloc() must be paired with vmem_xfree
         */
        if (type == AMD_IOMMU_VMEM_MAP && !amd_iommu_unity_map) {
                vmem_xfree(dp->d_vmem,
                    (void *)(uintptr_t)(pg_start << MMU_PAGESHIFT), actual_sz);
        }

        if (domain_freed)
                amd_iommu_teardown_domain(iommu, dp);

        error = DDI_SUCCESS;
out:
        if (pathfree)
                kmem_free(path, MAXPATHLEN);
        return (error);
}