/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Portions Copyright (c) 2010, Oracle and/or its affiliates.
 * All rights reserved.
 */

/*
 * immu_regs.c  - File that operates on a IMMU unit's regsiters
 */
#include <sys/dditypes.h>
#include <sys/ddi.h>
#include <sys/archsystm.h>
#include <sys/x86_archext.h>
#include <sys/spl.h>
#include <sys/sysmacros.h>
#include <sys/immu.h>
#include <sys/cpu.h>

#define get_reg32(immu, offset) ddi_get32((immu)->immu_regs_handle, \
                (uint32_t *)(immu->immu_regs_addr + (offset)))
#define get_reg64(immu, offset) ddi_get64((immu)->immu_regs_handle, \
                (uint64_t *)(immu->immu_regs_addr + (offset)))
#define put_reg32(immu, offset, val)    ddi_put32\
                ((immu)->immu_regs_handle, \
                (uint32_t *)(immu->immu_regs_addr + (offset)), val)
#define put_reg64(immu, offset, val)    ddi_put64\
                ((immu)->immu_regs_handle, \
                (uint64_t *)(immu->immu_regs_addr + (offset)), val)

static void immu_regs_inv_wait(immu_inv_wait_t *iwp);

struct immu_flushops immu_regs_flushops = {
        immu_regs_context_fsi,
        immu_regs_context_dsi,
        immu_regs_context_gbl,
        immu_regs_iotlb_psi,
        immu_regs_iotlb_dsi,
        immu_regs_iotlb_gbl,
        immu_regs_inv_wait
};

/*
 * wait max 60s for the hardware completion
 */
#define IMMU_MAX_WAIT_TIME              60000000
#define wait_completion(immu, offset, getf, completion, status) \
{ \
        clock_t stick = ddi_get_lbolt(); \
        clock_t ntick; \
        _NOTE(CONSTCOND) \
        while (1) { \
                status = getf(immu, offset); \
                ntick = ddi_get_lbolt(); \
                if (completion) { \
                        break; \
                } \
                if (ntick - stick >= drv_usectohz(IMMU_MAX_WAIT_TIME)) { \
                        ddi_err(DER_PANIC, NULL, \
                            "immu wait completion time out");           \
                        /*NOTREACHED*/   \
                } else { \
                        ht_pause();\
                }\
        }\
}

static ddi_device_acc_attr_t immu_regs_attr = {
        DDI_DEVICE_ATTR_V0,
        DDI_NEVERSWAP_ACC,
        DDI_STRICTORDER_ACC,
};

/*
 * iotlb_flush()
 *   flush the iotlb cache
 */
static void
iotlb_flush(immu_t *immu, uint_t domain_id,
    uint64_t addr, uint_t am, uint_t hint, immu_iotlb_inv_t type)
{
        uint64_t command = 0, iva = 0;
        uint_t iva_offset, iotlb_offset;
        uint64_t status = 0;

        /* no lock needed since cap and excap fields are RDONLY */
        iva_offset = IMMU_ECAP_GET_IRO(immu->immu_regs_excap);
        iotlb_offset = iva_offset + 8;

        /*
         * prepare drain read/write command
         */
        if (IMMU_CAP_GET_DWD(immu->immu_regs_cap)) {
                command |= TLB_INV_DRAIN_WRITE;
        }

        if (IMMU_CAP_GET_DRD(immu->immu_regs_cap)) {
                command |= TLB_INV_DRAIN_READ;
        }

        /*
         * if the hardward doesn't support page selective invalidation, we
         * will use domain type. Otherwise, use global type
         */
        switch (type) {
        case IOTLB_PSI:
                command |= TLB_INV_PAGE | TLB_INV_IVT |
                    TLB_INV_DID(domain_id);
                iva = addr | am | TLB_IVA_HINT(hint);
                break;
        case IOTLB_DSI:
                command |= TLB_INV_DOMAIN | TLB_INV_IVT |
                    TLB_INV_DID(domain_id);
                break;
        case IOTLB_GLOBAL:
                command |= TLB_INV_GLOBAL | TLB_INV_IVT;
                break;
        default:
                ddi_err(DER_MODE, NULL, "%s: incorrect iotlb flush type",
                    immu->immu_name);
                return;
        }

        if (iva)
                put_reg64(immu, iva_offset, iva);
        put_reg64(immu, iotlb_offset, command);
        wait_completion(immu, iotlb_offset, get_reg64,
            (!(status & TLB_INV_IVT)), status);
}

/*
 * immu_regs_iotlb_psi()
 *   iotlb page specific invalidation
 */
/*ARGSUSED*/
void
immu_regs_iotlb_psi(immu_t *immu, uint_t did, uint64_t dvma, uint_t snpages,
    uint_t hint, immu_inv_wait_t *iwp)
{
        int dvma_am;
        int npg_am;
        int max_am;
        int am;
        uint64_t align;
        int npages_left;
        int npages;
        int i;

        if (!IMMU_CAP_GET_PSI(immu->immu_regs_cap)) {
                immu_regs_iotlb_dsi(immu, did, iwp);
                return;
        }

        max_am = IMMU_CAP_GET_MAMV(immu->immu_regs_cap);

        mutex_enter(&(immu->immu_regs_lock));

        npages_left = snpages;
        for (i = 0; i < immu_flush_gran && npages_left > 0; i++) {
                /* First calculate alignment of DVMA */

                if (dvma == 0) {
                        dvma_am = max_am;
                } else {
                        for (align = (1 << 12), dvma_am = 1;
                            (dvma & align) == 0; align <<= 1, dvma_am++)
                                ;
                        dvma_am--;
                }

                /* Calculate the npg_am */
                npages = npages_left;
                for (npg_am = 0, npages >>= 1; npages; npages >>= 1, npg_am++)
                        ;

                am = MIN(max_am, MIN(dvma_am, npg_am));

                iotlb_flush(immu, did, dvma, am, hint, IOTLB_PSI);

                npages = (1 << am);
                npages_left -= npages;
                dvma += (npages * IMMU_PAGESIZE);
        }

        if (npages_left) {
                iotlb_flush(immu, did, 0, 0, 0, IOTLB_DSI);
        }
        mutex_exit(&(immu->immu_regs_lock));
}

/*
 * immu_regs_iotlb_dsi()
 *      domain specific invalidation
 */
/*ARGSUSED*/
void
immu_regs_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)
{
        mutex_enter(&(immu->immu_regs_lock));
        iotlb_flush(immu, domain_id, 0, 0, 0, IOTLB_DSI);
        mutex_exit(&(immu->immu_regs_lock));
}

/*
 * immu_regs_iotlb_gbl()
 *     global iotlb invalidation
 */
/*ARGSUSED*/
void
immu_regs_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp)
{
        mutex_enter(&(immu->immu_regs_lock));
        iotlb_flush(immu, 0, 0, 0, 0, IOTLB_GLOBAL);
        mutex_exit(&(immu->immu_regs_lock));
}


static int
gaw2agaw(int gaw)
{
        int r, agaw;

        r = (gaw - 12) % 9;

        if (r == 0)
                agaw = gaw;
        else
                agaw = gaw + 9 - r;

        if (agaw > 64)
                agaw = 64;

        return (agaw);
}

/*
 * set_immu_agaw()
 *      calculate agaw for a IOMMU unit
 */
static int
set_agaw(immu_t *immu)
{
        int mgaw, magaw, agaw;
        uint_t bitpos;
        int max_sagaw_mask, sagaw_mask, mask;
        int nlevels;

        /*
         * mgaw is the maximum guest address width.
         * Addresses above this value will be
         * blocked by the IOMMU unit.
         * sagaw is a bitmask that lists all the
         * AGAWs supported by this IOMMU unit.
         */
        mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap);
        sagaw_mask = IMMU_CAP_SAGAW(immu->immu_regs_cap);

        magaw = gaw2agaw(mgaw);

        /*
         * Get bitpos corresponding to
         * magaw
         */

        /*
         * Maximum SAGAW is specified by
         * Vt-d spec.
         */
        max_sagaw_mask = ((1 << 5) - 1);

        if (sagaw_mask > max_sagaw_mask) {
                ddi_err(DER_WARN, NULL, "%s: SAGAW bitmask (%x) "
                    "is larger than maximu SAGAW bitmask "
                    "(%x) specified by Intel Vt-d spec",
                    immu->immu_name, sagaw_mask, max_sagaw_mask);
                return (DDI_FAILURE);
        }

        /*
         * Find a supported AGAW <= magaw
         *
         *      sagaw_mask    bitpos   AGAW (bits)  nlevels
         *      ==============================================
         *      0 0 0 0 1       0       30              2
         *      0 0 0 1 0       1       39              3
         *      0 0 1 0 0       2       48              4
         *      0 1 0 0 0       3       57              5
         *      1 0 0 0 0       4       64(66)          6
         */
        mask = 1;
        nlevels = 0;
        agaw = 0;
        for (mask = 1, bitpos = 0; bitpos < 5;
            bitpos++, mask <<= 1) {
                if (mask & sagaw_mask) {
                        nlevels = bitpos + 2;
                        agaw = 30 + (bitpos * 9);
                }
        }

        /* calculated agaw can be > 64 */
        agaw = (agaw > 64) ? 64 : agaw;

        if (agaw < 30 || agaw > magaw) {
                ddi_err(DER_WARN, NULL, "%s: Calculated AGAW (%d) "
                    "is outside valid limits [30,%d] specified by Vt-d spec "
                    "and magaw",  immu->immu_name, agaw, magaw);
                return (DDI_FAILURE);
        }

        if (nlevels < 2 || nlevels > 6) {
                ddi_err(DER_WARN, NULL, "%s: Calculated pagetable "
                    "level (%d) is outside valid limits [2,6]",
                    immu->immu_name, nlevels);
                return (DDI_FAILURE);
        }

        ddi_err(DER_LOG, NULL, "Calculated pagetable "
            "level (%d), agaw = %d", nlevels, agaw);

        immu->immu_dvma_nlevels = nlevels;
        immu->immu_dvma_agaw = agaw;

        return (DDI_SUCCESS);
}

static int
setup_regs(immu_t *immu)
{
        int error;

        /*
         * This lock may be acquired by the IOMMU interrupt handler
         */
        mutex_init(&(immu->immu_regs_lock), NULL, MUTEX_DRIVER,
            (void *)ipltospl(IMMU_INTR_IPL));

        /*
         * map the register address space
         */
        error = ddi_regs_map_setup(immu->immu_dip, 0,
            (caddr_t *)&(immu->immu_regs_addr), (offset_t)0,
            (offset_t)IMMU_REGSZ, &immu_regs_attr,
            &(immu->immu_regs_handle));

        if (error == DDI_FAILURE) {
                ddi_err(DER_WARN, NULL, "%s: Intel IOMMU register map failed",
                    immu->immu_name);
                mutex_destroy(&(immu->immu_regs_lock));
                return (DDI_FAILURE);
        }

        /*
         * get the register value
         */
        immu->immu_regs_cap = get_reg64(immu, IMMU_REG_CAP);
        immu->immu_regs_excap = get_reg64(immu, IMMU_REG_EXCAP);

        /*
         * if the hardware access is non-coherent, we need clflush
         */
        if (IMMU_ECAP_GET_C(immu->immu_regs_excap)) {
                immu->immu_dvma_coherent = B_TRUE;
        } else {
                immu->immu_dvma_coherent = B_FALSE;
                if (!is_x86_feature(x86_featureset, X86FSET_CLFSH)) {
                        ddi_err(DER_WARN, NULL,
                            "immu unit %s can't be enabled due to "
                            "missing clflush functionality", immu->immu_name);
                        ddi_regs_map_free(&(immu->immu_regs_handle));
                        mutex_destroy(&(immu->immu_regs_lock));
                        return (DDI_FAILURE);
                }
        }

        /* Setup SNP and TM reserved fields */
        immu->immu_SNP_reserved = immu_regs_is_SNP_reserved(immu);
        immu->immu_TM_reserved = immu_regs_is_TM_reserved(immu);

        if (IMMU_ECAP_GET_CH(immu->immu_regs_excap) && immu_use_tm)
                immu->immu_ptemask = PDTE_MASK_TM;
        else
                immu->immu_ptemask = 0;

        /*
         * Check for Mobile 4 series chipset
         */
        if (immu_quirk_mobile4 == B_TRUE &&
            !IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) {
                ddi_err(DER_LOG, NULL,
                    "IMMU: Mobile 4 chipset quirk detected. "
                    "Force-setting RWBF");
                IMMU_CAP_SET_RWBF(immu->immu_regs_cap);
        }

        /*
         * retrieve the maximum number of domains
         */
        immu->immu_max_domains = IMMU_CAP_ND(immu->immu_regs_cap);

        /*
         * calculate the agaw
         */
        if (set_agaw(immu) != DDI_SUCCESS) {
                ddi_regs_map_free(&(immu->immu_regs_handle));
                mutex_destroy(&(immu->immu_regs_lock));
                return (DDI_FAILURE);
        }
        immu->immu_regs_cmdval = 0;

        immu->immu_flushops = &immu_regs_flushops;

        return (DDI_SUCCESS);
}

/* ############### Functions exported ################## */

/*
 * immu_regs_setup()
 *       Setup mappings to a IMMU unit's registers
 *       so that they can be read/written
 */
void
immu_regs_setup(list_t *listp)
{
        int i;
        immu_t *immu;

        for (i = 0; i < IMMU_MAXSEG; i++) {
                immu = list_head(listp);
                for (; immu; immu = list_next(listp, immu)) {
                        /* do your best, continue on error */
                        if (setup_regs(immu) != DDI_SUCCESS) {
                                immu->immu_regs_setup = B_FALSE;
                        } else {
                                immu->immu_regs_setup = B_TRUE;
                        }
                }
        }
}

/*
 * immu_regs_map()
 */
int
immu_regs_resume(immu_t *immu)
{
        int error;

        /*
         * remap the register address space
         */
        error = ddi_regs_map_setup(immu->immu_dip, 0,
            (caddr_t *)&(immu->immu_regs_addr), (offset_t)0,
            (offset_t)IMMU_REGSZ, &immu_regs_attr,
            &(immu->immu_regs_handle));
        if (error != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        immu_regs_set_root_table(immu);

        immu_regs_intr_enable(immu, immu->immu_regs_intr_msi_addr,
            immu->immu_regs_intr_msi_data, immu->immu_regs_intr_uaddr);

        (void) immu_intr_handler((caddr_t)immu, NULL);

        immu_regs_intrmap_enable(immu, immu->immu_intrmap_irta_reg);

        immu_regs_qinv_enable(immu, immu->immu_qinv_reg_value);


        return (error);
}

/*
 * immu_regs_suspend()
 */
void
immu_regs_suspend(immu_t *immu)
{

        immu->immu_intrmap_running = B_FALSE;

        /* Finally, unmap the regs */
        ddi_regs_map_free(&(immu->immu_regs_handle));
}

/*
 * immu_regs_startup()
 *      set a IMMU unit's registers to startup the unit
 */
void
immu_regs_startup(immu_t *immu)
{
        uint32_t status;

        if (immu->immu_regs_setup == B_FALSE) {
                return;
        }

        mutex_enter(&(immu->immu_regs_lock));
        put_reg32(immu, IMMU_REG_GLOBAL_CMD,
            immu->immu_regs_cmdval | IMMU_GCMD_TE);
        wait_completion(immu, IMMU_REG_GLOBAL_STS,
            get_reg32, (status & IMMU_GSTS_TES), status);
        immu->immu_regs_cmdval |= IMMU_GCMD_TE;
        immu->immu_regs_running = B_TRUE;
        mutex_exit(&(immu->immu_regs_lock));

        ddi_err(DER_NOTE, NULL, "%s running", immu->immu_name);
}

/*
 * immu_regs_shutdown()
 *      shutdown a unit
 */
void
immu_regs_shutdown(immu_t *immu)
{
        uint32_t status;

        if (immu->immu_regs_running == B_FALSE) {
                return;
        }

        mutex_enter(&(immu->immu_regs_lock));
        immu->immu_regs_cmdval &= ~IMMU_GCMD_TE;
        put_reg32(immu, IMMU_REG_GLOBAL_CMD,
            immu->immu_regs_cmdval);
        wait_completion(immu, IMMU_REG_GLOBAL_STS,
            get_reg32, !(status & IMMU_GSTS_TES), status);
        immu->immu_regs_running = B_FALSE;
        mutex_exit(&(immu->immu_regs_lock));

        ddi_err(DER_NOTE, NULL, "IOMMU %s stopped", immu->immu_name);
}

/*
 * immu_regs_intr()
 *        Set a IMMU unit regs to setup a IMMU unit's
 *        interrupt handler
 */
void
immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data,
    uint32_t uaddr)
{
        mutex_enter(&(immu->immu_regs_lock));
        immu->immu_regs_intr_msi_addr = msi_addr;
        immu->immu_regs_intr_uaddr = uaddr;
        immu->immu_regs_intr_msi_data = msi_data;
        put_reg32(immu, IMMU_REG_FEVNT_ADDR, msi_addr);
        put_reg32(immu, IMMU_REG_FEVNT_UADDR, uaddr);
        put_reg32(immu, IMMU_REG_FEVNT_DATA, msi_data);
        put_reg32(immu, IMMU_REG_FEVNT_CON, 0);
        mutex_exit(&(immu->immu_regs_lock));
}

/*
 * immu_regs_passthru_supported()
 *       Returns B_TRUE ifi passthru is supported
 */
boolean_t
immu_regs_passthru_supported(immu_t *immu)
{
        if (IMMU_ECAP_GET_PT(immu->immu_regs_excap)) {
                return (B_TRUE);
        }

        ddi_err(DER_WARN, NULL, "Passthru not supported");
        return (B_FALSE);
}

/*
 * immu_regs_is_TM_reserved()
 *       Returns B_TRUE if TM field is reserved
 */
boolean_t
immu_regs_is_TM_reserved(immu_t *immu)
{
        if (IMMU_ECAP_GET_DI(immu->immu_regs_excap) ||
            IMMU_ECAP_GET_CH(immu->immu_regs_excap)) {
                return (B_FALSE);
        }
        return (B_TRUE);
}

/*
 * immu_regs_is_SNP_reserved()
 *       Returns B_TRUE if SNP field is reserved
 */
boolean_t
immu_regs_is_SNP_reserved(immu_t *immu)
{

        return (IMMU_ECAP_GET_SC(immu->immu_regs_excap) ? B_FALSE : B_TRUE);
}

/*
 * immu_regs_wbf_flush()
 *     If required and supported, write to IMMU
 *     unit's regs to flush DMA write buffer(s)
 */
void
immu_regs_wbf_flush(immu_t *immu)
{
        uint32_t status;

        if (!IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) {
                return;
        }

        mutex_enter(&(immu->immu_regs_lock));
        put_reg32(immu, IMMU_REG_GLOBAL_CMD,
            immu->immu_regs_cmdval | IMMU_GCMD_WBF);
        wait_completion(immu, IMMU_REG_GLOBAL_STS,
            get_reg32, (!(status & IMMU_GSTS_WBFS)), status);
        mutex_exit(&(immu->immu_regs_lock));
}

/*
 * immu_regs_cpu_flush()
 *      flush the cpu cache line after CPU memory writes, so
 *      IOMMU can see the writes
 */
void
immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size)
{
        uintptr_t startline, endline;

        if (immu->immu_dvma_coherent == B_TRUE)
                return;

        startline = (uintptr_t)addr  & ~(uintptr_t)(x86_clflush_size - 1);
        endline = ((uintptr_t)addr + size - 1) &
            ~(uintptr_t)(x86_clflush_size - 1);
        while (startline <= endline) {
                clflush_insn((caddr_t)startline);
                startline += x86_clflush_size;
        }

        mfence_insn();
}

/*
 * immu_regs_context_flush()
 *   flush the context cache
 */
static void
context_flush(immu_t *immu, uint8_t function_mask,
    uint16_t sid, uint_t did, immu_context_inv_t type)
{
        uint64_t command = 0;
        uint64_t status;

        /*
         * define the command
         */
        switch (type) {
        case CONTEXT_FSI:
                command |= CCMD_INV_ICC | CCMD_INV_DEVICE
                    | CCMD_INV_DID(did)
                    | CCMD_INV_SID(sid) | CCMD_INV_FM(function_mask);
                break;
        case CONTEXT_DSI:
                command |= CCMD_INV_ICC | CCMD_INV_DOMAIN
                    | CCMD_INV_DID(did);
                break;
        case CONTEXT_GLOBAL:
                command |= CCMD_INV_ICC | CCMD_INV_GLOBAL;
                break;
        default:
                ddi_err(DER_PANIC, NULL,
                    "%s: incorrect context cache flush type",
                    immu->immu_name);
                /*NOTREACHED*/
        }

        mutex_enter(&(immu->immu_regs_lock));
        put_reg64(immu, IMMU_REG_CONTEXT_CMD, command);
        wait_completion(immu, IMMU_REG_CONTEXT_CMD, get_reg64,
            (!(status & CCMD_INV_ICC)), status);
        mutex_exit(&(immu->immu_regs_lock));
}

/*ARGSUSED*/
void
immu_regs_context_fsi(immu_t *immu, uint8_t function_mask,
    uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp)
{
        context_flush(immu, function_mask, source_id, domain_id, CONTEXT_FSI);
}

/*ARGSUSED*/
void
immu_regs_context_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)
{
        context_flush(immu, 0, 0, domain_id, CONTEXT_DSI);
}

/*ARGSUSED*/
void
immu_regs_context_gbl(immu_t *immu, immu_inv_wait_t *iwp)
{
        context_flush(immu, 0, 0, 0, CONTEXT_GLOBAL);
}

/*
 * Nothing to do, all register operations are synchronous.
 */
/*ARGSUSED*/
static void
immu_regs_inv_wait(immu_inv_wait_t *iwp)
{
}

void
immu_regs_set_root_table(immu_t *immu)
{
        uint32_t status;

        mutex_enter(&(immu->immu_regs_lock));
        put_reg64(immu, IMMU_REG_ROOTENTRY,
            immu->immu_ctx_root->hwpg_paddr);
        put_reg32(immu, IMMU_REG_GLOBAL_CMD,
            immu->immu_regs_cmdval | IMMU_GCMD_SRTP);
        wait_completion(immu, IMMU_REG_GLOBAL_STS,
            get_reg32, (status & IMMU_GSTS_RTPS), status);
        mutex_exit(&(immu->immu_regs_lock));
}


/* enable queued invalidation interface */
void
immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value)
{
        uint32_t status;

        if (immu_qinv_enable == B_FALSE)
                return;

        mutex_enter(&immu->immu_regs_lock);
        immu->immu_qinv_reg_value = qinv_reg_value;
        /* Initialize the Invalidation Queue Tail register to zero */
        put_reg64(immu, IMMU_REG_INVAL_QT, 0);

        /* set invalidation queue base address register */
        put_reg64(immu, IMMU_REG_INVAL_QAR, qinv_reg_value);

        /* enable queued invalidation interface */
        put_reg32(immu, IMMU_REG_GLOBAL_CMD,
            immu->immu_regs_cmdval | IMMU_GCMD_QIE);
        wait_completion(immu, IMMU_REG_GLOBAL_STS,
            get_reg32, (status & IMMU_GSTS_QIES), status);
        mutex_exit(&immu->immu_regs_lock);

        immu->immu_regs_cmdval |= IMMU_GCMD_QIE;
        immu->immu_qinv_running = B_TRUE;

}

/* enable interrupt remapping hardware unit */
void
immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg)
{
        uint32_t status;

        if (immu_intrmap_enable == B_FALSE)
                return;

        /* set interrupt remap table pointer */
        mutex_enter(&(immu->immu_regs_lock));
        immu->immu_intrmap_irta_reg = irta_reg;
        put_reg64(immu, IMMU_REG_IRTAR, irta_reg);
        put_reg32(immu, IMMU_REG_GLOBAL_CMD,
            immu->immu_regs_cmdval | IMMU_GCMD_SIRTP);
        wait_completion(immu, IMMU_REG_GLOBAL_STS,
            get_reg32, (status & IMMU_GSTS_IRTPS), status);
        mutex_exit(&(immu->immu_regs_lock));

        /* global flush intr entry cache */
        immu_qinv_intr_global(immu, &immu->immu_intrmap_inv_wait);

        /* enable interrupt remapping */
        mutex_enter(&(immu->immu_regs_lock));
        put_reg32(immu, IMMU_REG_GLOBAL_CMD,
            immu->immu_regs_cmdval | IMMU_GCMD_IRE);
        wait_completion(immu, IMMU_REG_GLOBAL_STS,
            get_reg32, (status & IMMU_GSTS_IRES),
            status);
        immu->immu_regs_cmdval |= IMMU_GCMD_IRE;

        /* set compatible mode */
        put_reg32(immu, IMMU_REG_GLOBAL_CMD,
            immu->immu_regs_cmdval | IMMU_GCMD_CFI);
        wait_completion(immu, IMMU_REG_GLOBAL_STS,
            get_reg32, (status & IMMU_GSTS_CFIS),
            status);
        immu->immu_regs_cmdval |= IMMU_GCMD_CFI;
        mutex_exit(&(immu->immu_regs_lock));

        immu->immu_intrmap_running = B_TRUE;
}

uint64_t
immu_regs_get64(immu_t *immu, uint_t reg)
{
        return (get_reg64(immu, reg));
}

uint32_t
immu_regs_get32(immu_t *immu, uint_t reg)
{
        return (get_reg32(immu, reg));
}

void
immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val)
{
        put_reg64(immu, reg, val);
}

void
immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val)
{
        put_reg32(immu, reg, val);
}