/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Driver to retire/unretire L2/L3 cachelines on panther
 */
#include <sys/types.h>
#include <sys/types32.h>
#include <sys/time.h>
#include <sys/errno.h>
#include <sys/cmn_err.h>
#include <sys/param.h>
#include <sys/modctl.h>
#include <sys/conf.h>
#include <sys/open.h>
#include <sys/stat.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/file.h>
#include <sys/cpuvar.h>
#include <sys/x_call.h>
#include <sys/cheetahregs.h>
#include <sys/mem_cache.h>
#include <sys/mem_cache_ioctl.h>

extern int      retire_l2(uint64_t, uint64_t);
extern int      retire_l2_alternate(uint64_t, uint64_t);
extern int      unretire_l2(uint64_t, uint64_t);
extern int      unretire_l2_alternate(uint64_t, uint64_t);
extern int      retire_l3(uint64_t, uint64_t);
extern int      retire_l3_alternate(uint64_t, uint64_t);
extern int      unretire_l3(uint64_t, uint64_t);
extern int      unretire_l3_alternate(uint64_t, uint64_t);

extern void     retire_l2_start(uint64_t, uint64_t);
extern void     retire_l2_end(uint64_t, uint64_t);
extern void     unretire_l2_start(uint64_t, uint64_t);
extern void     unretire_l2_end(uint64_t, uint64_t);
extern void     retire_l3_start(uint64_t, uint64_t);
extern void     retire_l3_end(uint64_t, uint64_t);
extern void     unretire_l3_start(uint64_t, uint64_t);
extern void     unretire_l3_end(uint64_t, uint64_t);

extern void     get_ecache_dtags_tl1(uint64_t, ch_cpu_logout_t *);
extern void     get_l2_tag_tl1(uint64_t, uint64_t);
extern void     get_l3_tag_tl1(uint64_t, uint64_t);
extern const int _ncpu;

/* Macro for putting 64-bit onto stack as two 32-bit ints */
#define PRTF_64_TO_32(x)        (uint32_t)((x)>>32), (uint32_t)(x)


uint_t l2_flush_retries_done = 0;
int mem_cache_debug = 0x0;
uint64_t pattern = 0;
uint32_t retire_failures = 0;
#ifdef DEBUG
int     inject_anonymous_tag_error = 0;
int32_t last_error_injected_way = 0;
uint8_t last_error_injected_bit = 0;
int32_t last_l3tag_error_injected_way;
uint8_t last_l3tag_error_injected_bit;
int32_t last_l2tag_error_injected_way;
uint8_t last_l2tag_error_injected_bit;
#endif

/* dev_ops and cb_ops entry point function declarations */
static int      mem_cache_attach(dev_info_t *, ddi_attach_cmd_t);
static int      mem_cache_detach(dev_info_t *, ddi_detach_cmd_t);
static int      mem_cache_getinfo(dev_info_t *, ddi_info_cmd_t, void *,
                                void **);
static int      mem_cache_open(dev_t *, int, int, cred_t *);
static int      mem_cache_close(dev_t, int, int, cred_t *);
static int      mem_cache_ioctl_ops(int, int, cache_info_t *);
static int      mem_cache_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);

struct cb_ops mem_cache_cb_ops = {
        mem_cache_open,
        mem_cache_close,
        nodev,
        nodev,
        nodev,                  /* dump */
        nodev,
        nodev,
        mem_cache_ioctl,
        nodev,                  /* devmap */
        nodev,
        ddi_segmap,             /* segmap */
        nochpoll,
        ddi_prop_op,
        NULL,                   /* for STREAMS drivers */
        D_NEW | D_MP            /* driver compatibility flag */
};

static struct dev_ops mem_cache_dev_ops = {
        DEVO_REV,               /* driver build version */
        0,                      /* device reference count */
        mem_cache_getinfo,
        nulldev,
        nulldev,                /* probe */
        mem_cache_attach,
        mem_cache_detach,
        nulldev,                /* reset */
        &mem_cache_cb_ops,
        (struct bus_ops *)NULL,
        nulldev,                        /* power */
        ddi_quiesce_not_needed,         /* quiesce */
};

/*
 * Soft state
 */
struct mem_cache_softc {
        dev_info_t      *dip;
        kmutex_t        mutex;
};
#define getsoftc(inst)  ((struct mem_cache_softc *)ddi_get_soft_state(statep,\
                        (inst)))

/* module configuration stuff */
static void *statep;
extern struct mod_ops mod_driverops;

static struct modldrv modldrv = {
        &mod_driverops,
        "mem_cache_driver (08/01/30) ",
        &mem_cache_dev_ops
};

static struct modlinkage modlinkage = {
        MODREV_1,
        &modldrv,
        0
};

extern const int _ncpu; /* Pull the kernel's global _ncpu definition */

int
_init(void)
{
        int e;

        if (e = ddi_soft_state_init(&statep, sizeof (struct mem_cache_softc),
            MAX_MEM_CACHE_INSTANCES)) {
                return (e);
        }

        if ((e = mod_install(&modlinkage)) != 0)
                ddi_soft_state_fini(&statep);

        return (e);
}

int
_fini(void)
{
        int e;

        if ((e = mod_remove(&modlinkage)) != 0)
                return (e);

        ddi_soft_state_fini(&statep);

        return (DDI_SUCCESS);
}

int
_info(struct modinfo *modinfop)
{
        return (mod_info(&modlinkage, modinfop));
}

/*ARGSUSED*/
static int
mem_cache_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
{
        int     inst;
        int     retval = DDI_SUCCESS;
        struct mem_cache_softc *softc;

        inst = getminor((dev_t)arg);

        switch (cmd) {
        case DDI_INFO_DEVT2DEVINFO:
                if ((softc = getsoftc(inst)) == NULL) {
                        *result = (void *)NULL;
                        retval = DDI_FAILURE;
                } else
                        *result = (void *)softc->dip;
                break;

        case DDI_INFO_DEVT2INSTANCE:
                *result = (void *)((uintptr_t)inst);
                break;

        default:
                retval = DDI_FAILURE;
        }

        return (retval);
}

static int
mem_cache_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
        int inst;
        struct mem_cache_softc *softc = NULL;
        char name[80];

        switch (cmd) {
        case DDI_ATTACH:
                inst = ddi_get_instance(dip);
                if (inst >= MAX_MEM_CACHE_INSTANCES) {
                        cmn_err(CE_WARN, "attach failed, too many instances\n");
                        return (DDI_FAILURE);
                }
                (void) sprintf(name, MEM_CACHE_DRIVER_NAME"%d", inst);
                if (ddi_create_priv_minor_node(dip, name,
                    S_IFCHR,
                    inst,
                    DDI_PSEUDO,
                    0, NULL, "all", 0640) ==
                    DDI_FAILURE) {
                        ddi_remove_minor_node(dip, NULL);
                        return (DDI_FAILURE);
                }

                /* Allocate a soft state structure for this instance */
                if (ddi_soft_state_zalloc(statep, inst) != DDI_SUCCESS) {
                        cmn_err(CE_WARN, " ddi_soft_state_zalloc() failed "
                            "for inst %d\n", inst);
                        goto attach_failed;
                }

                /* Setup soft state */
                softc = getsoftc(inst);
                softc->dip = dip;
                mutex_init(&softc->mutex, NULL, MUTEX_DRIVER, NULL);

                /* Create main environmental node */
                ddi_report_dev(dip);

                return (DDI_SUCCESS);

        case DDI_RESUME:
                return (DDI_SUCCESS);

        default:
                return (DDI_FAILURE);
        }

attach_failed:

        /* Free soft state, if allocated. remove minor node if added earlier */
        if (softc)
                ddi_soft_state_free(statep, inst);

        ddi_remove_minor_node(dip, NULL);

        return (DDI_FAILURE);
}

static int
mem_cache_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
        int inst;
        struct mem_cache_softc *softc;

        switch (cmd) {
        case DDI_DETACH:
                inst = ddi_get_instance(dip);
                if ((softc = getsoftc(inst)) == NULL)
                        return (ENXIO);

                /* Free the soft state and remove minor node added earlier */
                mutex_destroy(&softc->mutex);
                ddi_soft_state_free(statep, inst);
                ddi_remove_minor_node(dip, NULL);
                return (DDI_SUCCESS);

        case DDI_SUSPEND:
                return (DDI_SUCCESS);

        default:
                return (DDI_FAILURE);
        }
}

/*ARGSUSED*/
static int
mem_cache_open(dev_t *devp, int flag, int otyp, cred_t *credp)
{
        int     inst = getminor(*devp);

        return (getsoftc(inst) == NULL ? ENXIO : 0);
}

/*ARGSUSED*/
static int
mem_cache_close(dev_t dev, int flag, int otyp, cred_t *credp)
{
        int     inst = getminor(dev);

        return (getsoftc(inst) == NULL ? ENXIO : 0);
}

static char *tstate_to_desc[] = {
        "Invalid",                      /* 0 */
        "Shared",                       /* 1 */
        "Exclusive",                    /* 2 */
        "Owner",                        /* 3 */
        "Modified",                     /* 4 */
        "NA",                           /* 5 */
        "Owner/Shared",                 /* 6 */
        "Reserved(7)",                  /* 7 */
};

static char *
tag_state_to_desc(uint8_t tagstate)
{
        return (tstate_to_desc[tagstate & CH_ECSTATE_MASK]);
}

void
print_l2_tag(uint64_t tag_addr, uint64_t l2_tag)
{
        uint64_t l2_subaddr;
        uint8_t l2_state;

        l2_subaddr = PN_L2TAG_TO_PA(l2_tag);
        l2_subaddr |= (tag_addr & PN_L2_INDEX_MASK);

        l2_state = (l2_tag & CH_ECSTATE_MASK);
        cmn_err(CE_CONT,
            "PA=0x%08x.%08x E$tag 0x%08x.%08x E$state %s\n",
            PRTF_64_TO_32(l2_subaddr),
            PRTF_64_TO_32(l2_tag),
            tag_state_to_desc(l2_state));
}

void
print_l2cache_line(ch_cpu_logout_t *clop)
{
        uint64_t l2_subaddr;
        int i, offset;
        uint8_t way, l2_state;
        ch_ec_data_t *ecp;


        for (way = 0; way < PN_CACHE_NWAYS; way++) {
                ecp = &clop->clo_data.chd_l2_data[way];
                l2_subaddr = PN_L2TAG_TO_PA(ecp->ec_tag);
                l2_subaddr |= (ecp->ec_idx & PN_L2_INDEX_MASK);

                l2_state = (ecp->ec_tag & CH_ECSTATE_MASK);
                cmn_err(CE_CONT,
                    "\nWAY = %d index = 0x%08x PA=0x%08x.%08x\n"
                    "E$tag 0x%08x.%08x E$state %s",
                    way, (uint32_t)ecp->ec_idx, PRTF_64_TO_32(l2_subaddr),
                    PRTF_64_TO_32(ecp->ec_tag),
                    tag_state_to_desc(l2_state));
                /*
                 * Dump out Ecache subblock data captured.
                 * For Cheetah, we need to compute the ECC for each 16-byte
                 * chunk and compare it with the captured chunk ECC to figure
                 * out which chunk is bad.
                 */
                for (i = 0; i < (CH_ECACHE_SUBBLK_SIZE/16); i++) {
                        ec_data_elm_t *ecdptr;
                        uint64_t d_low, d_high;
                        uint32_t ecc;
                        int l2_data_idx = (i/2);

                        offset = i * 16;
                        ecdptr = &clop->clo_data.chd_l2_data[way].ec_data
                            [l2_data_idx];
                        if ((i & 1) == 0) {
                                ecc = (ecdptr->ec_eccd >> 9) & 0x1ff;
                                d_high = ecdptr->ec_d8[0];
                                d_low  = ecdptr->ec_d8[1];
                        } else {
                                ecc = ecdptr->ec_eccd & 0x1ff;
                                d_high = ecdptr->ec_d8[2];
                                d_low  = ecdptr->ec_d8[3];
                        }

                        cmn_err(CE_CONT,
                            "\nE$Data (0x%02x) 0x%08x.%08x 0x%08x.%08x"
                            " ECC 0x%03x",
                            offset, PRTF_64_TO_32(d_high),
                            PRTF_64_TO_32(d_low), ecc);
                }
        }       /* end of for way loop */
}

void
print_ecache_line(ch_cpu_logout_t *clop)
{
        uint64_t ec_subaddr;
        int i, offset;
        uint8_t way, ec_state;
        ch_ec_data_t *ecp;


        for (way = 0; way < PN_CACHE_NWAYS; way++) {
                ecp = &clop->clo_data.chd_ec_data[way];
                ec_subaddr = PN_L3TAG_TO_PA(ecp->ec_tag);
                ec_subaddr |= (ecp->ec_idx & PN_L3_TAG_RD_MASK);

                ec_state = (ecp->ec_tag & CH_ECSTATE_MASK);
                cmn_err(CE_CONT,
                    "\nWAY = %d index = 0x%08x PA=0x%08x.%08x\n"
                    "E$tag 0x%08x.%08x E$state %s",
                    way, (uint32_t)ecp->ec_idx, PRTF_64_TO_32(ec_subaddr),
                    PRTF_64_TO_32(ecp->ec_tag),
                    tag_state_to_desc(ec_state));
                /*
                 * Dump out Ecache subblock data captured.
                 * For Cheetah, we need to compute the ECC for each 16-byte
                 * chunk and compare it with the captured chunk ECC to figure
                 * out which chunk is bad.
                 */
                for (i = 0; i < (CH_ECACHE_SUBBLK_SIZE/16); i++) {
                        ec_data_elm_t *ecdptr;
                        uint64_t d_low, d_high;
                        uint32_t ecc;
                        int ec_data_idx = (i/2);

                        offset = i * 16;
                        ecdptr =
                            &clop->clo_data.chd_ec_data[way].ec_data
                            [ec_data_idx];
                        if ((i & 1) == 0) {
                                ecc = (ecdptr->ec_eccd >> 9) & 0x1ff;
                                d_high = ecdptr->ec_d8[0];
                                d_low  = ecdptr->ec_d8[1];
                        } else {
                                ecc = ecdptr->ec_eccd & 0x1ff;
                                d_high = ecdptr->ec_d8[2];
                                d_low  = ecdptr->ec_d8[3];
                        }

                        cmn_err(CE_CONT,
                            "\nE$Data (0x%02x) 0x%08x.%08x 0x%08x.%08x"
                            " ECC 0x%03x",
                            offset, PRTF_64_TO_32(d_high),
                            PRTF_64_TO_32(d_low), ecc);
                }
        }
}

static boolean_t
tag_addr_collides(uint64_t tag_addr, cache_id_t type,
    retire_func_t start_of_func, retire_func_t end_of_func)
{
        uint64_t start_paddr, end_paddr;
        char *type_str;

        start_paddr = va_to_pa((void *)start_of_func);
        end_paddr = va_to_pa((void *)end_of_func);
        switch (type) {
                case L2_CACHE_TAG:
                case L2_CACHE_DATA:
                        tag_addr &= PN_L2_INDEX_MASK;
                        start_paddr &= PN_L2_INDEX_MASK;
                        end_paddr &= PN_L2_INDEX_MASK;
                        type_str = "L2:";
                        break;
                case L3_CACHE_TAG:
                case L3_CACHE_DATA:
                        tag_addr &= PN_L3_TAG_RD_MASK;
                        start_paddr &= PN_L3_TAG_RD_MASK;
                        end_paddr &= PN_L3_TAG_RD_MASK;
                        type_str = "L3:";
                        break;
                default:
                        /*
                         * Should never reach here.
                         */
                        ASSERT(0);
                        return (B_FALSE);
        }
        if ((tag_addr > (start_paddr - 0x100)) &&
            (tag_addr < (end_paddr + 0x100))) {
                if (mem_cache_debug & 0x1)
                        cmn_err(CE_CONT,
                            "%s collision detected tag_addr = 0x%08x"
                            " start_paddr = 0x%08x end_paddr = 0x%08x\n",
                            type_str, (uint32_t)tag_addr, (uint32_t)start_paddr,
                            (uint32_t)end_paddr);
                return (B_TRUE);
        }
        else
                return (B_FALSE);
}

static uint64_t
get_tag_addr(cache_info_t *cache_info)
{
        uint64_t tag_addr, scratch;

        switch (cache_info->cache) {
                case L2_CACHE_TAG:
                case L2_CACHE_DATA:
                        tag_addr = (uint64_t)(cache_info->index <<
                            PN_CACHE_LINE_SHIFT);
                        scratch = (uint64_t)(cache_info->way <<
                            PN_L2_WAY_SHIFT);
                        tag_addr |= scratch;
                        tag_addr |= PN_L2_IDX_HW_ECC_EN;
                        break;
                case L3_CACHE_TAG:
                case L3_CACHE_DATA:
                        tag_addr = (uint64_t)(cache_info->index <<
                            PN_CACHE_LINE_SHIFT);
                        scratch = (uint64_t)(cache_info->way <<
                            PN_L3_WAY_SHIFT);
                        tag_addr |= scratch;
                        tag_addr |= PN_L3_IDX_HW_ECC_EN;
                        break;
                default:
                        /*
                         * Should never reach here.
                         */
                        ASSERT(0);
                        return (uint64_t)(0);
        }
        return (tag_addr);
}

static int
mem_cache_ioctl_ops(int cmd, int mode, cache_info_t *cache_info)
{
        int     ret_val = 0;
        uint64_t afar, tag_addr;
        ch_cpu_logout_t clop;
        uint64_t Lxcache_tag_data[PN_CACHE_NWAYS];
        int     i, retire_retry_count;
        cpu_t   *cpu;
        uint64_t tag_data;
        uint8_t state;

        if (cache_info->way >= PN_CACHE_NWAYS)
                return (EINVAL);
        switch (cache_info->cache) {
                case L2_CACHE_TAG:
                case L2_CACHE_DATA:
                        if (cache_info->index >=
                            (PN_L2_SET_SIZE/PN_L2_LINESIZE))
                                return (EINVAL);
                        break;
                case L3_CACHE_TAG:
                case L3_CACHE_DATA:
                        if (cache_info->index >=
                            (PN_L3_SET_SIZE/PN_L3_LINESIZE))
                                return (EINVAL);
                        break;
                default:
                        return (ENOTSUP);
        }
        /*
         * Check if we have a valid cpu ID and that
         * CPU is ONLINE.
         */
        mutex_enter(&cpu_lock);
        cpu = cpu_get(cache_info->cpu_id);
        if ((cpu == NULL) || (!cpu_is_online(cpu))) {
                mutex_exit(&cpu_lock);
                return (EINVAL);
        }
        mutex_exit(&cpu_lock);
        pattern = 0;    /* default value of TAG PA when cacheline is retired. */
        switch (cmd) {
                case MEM_CACHE_RETIRE:
                        tag_addr = get_tag_addr(cache_info);
                        pattern |= PN_ECSTATE_NA;
                        retire_retry_count = 0;
                        affinity_set(cache_info->cpu_id);
                        switch (cache_info->cache) {
                                case L2_CACHE_DATA:
                                case L2_CACHE_TAG:
                                        if ((cache_info->bit & MSB_BIT_MASK) ==
                                            MSB_BIT_MASK)
                                                pattern |= PN_L2TAG_PA_MASK;
retry_l2_retire:
                                        if (tag_addr_collides(tag_addr,
                                            cache_info->cache,
                                            retire_l2_start, retire_l2_end))
                                                ret_val =
                                                    retire_l2_alternate(
                                                    tag_addr, pattern);
                                        else
                                                ret_val = retire_l2(tag_addr,
                                                    pattern);
                                        if (ret_val == 1) {
                                                /*
                                                 * cacheline was in retired
                                                 * STATE already.
                                                 * so return success.
                                                 */
                                                ret_val = 0;
                                        }
                                        if (ret_val < 0) {
                                                cmn_err(CE_WARN,
                "retire_l2() failed. index = 0x%x way %d. Retrying...\n",
                                                    cache_info->index,
                                                    cache_info->way);
                                                if (retire_retry_count >= 2) {
                                                        retire_failures++;
                                                        affinity_clear();
                                                        return (EIO);
                                                }
                                                retire_retry_count++;
                                                goto retry_l2_retire;
                                        }
                                        if (ret_val == 2)
                                                l2_flush_retries_done++;
                        /*
                         * We bind ourself to a CPU and send cross trap to
                         * ourself. On return from xt_one we can rely on the
                         * data in tag_data being filled in. Normally one would
                         * do a xt_sync to make sure that the CPU has completed
                         * the cross trap call xt_one.
                         */
                                        xt_one(cache_info->cpu_id,
                                            (xcfunc_t *)(get_l2_tag_tl1),
                                            tag_addr, (uint64_t)(&tag_data));
                                        state = tag_data & CH_ECSTATE_MASK;
                                        if (state != PN_ECSTATE_NA) {
                                                retire_failures++;
                                                print_l2_tag(tag_addr,
                                                    tag_data);
                                                cmn_err(CE_WARN,
                "L2 RETIRE:failed for index 0x%x way %d. Retrying...\n",
                                                    cache_info->index,
                                                    cache_info->way);
                                                if (retire_retry_count >= 2) {
                                                        retire_failures++;
                                                        affinity_clear();
                                                        return (EIO);
                                                }
                                                retire_retry_count++;
                                                goto retry_l2_retire;
                                        }
                                        break;
                                case L3_CACHE_TAG:
                                case L3_CACHE_DATA:
                                        if ((cache_info->bit & MSB_BIT_MASK) ==
                                            MSB_BIT_MASK)
                                                pattern |= PN_L3TAG_PA_MASK;
                                        if (tag_addr_collides(tag_addr,
                                            cache_info->cache,
                                            retire_l3_start, retire_l3_end))
                                                ret_val =
                                                    retire_l3_alternate(
                                                    tag_addr, pattern);
                                        else
                                                ret_val = retire_l3(tag_addr,
                                                    pattern);
                                        if (ret_val == 1) {
                                                /*
                                                 * cacheline was in retired
                                                 * STATE already.
                                                 * so return success.
                                                 */
                                                ret_val = 0;
                                        }
                                        if (ret_val < 0) {
                                                cmn_err(CE_WARN,
                        "retire_l3() failed. ret_val = %d index = 0x%x\n",
                                                    ret_val,
                                                    cache_info->index);
                                                retire_failures++;
                                                affinity_clear();
                                                return (EIO);
                                        }
                        /*
                         * We bind ourself to a CPU and send cross trap to
                         * ourself. On return from xt_one we can rely on the
                         * data in tag_data being filled in. Normally one would
                         * do a xt_sync to make sure that the CPU has completed
                         * the cross trap call xt_one.
                         */
                                        xt_one(cache_info->cpu_id,
                                            (xcfunc_t *)(get_l3_tag_tl1),
                                            tag_addr, (uint64_t)(&tag_data));
                                        state = tag_data & CH_ECSTATE_MASK;
                                        if (state != PN_ECSTATE_NA) {
                                                cmn_err(CE_WARN,
                                        "L3 RETIRE failed for index 0x%x\n",
                                                    cache_info->index);
                                                retire_failures++;
                                                affinity_clear();
                                                return (EIO);
                                        }

                                        break;
                        }
                        affinity_clear();
                        break;
                case MEM_CACHE_UNRETIRE:
                        tag_addr = get_tag_addr(cache_info);
                        pattern = PN_ECSTATE_INV;
                        affinity_set(cache_info->cpu_id);
                        switch (cache_info->cache) {
                                case L2_CACHE_DATA:
                                case L2_CACHE_TAG:
                        /*
                         * We bind ourself to a CPU and send cross trap to
                         * ourself. On return from xt_one we can rely on the
                         * data in tag_data being filled in. Normally one would
                         * do a xt_sync to make sure that the CPU has completed
                         * the cross trap call xt_one.
                         */
                                        xt_one(cache_info->cpu_id,
                                            (xcfunc_t *)(get_l2_tag_tl1),
                                            tag_addr, (uint64_t)(&tag_data));
                                        state = tag_data & CH_ECSTATE_MASK;
                                        if (state != PN_ECSTATE_NA) {
                                                affinity_clear();
                                                return (EINVAL);
                                        }
                                        if (tag_addr_collides(tag_addr,
                                            cache_info->cache,
                                            unretire_l2_start, unretire_l2_end))
                                                ret_val =
                                                    unretire_l2_alternate(
                                                    tag_addr, pattern);
                                        else
                                                ret_val =
                                                    unretire_l2(tag_addr,
                                                    pattern);
                                        if (ret_val != 0) {
                                                cmn_err(CE_WARN,
                        "unretire_l2() failed. ret_val = %d index = 0x%x\n",
                                                    ret_val,
                                                    cache_info->index);
                                                retire_failures++;
                                                affinity_clear();
                                                return (EIO);
                                        }
                                        break;
                                case L3_CACHE_TAG:
                                case L3_CACHE_DATA:
                        /*
                         * We bind ourself to a CPU and send cross trap to
                         * ourself. On return from xt_one we can rely on the
                         * data in tag_data being filled in. Normally one would
                         * do a xt_sync to make sure that the CPU has completed
                         * the cross trap call xt_one.
                         */
                                        xt_one(cache_info->cpu_id,
                                            (xcfunc_t *)(get_l3_tag_tl1),
                                            tag_addr, (uint64_t)(&tag_data));
                                        state = tag_data & CH_ECSTATE_MASK;
                                        if (state != PN_ECSTATE_NA) {
                                                affinity_clear();
                                                return (EINVAL);
                                        }
                                        if (tag_addr_collides(tag_addr,
                                            cache_info->cache,
                                            unretire_l3_start, unretire_l3_end))
                                                ret_val =
                                                    unretire_l3_alternate(
                                                    tag_addr, pattern);
                                        else
                                                ret_val =
                                                    unretire_l3(tag_addr,
                                                    pattern);
                                        if (ret_val != 0) {
                                                cmn_err(CE_WARN,
                        "unretire_l3() failed. ret_val = %d index = 0x%x\n",
                                                    ret_val,
                                                    cache_info->index);
                                                affinity_clear();
                                                return (EIO);
                                        }
                                        break;
                        }
                        affinity_clear();
                        break;
                case MEM_CACHE_ISRETIRED:
                case MEM_CACHE_STATE:
                        return (ENOTSUP);
                case MEM_CACHE_READ_TAGS:
#ifdef DEBUG
                case MEM_CACHE_READ_ERROR_INJECTED_TAGS:
#endif
                        /*
                         * Read tag and data for all the ways at a given afar
                         */
                        afar = (uint64_t)(cache_info->index
                            << PN_CACHE_LINE_SHIFT);
                        mutex_enter(&cpu_lock);
                        affinity_set(cache_info->cpu_id);
                        pause_cpus(NULL, NULL);
                        mutex_exit(&cpu_lock);
                        /*
                         * We bind ourself to a CPU and send cross trap to
                         * ourself. On return from xt_one we can rely on the
                         * data in clop being filled in. Normally one would
                         * do a xt_sync to make sure that the CPU has completed
                         * the cross trap call xt_one.
                         */
                        xt_one(cache_info->cpu_id,
                            (xcfunc_t *)(get_ecache_dtags_tl1),
                            afar, (uint64_t)(&clop));
                        mutex_enter(&cpu_lock);
                        (void) start_cpus();
                        mutex_exit(&cpu_lock);
                        affinity_clear();
                        switch (cache_info->cache) {
                                case L2_CACHE_TAG:
                                        for (i = 0; i < PN_CACHE_NWAYS; i++) {
                                                Lxcache_tag_data[i] =
                                                    clop.clo_data.chd_l2_data
                                                    [i].ec_tag;
                                        }
#ifdef DEBUG
                                        last_error_injected_bit =
                                            last_l2tag_error_injected_bit;
                                        last_error_injected_way =
                                            last_l2tag_error_injected_way;
#endif
                                        break;
                                case L3_CACHE_TAG:
                                        for (i = 0; i < PN_CACHE_NWAYS; i++) {
                                                Lxcache_tag_data[i] =
                                                    clop.clo_data.chd_ec_data
                                                    [i].ec_tag;
                                        }
#ifdef DEBUG
                                        last_error_injected_bit =
                                            last_l3tag_error_injected_bit;
                                        last_error_injected_way =
                                            last_l3tag_error_injected_way;
#endif
                                        break;
                                default:
                                        return (ENOTSUP);
                        }       /* end if switch(cache) */
#ifdef DEBUG
                        if ((cmd == MEM_CACHE_READ_ERROR_INJECTED_TAGS) &&
                            (inject_anonymous_tag_error == 0) &&
                            (last_error_injected_way >= 0) &&
                            (last_error_injected_way <= 3)) {
                                pattern = ((uint64_t)1 <<
                                    last_error_injected_bit);
                                /*
                                 * If error bit is ECC we need to make sure
                                 * ECC on all all WAYS are corrupted.
                                 */
                                if ((last_error_injected_bit >= 6) &&
                                    (last_error_injected_bit <= 14)) {
                                        for (i = 0; i < PN_CACHE_NWAYS; i++)
                                                Lxcache_tag_data[i] ^=
                                                    pattern;
                                } else
                                        Lxcache_tag_data
                                            [last_error_injected_way] ^=
                                            pattern;
                        }
#endif
                        if (ddi_copyout((caddr_t)Lxcache_tag_data,
                            (caddr_t)cache_info->datap,
                            sizeof (Lxcache_tag_data), mode)
                            != DDI_SUCCESS) {
                                return (EFAULT);
                        }
                        break;  /* end of READ_TAGS */
                default:
                        return (ENOTSUP);
        }       /* end if switch(cmd) */
        return (ret_val);
}

/*ARGSUSED*/
static int
mem_cache_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
                int *rvalp)
{
        int     inst;
        struct mem_cache_softc *softc;
        cache_info_t    cache_info;
        cache_info32_t  cache_info32;
        int     ret_val;
        int     is_panther;

        inst = getminor(dev);
        if ((softc = getsoftc(inst)) == NULL)
                return (ENXIO);

        mutex_enter(&softc->mutex);

#ifdef _MULTI_DATAMODEL
        if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
                if (ddi_copyin((cache_info32_t *)arg, &cache_info32,
                    sizeof (cache_info32), mode) != DDI_SUCCESS) {
                        mutex_exit(&softc->mutex);
                        return (EFAULT);
                }
                cache_info.cache = cache_info32.cache;
                cache_info.index = cache_info32.index;
                cache_info.way = cache_info32.way;
                cache_info.cpu_id = cache_info32.cpu_id;
                cache_info.bit = cache_info32.bit;
                cache_info.datap = (void *)((uint64_t)cache_info32.datap);
        } else
#endif
        if (ddi_copyin((cache_info_t *)arg, &cache_info,
            sizeof (cache_info), mode) != DDI_SUCCESS) {
                mutex_exit(&softc->mutex);
                return (EFAULT);
        }

        if ((cache_info.cpu_id < 0) || (cache_info.cpu_id >= _ncpu)) {
                mutex_exit(&softc->mutex);
                return (EINVAL);
        }
        is_panther = IS_PANTHER(cpunodes[cache_info.cpu_id].implementation);
        if (!is_panther) {
                mutex_exit(&softc->mutex);
                return (ENOTSUP);
        }
        switch (cmd) {
                case MEM_CACHE_RETIRE:
                case MEM_CACHE_UNRETIRE:
                        if ((mode & FWRITE) == 0) {
                                ret_val = EBADF;
                                break;
                        }
                /*FALLTHROUGH*/
                case MEM_CACHE_ISRETIRED:
                case MEM_CACHE_STATE:
                case MEM_CACHE_READ_TAGS:
#ifdef DEBUG
                case MEM_CACHE_READ_ERROR_INJECTED_TAGS:
#endif
                        ret_val =  mem_cache_ioctl_ops(cmd, mode, &cache_info);
                        break;
                default:
                        ret_val = ENOTSUP;
                        break;
        }
        mutex_exit(&softc->mutex);
        return (ret_val);
}