/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/systm.h>
#include <sys/ddi.h>
#include <sys/sysmacros.h>
#include <sys/archsystm.h>
#include <sys/vmsystm.h>
#include <sys/machparam.h>
#include <sys/machsystm.h>
#include <sys/machthread.h>
#include <sys/cpu.h>
#include <sys/cmp.h>
#include <sys/elf_SPARC.h>
#include <vm/hat_sfmmu.h>
#include <vm/seg_kmem.h>
#include <sys/cpuvar.h>
#include <sys/cheetahregs.h>
#include <sys/us3_module.h>
#include <sys/async.h>
#include <sys/cmn_err.h>
#include <sys/debug.h>
#include <sys/dditypes.h>
#include <sys/prom_debug.h>
#include <sys/prom_plat.h>
#include <sys/cpu_module.h>
#include <sys/sysmacros.h>
#include <sys/intreg.h>
#include <sys/clock.h>
#include <sys/platform_module.h>
#include <sys/machtrap.h>
#include <sys/ontrap.h>
#include <sys/panic.h>
#include <sys/memlist.h>
#include <sys/bootconf.h>
#include <sys/ivintr.h>
#include <sys/atomic.h>
#include <sys/fm/protocol.h>
#include <sys/fm/cpu/UltraSPARC-III.h>
#include <sys/fm/util.h>
#include <sys/pghw.h>

#ifdef  CHEETAHPLUS_ERRATUM_25
#include <sys/cyclic.h>
#endif  /* CHEETAHPLUS_ERRATUM_25 */

/*
 * See comment above cpu_scrub_cpu_setup() for description
 */
#define SCRUBBER_NEITHER_CORE_ONLINE    0x0
#define SCRUBBER_CORE_0_ONLINE          0x1
#define SCRUBBER_CORE_1_ONLINE          0x2
#define SCRUBBER_BOTH_CORES_ONLINE      (SCRUBBER_CORE_0_ONLINE | \
                                        SCRUBBER_CORE_1_ONLINE)

static int pn_matching_valid_l2_line(uint64_t faddr, ch_ec_data_t *clo_l2_data);
static void cpu_async_log_tlb_parity_err(void *flt);
static cpu_t *cpu_get_sibling_core(cpu_t *cpup);


/*
 * Setup trap handlers.
 */
void
cpu_init_trap(void)
{
        CH_SET_TRAP(pil15_epilogue, ch_pil15_interrupt_instr);

        CH_SET_TRAP(tt0_fecc, fecc_err_instr);
        CH_SET_TRAP(tt1_fecc, fecc_err_tl1_instr);
        CH_SET_TRAP(tt1_swtrap0, fecc_err_tl1_cont_instr);

        CH_SET_TRAP(tt0_dperr, dcache_parity_instr);
        CH_SET_TRAP(tt1_dperr, dcache_parity_tl1_instr);
        CH_SET_TRAP(tt1_swtrap1, dcache_parity_tl1_cont_instr);

        CH_SET_TRAP(tt0_iperr, icache_parity_instr);
        CH_SET_TRAP(tt1_iperr, icache_parity_tl1_instr);
        CH_SET_TRAP(tt1_swtrap2, icache_parity_tl1_cont_instr);
}

/*
 * Set the magic constants of the implementation.
 */
/*ARGSUSED*/
void
cpu_fiximp(pnode_t dnode)
{
        int i, a;
        extern int vac_size, vac_shift;
        extern uint_t vac_mask;

        dcache_size = CH_DCACHE_SIZE;
        dcache_linesize = CH_DCACHE_LSIZE;

        icache_size = CHP_ICACHE_MAX_SIZE;
        icache_linesize = CHP_ICACHE_MIN_LSIZE;

        ecache_size = CH_ECACHE_MAX_SIZE;
        ecache_alignsize = CH_ECACHE_MAX_LSIZE;
        ecache_associativity = CHP_ECACHE_MIN_NWAY;

        /*
         * ecache_setsize needs to maximum of all cpu ecache setsizes
         */
        ecache_setsize = CHP_ECACHE_MAX_SETSIZE;
        ASSERT(ecache_setsize >= (ecache_size / ecache_associativity));

        vac_size = CH_VAC_SIZE;
        vac_mask = MMU_PAGEMASK & (vac_size - 1);
        i = 0; a = vac_size;
        while (a >>= 1)
                ++i;
        vac_shift = i;
        shm_alignment = vac_size;
        vac = 1;
}

/*
 * Use Panther values for Panther-only domains.
 * See Panther PRM, 1.5.4 Cache Hierarchy
 */
void
cpu_fix_allpanther(void)
{
        /* dcache same as Ch+ */
        icache_size = PN_ICACHE_SIZE;
        icache_linesize = PN_ICACHE_LSIZE;
        ecache_size = PN_L3_SIZE;
        ecache_alignsize = PN_L3_LINESIZE;
        ecache_associativity = PN_L3_NWAYS;
        ecache_setsize = PN_L3_SET_SIZE;
        ASSERT(ecache_setsize >= (ecache_size / ecache_associativity));
        /* vac same as Ch+ */
        /* fix hwcaps for USIV+-only domains */
        cpu_hwcap_flags |= AV_SPARC_POPC;
}

void
send_mondo_set(cpuset_t set)
{
        int lo, busy, nack, shipped = 0;
        uint16_t i, cpuids[IDSR_BN_SETS];
        uint64_t idsr, nackmask = 0, busymask, curnack, curbusy;
        uint64_t starttick, endtick, tick, lasttick;
#if (NCPU > IDSR_BN_SETS)
        int index = 0;
        int ncpuids = 0;
#endif
#ifdef  CHEETAHPLUS_ERRATUM_25
        int recovered = 0;
        int cpuid;
#endif

        ASSERT(!CPUSET_ISNULL(set));
        starttick = lasttick = gettick();

#if (NCPU <= IDSR_BN_SETS)
        for (i = 0; i < NCPU; i++)
                if (CPU_IN_SET(set, i)) {
                        shipit(i, shipped);
                        nackmask |= IDSR_NACK_BIT(shipped);
                        cpuids[shipped++] = i;
                        CPUSET_DEL(set, i);
                        if (CPUSET_ISNULL(set))
                                break;
                }
        CPU_STATS_ADDQ(CPU, sys, xcalls, shipped);
#else
        for (i = 0; i < NCPU; i++)
                if (CPU_IN_SET(set, i)) {
                        ncpuids++;

                        /*
                         * Ship only to the first (IDSR_BN_SETS) CPUs.  If we
                         * find we have shipped to more than (IDSR_BN_SETS)
                         * CPUs, set "index" to the highest numbered CPU in
                         * the set so we can ship to other CPUs a bit later on.
                         */
                        if (shipped < IDSR_BN_SETS) {
                                shipit(i, shipped);
                                nackmask |= IDSR_NACK_BIT(shipped);
                                cpuids[shipped++] = i;
                                CPUSET_DEL(set, i);
                                if (CPUSET_ISNULL(set))
                                        break;
                        } else
                                index = (int)i;
                }

        CPU_STATS_ADDQ(CPU, sys, xcalls, ncpuids);
#endif

        busymask = IDSR_NACK_TO_BUSY(nackmask);
        busy = nack = 0;
        endtick = starttick + xc_tick_limit;
        for (;;) {
                idsr = getidsr();
#if (NCPU <= IDSR_BN_SETS)
                if (idsr == 0)
                        break;
#else
                if (idsr == 0 && shipped == ncpuids)
                        break;
#endif
                tick = gettick();
                /*
                 * If there is a big jump between the current tick
                 * count and lasttick, we have probably hit a break
                 * point.  Adjust endtick accordingly to avoid panic.
                 */
                if (tick > (lasttick + xc_tick_jump_limit))
                        endtick += (tick - lasttick);
                lasttick = tick;
                if (tick > endtick) {
                        if (panic_quiesce)
                                return;
#ifdef  CHEETAHPLUS_ERRATUM_25
                        cpuid = -1;
                        for (i = 0; i < IDSR_BN_SETS; i++) {
                                if (idsr & (IDSR_NACK_BIT(i) |
                                    IDSR_BUSY_BIT(i))) {
                                        cpuid = cpuids[i];
                                        break;
                                }
                        }
                        if (cheetah_sendmondo_recover && cpuid != -1 &&
                            recovered == 0) {
                                if (mondo_recover(cpuid, i)) {
                                        /*
                                         * We claimed the whole memory or
                                         * full scan is disabled.
                                         */
                                        recovered++;
                                }
                                tick = gettick();
                                endtick = tick + xc_tick_limit;
                                lasttick = tick;
                                /*
                                 * Recheck idsr
                                 */
                                continue;
                        } else
#endif  /* CHEETAHPLUS_ERRATUM_25 */
                        {
                                cmn_err(CE_CONT, "send mondo timeout "
                                    "[%d NACK %d BUSY]\nIDSR 0x%"
                                    "" PRIx64 "  cpuids:", nack, busy, idsr);
                                for (i = 0; i < IDSR_BN_SETS; i++) {
                                        if (idsr & (IDSR_NACK_BIT(i) |
                                            IDSR_BUSY_BIT(i))) {
                                                cmn_err(CE_CONT, " 0x%x",
                                                    cpuids[i]);
                                        }
                                }
                                cmn_err(CE_CONT, "\n");
                                cmn_err(CE_PANIC, "send_mondo_set: timeout");
                        }
                }
                curnack = idsr & nackmask;
                curbusy = idsr & busymask;
#if (NCPU > IDSR_BN_SETS)
                if (shipped < ncpuids) {
                        uint64_t cpus_left;
                        uint16_t next = (uint16_t)index;

                        cpus_left = ~(IDSR_NACK_TO_BUSY(curnack) | curbusy) &
                            busymask;

                        if (cpus_left) {
                                do {
                                        /*
                                         * Sequence through and ship to the
                                         * remainder of the CPUs in the system
                                         * (e.g. other than the first
                                         * (IDSR_BN_SETS)) in reverse order.
                                         */
                                        lo = lowbit(cpus_left) - 1;
                                        i = IDSR_BUSY_IDX(lo);
                                        shipit(next, i);
                                        shipped++;
                                        cpuids[i] = next;

                                        /*
                                         * If we've processed all the CPUs,
                                         * exit the loop now and save
                                         * instructions.
                                         */
                                        if (shipped == ncpuids)
                                                break;

                                        for ((index = ((int)next - 1));
                                            index >= 0; index--)
                                                if (CPU_IN_SET(set, index)) {
                                                        next = (uint16_t)index;
                                                        break;
                                                }

                                        cpus_left &= ~(1ull << lo);
                                } while (cpus_left);
#ifdef  CHEETAHPLUS_ERRATUM_25
                                /*
                                 * Clear recovered because we are sending to
                                 * a new set of targets.
                                 */
                                recovered = 0;
#endif
                                continue;
                        }
                }
#endif
                if (curbusy) {
                        busy++;
                        continue;
                }

#ifdef SEND_MONDO_STATS
                {
                        int n = gettick() - starttick;
                        if (n < 8192)
                                x_nack_stimes[n >> 7]++;
                }
#endif
                while (gettick() < (tick + sys_clock_mhz))
                        ;
                do {
                        lo = lowbit(curnack) - 1;
                        i = IDSR_NACK_IDX(lo);
                        shipit(cpuids[i], i);
                        curnack &= ~(1ull << lo);
                } while (curnack);
                nack++;
                busy = 0;
        }
#ifdef SEND_MONDO_STATS
        {
                int n = gettick() - starttick;
                if (n < 8192)
                        x_set_stimes[n >> 7]++;
                else
                        x_set_ltimes[(n >> 13) & 0xf]++;
        }
        x_set_cpus[shipped]++;
#endif
}

/*
 * Handles error logging for implementation specific error types
 */
/*ARGSUSED1*/
int
cpu_impl_async_log_err(void *flt, errorq_elem_t *eqep)
{
        ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt;
        struct async_flt *aflt = (struct async_flt *)flt;

        switch (ch_flt->flt_type) {

        case CPU_IC_PARITY:
                cpu_async_log_ic_parity_err(flt);
                return (CH_ASYNC_LOG_DONE);

        case CPU_DC_PARITY:
                cpu_async_log_dc_parity_err(flt);
                return (CH_ASYNC_LOG_DONE);

        case CPU_DUE:
                cpu_log_err(aflt);
                cpu_page_retire(ch_flt);
                return (CH_ASYNC_LOG_DONE);

        case CPU_ITLB_PARITY:
        case CPU_DTLB_PARITY:
                cpu_async_log_tlb_parity_err(flt);
                return (CH_ASYNC_LOG_DONE);

        /* report the error and continue */
        case CPU_L3_ADDR_PE:
                cpu_log_err(aflt);
                return (CH_ASYNC_LOG_DONE);

        default:
                return (CH_ASYNC_LOG_UNKNOWN);
        }
}

/*
 * Figure out if Ecache is direct-mapped (Cheetah or Cheetah+ with Ecache
 * control ECCR_ASSOC bit off or 2-way (Cheetah+ with ECCR_ASSOC on).
 * We need to do this on the fly because we may have mixed Cheetah+'s with
 * both direct and 2-way Ecaches. Panther only supports 4-way L3$.
 */
int
cpu_ecache_nway(void)
{
        if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation))
                return (PN_L3_NWAYS);
        return ((get_ecache_ctrl() & ECCR_ASSOC) ? 2 : 1);
}

/*
 * Note that these are entered into the table: Fatal Errors (PERR, IERR, ISAP,
 * EMU, IMU) first, orphaned UCU/UCC, AFAR Overwrite policy, finally IVU, IVC.
 * Afar overwrite policy is:
 *   Class 4:
 *      AFSR     -- UCC, UCU, TUE, TSCE, TUE_SH
 *      AFSR_EXT -- L3_UCC, L3_UCU, L3_TUE, L3_TUE_SH
 *   Class 3:
 *      AFSR     -- UE, DUE, EDU, WDU, CPU
 *      AFSR_EXT -- L3_EDU, L3_WDU, L3_CPU
 *   Class 2:
 *      AFSR     -- CE, EDC, EMC, WDC, CPC, THCE
 *      AFSR_EXT -- L3_EDC, L3_WDC, L3_CPC, L3_THCE
 *   Class 1:
 *      AFSR     -- TO, DTO, BERR, DBERR
 */
ecc_type_to_info_t ecc_type_to_info[] = {

        /* Fatal Errors */
        C_AFSR_PERR,            "PERR ",        ECC_ALL_TRAPS,
                CPU_FATAL,      "PERR Fatal",
                FM_EREPORT_PAYLOAD_SYSTEM2,
                FM_EREPORT_CPU_USIII_PERR,
        C_AFSR_IERR,            "IERR ",        ECC_ALL_TRAPS,
                CPU_FATAL,      "IERR Fatal",
                FM_EREPORT_PAYLOAD_SYSTEM2,
                FM_EREPORT_CPU_USIII_IERR,
        C_AFSR_ISAP,            "ISAP ",        ECC_ALL_TRAPS,
                CPU_FATAL,      "ISAP Fatal",
                FM_EREPORT_PAYLOAD_SYSTEM1,
                FM_EREPORT_CPU_USIII_ISAP,
        C_AFSR_L3_TUE_SH,       "L3_TUE_SH ",   ECC_C_TRAP,
                CPU_FATAL,      "L3_TUE_SH Fatal",
                FM_EREPORT_PAYLOAD_L3_TAG_ECC,
                FM_EREPORT_CPU_USIII_L3_TUE_SH,
        C_AFSR_L3_TUE,          "L3_TUE ",      ECC_C_TRAP,
                CPU_FATAL,      "L3_TUE Fatal",
                FM_EREPORT_PAYLOAD_L3_TAG_ECC,
                FM_EREPORT_CPU_USIII_L3_TUE,
        C_AFSR_TUE_SH,          "TUE_SH ",      ECC_C_TRAP,
                CPU_FATAL,      "TUE_SH Fatal",
                FM_EREPORT_PAYLOAD_L2_TAG_ECC,
                FM_EREPORT_CPU_USIII_TUE_SH,
        C_AFSR_TUE,             "TUE ",         ECC_ALL_TRAPS,
                CPU_FATAL,      "TUE Fatal",
                FM_EREPORT_PAYLOAD_L2_TAG_ECC,
                FM_EREPORT_CPU_USIII_TUE,
        C_AFSR_EMU,             "EMU ",         ECC_ASYNC_TRAPS,
                CPU_FATAL,      "EMU Fatal",
                FM_EREPORT_PAYLOAD_MEMORY,
                FM_EREPORT_CPU_USIII_EMU,
        C_AFSR_IMU,             "IMU ",         ECC_C_TRAP,
                CPU_FATAL,      "IMU Fatal",
                FM_EREPORT_PAYLOAD_SYSTEM1,
                FM_EREPORT_CPU_USIII_IMU,

        /* L3$ Address parity errors are reported via the MECC bit */
        C_AFSR_L3_MECC,         "L3_MECC ",     ECC_MECC_TRAPS,
                CPU_L3_ADDR_PE, "L3 Address Parity",
                FM_EREPORT_PAYLOAD_L3_DATA,
                FM_EREPORT_CPU_USIII_L3_MECC,

        /* Orphaned UCC/UCU Errors */
        C_AFSR_L3_UCU,          "L3_OUCU ",     ECC_ORPH_TRAPS,
                CPU_ORPH,       "Orphaned L3_UCU",
                FM_EREPORT_PAYLOAD_L3_DATA,
                FM_EREPORT_CPU_USIII_L3_UCU,
        C_AFSR_L3_UCC,          "L3_OUCC ",     ECC_ORPH_TRAPS,
                CPU_ORPH,       "Orphaned L3_UCC",
                FM_EREPORT_PAYLOAD_L3_DATA,
                FM_EREPORT_CPU_USIII_L3_UCC,
        C_AFSR_UCU,             "OUCU ",        ECC_ORPH_TRAPS,
                CPU_ORPH,       "Orphaned UCU",
                FM_EREPORT_PAYLOAD_L2_DATA,
                FM_EREPORT_CPU_USIII_UCU,
        C_AFSR_UCC,             "OUCC ",        ECC_ORPH_TRAPS,
                CPU_ORPH,       "Orphaned UCC",
                FM_EREPORT_PAYLOAD_L2_DATA,
                FM_EREPORT_CPU_USIII_UCC,

        /* UCU, UCC */
        C_AFSR_L3_UCU,          "L3_UCU ",      ECC_F_TRAP,
                CPU_UE_ECACHE,  "L3_UCU",
                FM_EREPORT_PAYLOAD_L3_DATA,
                FM_EREPORT_CPU_USIII_L3_UCU,
        C_AFSR_L3_UCC,          "L3_UCC ",      ECC_F_TRAP,
                CPU_CE_ECACHE,  "L3_UCC",
                FM_EREPORT_PAYLOAD_L3_DATA,
                FM_EREPORT_CPU_USIII_L3_UCC,
        C_AFSR_UCU,             "UCU ",         ECC_F_TRAP,
                CPU_UE_ECACHE,  "UCU",
                FM_EREPORT_PAYLOAD_L2_DATA,
                FM_EREPORT_CPU_USIII_UCU,
        C_AFSR_UCC,             "UCC ",         ECC_F_TRAP,
                CPU_CE_ECACHE,  "UCC",
                FM_EREPORT_PAYLOAD_L2_DATA,
                FM_EREPORT_CPU_USIII_UCC,
        C_AFSR_TSCE,            "TSCE ",        ECC_F_TRAP,
                CPU_CE_ECACHE,  "TSCE",
                FM_EREPORT_PAYLOAD_L2_TAG_ECC,
                FM_EREPORT_CPU_USIII_TSCE,

        /* UE, EDU:ST, EDU:BLD, WDU, CPU */
        C_AFSR_UE,              "UE ",          ECC_ASYNC_TRAPS,
                CPU_UE,         "Uncorrectable system bus (UE)",
                FM_EREPORT_PAYLOAD_MEMORY,
                FM_EREPORT_CPU_USIII_UE,
        C_AFSR_L3_EDU,          "L3_EDU ",      ECC_C_TRAP,
                CPU_UE_ECACHE_RETIRE,   "L3_EDU:ST",
                FM_EREPORT_PAYLOAD_L3_DATA,
                FM_EREPORT_CPU_USIII_L3_EDUST,
        C_AFSR_L3_EDU,          "L3_EDU ",      ECC_D_TRAP,
                CPU_UE_ECACHE_RETIRE,   "L3_EDU:BLD",
                FM_EREPORT_PAYLOAD_L3_DATA,
                FM_EREPORT_CPU_USIII_L3_EDUBL,
        C_AFSR_L3_WDU,          "L3_WDU ",      ECC_C_TRAP,
                CPU_UE_ECACHE_RETIRE,   "L3_WDU",
                FM_EREPORT_PAYLOAD_L3_DATA,
                FM_EREPORT_CPU_USIII_L3_WDU,
        C_AFSR_L3_CPU,          "L3_CPU ",      ECC_C_TRAP,
                CPU_UE_ECACHE,  "L3_CPU",
                FM_EREPORT_PAYLOAD_L3_DATA,
                FM_EREPORT_CPU_USIII_L3_CPU,
        C_AFSR_EDU,             "EDU ",         ECC_C_TRAP,
                CPU_UE_ECACHE_RETIRE,   "EDU:ST",
                FM_EREPORT_PAYLOAD_L2_DATA,
                FM_EREPORT_CPU_USIII_EDUST,
        C_AFSR_EDU,             "EDU ",         ECC_D_TRAP,
                CPU_UE_ECACHE_RETIRE,   "EDU:BLD",
                FM_EREPORT_PAYLOAD_L2_DATA,
                FM_EREPORT_CPU_USIII_EDUBL,
        C_AFSR_WDU,             "WDU ",         ECC_C_TRAP,
                CPU_UE_ECACHE_RETIRE,   "WDU",
                FM_EREPORT_PAYLOAD_L2_DATA,
                FM_EREPORT_CPU_USIII_WDU,
        C_AFSR_CPU,             "CPU ",         ECC_C_TRAP,
                CPU_UE_ECACHE,  "CPU",
                FM_EREPORT_PAYLOAD_L2_DATA,
                FM_EREPORT_CPU_USIII_CPU,
        C_AFSR_DUE,             "DUE ",         ECC_C_TRAP,
                CPU_DUE,        "DUE",
                FM_EREPORT_PAYLOAD_MEMORY,
                FM_EREPORT_CPU_USIII_DUE,

        /* CE, EDC, EMC, WDC, CPC */
        C_AFSR_CE,              "CE ",          ECC_C_TRAP,
                CPU_CE,         "Corrected system bus (CE)",
                FM_EREPORT_PAYLOAD_MEMORY,
                FM_EREPORT_CPU_USIII_CE,
        C_AFSR_L3_EDC,          "L3_EDC ",      ECC_C_TRAP,
                CPU_CE_ECACHE,  "L3_EDC",
                FM_EREPORT_PAYLOAD_L3_DATA,
                FM_EREPORT_CPU_USIII_L3_EDC,
        C_AFSR_EDC,             "EDC ",         ECC_C_TRAP,
                CPU_CE_ECACHE,  "EDC",
                FM_EREPORT_PAYLOAD_L2_DATA,
                FM_EREPORT_CPU_USIII_EDC,
        C_AFSR_EMC,             "EMC ",         ECC_C_TRAP,
                CPU_EMC,        "EMC",
                FM_EREPORT_PAYLOAD_MEMORY,
                FM_EREPORT_CPU_USIII_EMC,
        C_AFSR_L3_WDC,          "L3_WDC ",      ECC_C_TRAP,
                CPU_CE_ECACHE,  "L3_WDC",
                FM_EREPORT_PAYLOAD_L3_DATA,
                FM_EREPORT_CPU_USIII_L3_WDC,
        C_AFSR_L3_CPC,          "L3_CPC ",      ECC_C_TRAP,
                CPU_CE_ECACHE,  "L3_CPC",
                FM_EREPORT_PAYLOAD_L3_DATA,
                FM_EREPORT_CPU_USIII_L3_CPC,
        C_AFSR_L3_THCE,         "L3_THCE ",     ECC_C_TRAP,
                CPU_CE_ECACHE,  "L3_THCE",
                FM_EREPORT_PAYLOAD_L3_TAG_ECC,
                FM_EREPORT_CPU_USIII_L3_THCE,
        C_AFSR_WDC,             "WDC ",         ECC_C_TRAP,
                CPU_CE_ECACHE,  "WDC",
                FM_EREPORT_PAYLOAD_L2_DATA,
                FM_EREPORT_CPU_USIII_WDC,
        C_AFSR_CPC,             "CPC ",         ECC_C_TRAP,
                CPU_CE_ECACHE,  "CPC",
                FM_EREPORT_PAYLOAD_L2_DATA,
                FM_EREPORT_CPU_USIII_CPC,
        C_AFSR_THCE,            "THCE ",        ECC_C_TRAP,
                CPU_CE_ECACHE,  "THCE",
                FM_EREPORT_PAYLOAD_L2_TAG_ECC,
                FM_EREPORT_CPU_USIII_THCE,

        /* TO, BERR */
        C_AFSR_TO,              "TO ",          ECC_ASYNC_TRAPS,
                CPU_TO,         "Timeout (TO)",
                FM_EREPORT_PAYLOAD_IO,
                FM_EREPORT_CPU_USIII_TO,
        C_AFSR_BERR,            "BERR ",        ECC_ASYNC_TRAPS,
                CPU_BERR,       "Bus Error (BERR)",
                FM_EREPORT_PAYLOAD_IO,
                FM_EREPORT_CPU_USIII_BERR,
        C_AFSR_DTO,             "DTO ",         ECC_C_TRAP,
                CPU_TO,         "Disrupting Timeout (DTO)",
                FM_EREPORT_PAYLOAD_IO,
                FM_EREPORT_CPU_USIII_DTO,
        C_AFSR_DBERR,           "DBERR ",       ECC_C_TRAP,
                CPU_BERR,       "Disrupting Bus Error (DBERR)",
                FM_EREPORT_PAYLOAD_IO,
                FM_EREPORT_CPU_USIII_DBERR,

        /* IVU, IVC, IMC */
        C_AFSR_IVU,             "IVU ",         ECC_C_TRAP,
                CPU_IV,         "IVU",
                FM_EREPORT_PAYLOAD_SYSTEM1,
                FM_EREPORT_CPU_USIII_IVU,
        C_AFSR_IVC,             "IVC ",         ECC_C_TRAP,
                CPU_IV,         "IVC",
                FM_EREPORT_PAYLOAD_SYSTEM1,
                FM_EREPORT_CPU_USIII_IVC,
        C_AFSR_IMC,             "IMC ",         ECC_C_TRAP,
                CPU_IV,         "IMC",
                FM_EREPORT_PAYLOAD_SYSTEM1,
                FM_EREPORT_CPU_USIII_IMC,

        0,                      NULL,           0,
                0,              NULL,
                FM_EREPORT_PAYLOAD_UNKNOWN,
                FM_EREPORT_CPU_USIII_UNKNOWN,
};

/*
 * See Cheetah+ Delta PRM 10.9 and section P.6.1 of the Panther PRM
 *   Class 4:
 *      AFSR     -- UCC, UCU, TUE, TSCE, TUE_SH
 *      AFSR_EXT -- L3_UCC, L3_UCU, L3_TUE, L3_TUE_SH
 *   Class 3:
 *      AFSR     -- UE, DUE, EDU, EMU, WDU, CPU
 *      AFSR_EXT -- L3_EDU, L3_WDU, L3_CPU
 *   Class 2:
 *      AFSR     -- CE, EDC, EMC, WDC, CPC, THCE
 *      AFSR_EXT -- L3_EDC, L3_WDC, L3_CPC, L3_THCE
 *   Class 1:
 *      AFSR     -- TO, DTO, BERR, DBERR
 *      AFSR_EXT --
 */
uint64_t afar_overwrite[] = {
        /* class 4: */
        C_AFSR_UCC | C_AFSR_UCU | C_AFSR_TUE | C_AFSR_TSCE | C_AFSR_TUE_SH |
        C_AFSR_L3_UCC | C_AFSR_L3_UCU | C_AFSR_L3_TUE | C_AFSR_L3_TUE_SH,
        /* class 3: */
        C_AFSR_UE | C_AFSR_DUE | C_AFSR_EDU | C_AFSR_EMU | C_AFSR_WDU |
        C_AFSR_CPU | C_AFSR_L3_EDU | C_AFSR_L3_WDU | C_AFSR_L3_CPU,
        /* class 2: */
        C_AFSR_CE | C_AFSR_EDC | C_AFSR_EMC | C_AFSR_WDC | C_AFSR_CPC |
        C_AFSR_THCE | C_AFSR_L3_EDC | C_AFSR_L3_WDC | C_AFSR_L3_CPC |
        C_AFSR_L3_THCE,
        /* class 1: */
        C_AFSR_TO | C_AFSR_DTO | C_AFSR_BERR | C_AFSR_DBERR,

        0
};

/*
 * For Cheetah+, the E_SYND and M_SYND overwrite priorities are combined.
 * See Cheetah+ Delta PRM 10.9 and Cheetah+ PRM 11.6.2
 *   Class 2:  UE, DUE, IVU, EDU, EMU, WDU, UCU, CPU
 *   Class 1:  CE, IVC, EDC, EMC, WDC, UCC, CPC
 */
uint64_t esynd_overwrite[] = {
        /* class 2: */
        C_AFSR_UE | C_AFSR_DUE | C_AFSR_IVU | C_AFSR_EDU | C_AFSR_EMU |
            C_AFSR_WDU | C_AFSR_UCU | C_AFSR_CPU,
        /* class 1: */
        C_AFSR_CE | C_AFSR_IVC | C_AFSR_EDC | C_AFSR_EMC | C_AFSR_WDC |
            C_AFSR_UCC | C_AFSR_CPC,
        0
};

/*
 * In panther, the E_SYND overwrite policy changed a little bit
 * by adding one more level.
 * See Panther PRM P.6.2
 *   class 3:
 *      AFSR     -- UCU, UCC
 *      AFSR_EXT -- L3_UCU, L3_UCC
 *   Class 2:
 *      AFSR     -- UE, DUE, IVU, EDU, WDU, CPU
 *      AFSR_EXT -- L3_EDU, L3_WDU, L3_CPU
 *   Class 1:
 *      AFSR     -- CE, IVC, EDC, WDC, CPC
 *      AFSR_EXT -- L3_EDC, L3_WDC, L3_CPC
 */
uint64_t pn_esynd_overwrite[] = {
        /* class 3: */
        C_AFSR_UCU | C_AFSR_UCC |
        C_AFSR_L3_UCU | C_AFSR_L3_UCC,
        /* class 2: */
        C_AFSR_UE | C_AFSR_DUE | C_AFSR_IVU | C_AFSR_EDU | C_AFSR_WDU |
            C_AFSR_CPU |
        C_AFSR_L3_EDU | C_AFSR_L3_WDU | C_AFSR_L3_CPU,
        /* class 1: */
        C_AFSR_CE | C_AFSR_IVC | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_CPC |
        C_AFSR_L3_EDC | C_AFSR_L3_WDC | C_AFSR_L3_CPC,

        0
};

int
afsr_to_pn_esynd_status(uint64_t afsr, uint64_t afsr_bit)
{
        return (afsr_to_overw_status(afsr, afsr_bit, pn_esynd_overwrite));
}

/*
 * Prioritized list of Error bits for MSYND overwrite.
 * See Panther PRM P.6.2 (For Cheetah+, see esynd_overwrite classes)
 *   Class 2:  EMU, IMU
 *   Class 1:  EMC, IMC
 *
 * Panther adds IMU and IMC.
 */
uint64_t msynd_overwrite[] = {
        /* class 2: */
        C_AFSR_EMU | C_AFSR_IMU,
        /* class 1: */
        C_AFSR_EMC | C_AFSR_IMC,

        0
};

/*
 * change cpu speed bits -- new speed will be normal-speed/divisor.
 *
 * The Jalapeno memory controllers are required to drain outstanding
 * memory transactions within 32 JBus clocks in order to be ready
 * to enter Estar mode.  In some corner cases however, that time
 * fell short.
 *
 * A safe software solution is to force MCU to act like in Estar mode,
 * then delay 1us (in ppm code) prior to assert J_CHNG_L signal.
 * To reverse the effect, upon exiting Estar, software restores the
 * MCU to its original state.
 */
/* ARGSUSED1 */
void
cpu_change_speed(uint64_t divisor, uint64_t arg2)
{
        bus_config_eclk_t       *bceclk;
        uint64_t                reg;
        processor_info_t        *pi = &(CPU->cpu_type_info);

        for (bceclk = bus_config_eclk; bceclk->divisor; bceclk++) {
                if (bceclk->divisor != divisor)
                        continue;
                reg = get_safari_config();
                reg &= ~SAFARI_CONFIG_ECLK_MASK;
                reg |= bceclk->mask;
                set_safari_config(reg);
                CPU->cpu_m.divisor = (uchar_t)divisor;
                cpu_set_curr_clock(((uint64_t)pi->pi_clock * 1000000) /
                    divisor);
                return;
        }
        /*
         * We will reach here only if OBP and kernel don't agree on
         * the speeds supported by the CPU.
         */
        cmn_err(CE_WARN, "cpu_change_speed: bad divisor %" PRIu64, divisor);
}

/*
 * Cpu private initialization.  This includes allocating the cpu_private
 * data structure, initializing it, and initializing the scrubber for this
 * cpu.  This function calls cpu_init_ecache_scrub_dr to init the scrubber.
 * We use kmem_cache_create for the cheetah private data structure because
 * it needs to be allocated on a PAGESIZE (8192) byte boundary.
 */
void
cpu_init_private(struct cpu *cp)
{
        cheetah_private_t *chprp;
        int i;

        ASSERT(CPU_PRIVATE(cp) == NULL);

        /* LINTED: E_TRUE_LOGICAL_EXPR */
        ASSERT((offsetof(cheetah_private_t, chpr_tl1_err_data) +
            sizeof (ch_err_tl1_data_t) * CH_ERR_TL1_TLMAX) <= PAGESIZE);

        /*
         * Running with Cheetah CPUs in a Cheetah+, Jaguar, Panther or
         * mixed Cheetah+/Jaguar/Panther machine is not a supported
         * configuration. Attempting to do so may result in unpredictable
         * failures (e.g. running Cheetah+ CPUs with Cheetah E$ disp flush)
         * so don't allow it.
         *
         * This is just defensive code since this configuration mismatch
         * should have been caught prior to OS execution.
         */
        if (!(IS_CHEETAH_PLUS(cpunodes[cp->cpu_id].implementation) ||
            IS_JAGUAR(cpunodes[cp->cpu_id].implementation) ||
            IS_PANTHER(cpunodes[cp->cpu_id].implementation))) {
                cmn_err(CE_PANIC, "CPU%d: UltraSPARC-III not supported"
                    " on UltraSPARC-III+/IV/IV+ code\n", cp->cpu_id);
        }

        /*
         * If the ch_private_cache has not been created, create it.
         */
        if (ch_private_cache == NULL) {
                ch_private_cache = kmem_cache_create("ch_private_cache",
                    sizeof (cheetah_private_t), PAGESIZE, NULL, NULL,
                    NULL, NULL, static_arena, 0);
        }

        chprp = CPU_PRIVATE(cp) = kmem_cache_alloc(ch_private_cache, KM_SLEEP);

        bzero(chprp, sizeof (cheetah_private_t));
        chprp->chpr_fecctl0_logout.clo_data.chd_afar = LOGOUT_INVALID;
        chprp->chpr_cecc_logout.clo_data.chd_afar = LOGOUT_INVALID;
        chprp->chpr_async_logout.clo_data.chd_afar = LOGOUT_INVALID;
        chprp->chpr_tlb_logout.tlo_addr = LOGOUT_INVALID;
        for (i = 0; i < CH_ERR_TL1_TLMAX; i++)
                chprp->chpr_tl1_err_data[i].ch_err_tl1_logout.clo_data.chd_afar
                    = LOGOUT_INVALID;

        /* Panther has a larger Icache compared to cheetahplus or Jaguar */
        if (IS_PANTHER(cpunodes[cp->cpu_id].implementation)) {
                chprp->chpr_icache_size = PN_ICACHE_SIZE;
                chprp->chpr_icache_linesize = PN_ICACHE_LSIZE;
        } else {
                chprp->chpr_icache_size = CH_ICACHE_SIZE;
                chprp->chpr_icache_linesize = CH_ICACHE_LSIZE;
        }

        cpu_init_ecache_scrub_dr(cp);

        /*
         * Panther's L2$ and E$ are shared between cores, so the scrubber is
         * only needed on one of the cores.  At this point, we assume all cores
         * are online, and we only enable the scrubber on core 0.
         */
        if (IS_PANTHER(cpunodes[cp->cpu_id].implementation)) {
                chprp->chpr_scrub_misc.chsm_core_state =
                    SCRUBBER_BOTH_CORES_ONLINE;
                if (cp->cpu_id != (processorid_t)cmp_cpu_to_chip(cp->cpu_id)) {
                        chprp->chpr_scrub_misc.chsm_enable[
                            CACHE_SCRUBBER_INFO_E] = 0;
                }
        }

        chprp->chpr_ec_set_size = cpunodes[cp->cpu_id].ecache_size /
            cpu_ecache_nway();

        adjust_hw_copy_limits(cpunodes[cp->cpu_id].ecache_size);
        ch_err_tl1_paddrs[cp->cpu_id] = va_to_pa(chprp);
        ASSERT(ch_err_tl1_paddrs[cp->cpu_id] != -1);
}

/*
 * Clear the error state registers for this CPU.
 * For Cheetah+/Jaguar, just clear the AFSR but
 * for Panther we also have to clear the AFSR_EXT.
 */
void
set_cpu_error_state(ch_cpu_errors_t *cpu_error_regs)
{
        set_asyncflt(cpu_error_regs->afsr & ~C_AFSR_FATAL_ERRS);
        if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
                set_afsr_ext(cpu_error_regs->afsr_ext & ~C_AFSR_EXT_FATAL_ERRS);
        }
}

void
pn_cpu_log_diag_l2_info(ch_async_flt_t *ch_flt) {
        struct async_flt *aflt = (struct async_flt *)ch_flt;
        ch_ec_data_t *l2_data = &ch_flt->flt_diag_data.chd_l2_data[0];
        uint64_t faddr = aflt->flt_addr;
        uint8_t log_way_mask = 0;
        int i;

        /*
         * Only Panther CPUs have the additional L2$ data that needs
         * to be logged here
         */
        if (!IS_PANTHER(cpunodes[aflt->flt_inst].implementation))
                return;

        /*
         * We'll use a simple bit mask to keep track of which way(s)
         * of the stored cache line we want to log. The idea is to
         * log the entry if it is a valid line and it matches our
         * fault AFAR. If no match is found, we will simply log all
         * the ways.
         */
        for (i = 0; i < PN_L2_NWAYS; i++)
                if (pn_matching_valid_l2_line(faddr, &l2_data[i]))
                        log_way_mask |= (1 << i);

        /* If no matching valid lines were found, we log all ways */
        if (log_way_mask == 0)
                log_way_mask = (1 << PN_L2_NWAYS) - 1;

        /* Log the cache lines */
        for (i = 0; i < PN_L2_NWAYS; i++)
                if (log_way_mask & (1 << i))
                        l2_data[i].ec_logflag = EC_LOGFLAG_MAGIC;
}

/*
 * For this routine to return true, the L2 tag in question must be valid
 * and the tag PA must match the fault address (faddr) assuming the correct
 * index is being used.
 */
static int
pn_matching_valid_l2_line(uint64_t faddr, ch_ec_data_t *clo_l2_data) {
        if ((!PN_L2_LINE_INVALID(clo_l2_data->ec_tag)) &&
        ((faddr & P2ALIGN(C_AFAR_PA, PN_L2_SET_SIZE)) ==
            PN_L2TAG_TO_PA(clo_l2_data->ec_tag)))
                return (1);
        return (0);
}

/*
 * This array is used to convert the 3 digit PgSz encoding (as used in
 * various MMU registers such as MMU_TAG_ACCESS_EXT) into the corresponding
 * page size.
 */
static uint64_t tlb_pgsz_to_size[] = {
        /* 000 = 8KB: */
        0x2000,
        /* 001 = 64KB: */
        0x10000,
        /* 010 = 512KB: */
        0x80000,
        /* 011 = 4MB: */
        0x400000,
        /* 100 = 32MB: */
        0x2000000,
        /* 101 = 256MB: */
        0x10000000,
        /* undefined for encodings 110 and 111: */
        0, 0
};

/*
 * The itlb_parity_trap and dtlb_parity_trap handlers transfer control here
 * after collecting logout information related to the TLB parity error and
 * flushing the offending TTE entries from the ITLB or DTLB.
 *
 * DTLB traps which occur at TL>0 are not recoverable because we will most
 * likely be corrupting some other trap handler's alternate globals. As
 * such, we simply panic here when that happens. ITLB parity errors are
 * not expected to happen at TL>0.
 */
void
cpu_tlb_parity_error(struct regs *rp, ulong_t trap_va, ulong_t tlb_info) {
        ch_async_flt_t ch_flt;
        struct async_flt *aflt;
        pn_tlb_logout_t *tlop = NULL;
        int immu_parity = (tlb_info & PN_TLO_INFO_IMMU) != 0;
        int tl1_trap = (tlb_info & PN_TLO_INFO_TL1) != 0;
        char *error_class;

        bzero(&ch_flt, sizeof (ch_async_flt_t));

        /*
         * Get the CPU log out info. If we can't find our CPU private
         * pointer, or if the logout information does not correspond to
         * this error, then we will have to make due without detailed
         * logout information.
         */
        if (CPU_PRIVATE(CPU)) {
                tlop = CPU_PRIVATE_PTR(CPU, chpr_tlb_logout);
                if ((tlop->tlo_addr != trap_va) ||
                    (tlop->tlo_info != tlb_info))
                        tlop = NULL;
        }

        if (tlop) {
                ch_flt.tlb_diag_data = *tlop;

                /* Zero out + invalidate TLB logout. */
                bzero(tlop, sizeof (pn_tlb_logout_t));
                tlop->tlo_addr = LOGOUT_INVALID;
        } else {
                /*
                 * Copy what logout information we have and mark
                 * it incomplete.
                 */
                ch_flt.flt_data_incomplete = 1;
                ch_flt.tlb_diag_data.tlo_info = tlb_info;
                ch_flt.tlb_diag_data.tlo_addr = trap_va;
        }

        /*
         * Log the error.
         */
        aflt = (struct async_flt *)&ch_flt;
        aflt->flt_id = gethrtime_waitfree();
        aflt->flt_bus_id = getprocessorid();
        aflt->flt_inst = CPU->cpu_id;
        aflt->flt_pc = (caddr_t)rp->r_pc;
        aflt->flt_addr = trap_va;
        aflt->flt_prot = AFLT_PROT_NONE;
        aflt->flt_class = CPU_FAULT;
        aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ?  1 : 0;
        aflt->flt_tl = tl1_trap ? 1 : 0;
        aflt->flt_panic = tl1_trap ? 1 : 0;

        if (immu_parity) {
                aflt->flt_status = ECC_ITLB_TRAP;
                ch_flt.flt_type = CPU_ITLB_PARITY;
                error_class = FM_EREPORT_CPU_USIII_ITLBPE;
                aflt->flt_payload = FM_EREPORT_PAYLOAD_ITLB_PE;
        } else {
                aflt->flt_status = ECC_DTLB_TRAP;
                ch_flt.flt_type = CPU_DTLB_PARITY;
                error_class = FM_EREPORT_CPU_USIII_DTLBPE;
                aflt->flt_payload = FM_EREPORT_PAYLOAD_DTLB_PE;
        }

        /*
         * The TLB entries have already been flushed by the TL1 trap
         * handler so at this point the only thing left to do is log
         * the error message.
         */
        if (aflt->flt_panic) {
                cpu_errorq_dispatch(error_class, (void *)&ch_flt,
                    sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic);
                /*
                 * Panic here if aflt->flt_panic has been set.  Enqueued
                 * errors will be logged as part of the panic flow.
                 */
                fm_panic("%sError(s)", immu_parity ? "ITLBPE " : "DTLBPE ");
        } else {
                cpu_errorq_dispatch(error_class, (void *)&ch_flt,
                    sizeof (ch_async_flt_t), ce_queue, aflt->flt_panic);
        }
}

/*
 * This routine is called when a TLB parity error event is 'ue_drain'ed
 * or 'ce_drain'ed from the errorq.
 */
void
cpu_async_log_tlb_parity_err(void *flt) {
        ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt;
        struct async_flt *aflt = (struct async_flt *)flt;
#ifdef lint
        aflt = aflt;
#endif

        /*
         * We only capture TLB information if we encountered
         * a TLB parity error and Panther is the only CPU which
         * can detect a TLB parity error.
         */
        ASSERT(IS_PANTHER(cpunodes[aflt->flt_inst].implementation));
        ASSERT((ch_flt->flt_type == CPU_ITLB_PARITY) ||
            (ch_flt->flt_type == CPU_DTLB_PARITY));

        if (ch_flt->flt_data_incomplete == 0) {
                if (ch_flt->flt_type == CPU_ITLB_PARITY)
                        ch_flt->tlb_diag_data.tlo_logflag = IT_LOGFLAG_MAGIC;
                else /* parity error is in DTLB */
                        ch_flt->tlb_diag_data.tlo_logflag = DT_LOGFLAG_MAGIC;
        }
}

/*
 * Add L1 Prefetch cache data to the ereport payload.
 */
void
cpu_payload_add_pcache(struct async_flt *aflt, nvlist_t *nvl)
{
        ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
        ch_pc_data_t *pcp;
        ch_pc_data_t pcdata[CH_PCACHE_NWAY];
        uint_t nelem;
        int i, ways_logged = 0;

        /*
         * We only capture P$ information if we encountered
         * a P$ parity error and Panther is the only CPU which
         * can detect a P$ parity error.
         */
        ASSERT(IS_PANTHER(cpunodes[aflt->flt_inst].implementation));
        for (i = 0; i < CH_PCACHE_NWAY; i++) {
                pcp = &ch_flt->parity_data.dpe.cpl_pc[i];
                if (pcp->pc_logflag == PC_LOGFLAG_MAGIC) {
                        bcopy(pcp, &pcdata[ways_logged],
                            sizeof (ch_pc_data_t));
                        ways_logged++;
                }
        }

        /*
         * Add the pcache data to the payload.
         */
        fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1P_WAYS,
            DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL);
        if (ways_logged != 0) {
                nelem = sizeof (ch_pc_data_t) / sizeof (uint64_t) * ways_logged;
                fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1P_DATA,
                    DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)pcdata, NULL);
        }
}

/*
 * Add TLB diagnostic data to the ereport payload.
 */
void
cpu_payload_add_tlb(struct async_flt *aflt, nvlist_t *nvl)
{
        ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
        uint8_t num_entries, tlb_data_words;

        /*
         * We only capture TLB information if we encountered
         * a TLB parity error and Panther is the only CPU which
         * can detect a TLB parity error.
         */
        ASSERT(IS_PANTHER(cpunodes[aflt->flt_inst].implementation));
        ASSERT((ch_flt->flt_type == CPU_ITLB_PARITY) ||
            (ch_flt->flt_type == CPU_DTLB_PARITY));

        if (ch_flt->flt_type == CPU_ITLB_PARITY) {
                num_entries = (uint8_t)(PN_ITLB_NWAYS * PN_NUM_512_ITLBS);
                tlb_data_words = sizeof (ch_tte_entry_t) / sizeof (uint64_t) *
                    num_entries;

                /*
                 * Add the TLB diagnostic data to the payload
                 * if it was collected.
                 */
                if (ch_flt->tlb_diag_data.tlo_logflag == IT_LOGFLAG_MAGIC) {
                        fm_payload_set(nvl,
                            FM_EREPORT_PAYLOAD_NAME_ITLB_ENTRIES,
                            DATA_TYPE_UINT8, num_entries, NULL);
                        fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_ITLB_DATA,
                            DATA_TYPE_UINT64_ARRAY, tlb_data_words,
                            (uint64_t *)ch_flt->tlb_diag_data.tlo_itlb_tte,
                            NULL);
                }
        } else {
                num_entries = (uint8_t)(PN_DTLB_NWAYS * PN_NUM_512_DTLBS);
                tlb_data_words = sizeof (ch_tte_entry_t) / sizeof (uint64_t) *
                    num_entries;

                fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_VA,
                    DATA_TYPE_UINT64, ch_flt->tlb_diag_data.tlo_addr, NULL);

                /*
                 * Add the TLB diagnostic data to the payload
                 * if it was collected.
                 */
                if (ch_flt->tlb_diag_data.tlo_logflag == DT_LOGFLAG_MAGIC) {
                        fm_payload_set(nvl,
                            FM_EREPORT_PAYLOAD_NAME_DTLB_ENTRIES,
                            DATA_TYPE_UINT8, num_entries, NULL);
                        fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_DTLB_DATA,
                            DATA_TYPE_UINT64_ARRAY, tlb_data_words,
                            (uint64_t *)ch_flt->tlb_diag_data.tlo_dtlb_tte,
                            NULL);
                }
        }
}

/*
 * Panther Cache Scrubbing:
 *
 * In Jaguar, the E$ was split between cores, so the scrubber must run on both
 * cores.  For Panther, however, the L2$ and L3$ are shared across cores.
 * Therefore, the E$ scrubber only needs to run on one of the two cores.
 *
 * There are four possible states for the E$ scrubber:
 *
 * 0. If both cores are offline, add core 0 to cpu_offline_set so that
 *    the offline scrubber will run on it.
 * 1. If core 0 is online and core 1 off, we run the scrubber on core 0.
 * 2. If core 1 is online and core 0 off, we move the scrubber to run
 *    on core 1.
 * 3. If both cores are online, only run the scrubber on core 0.
 *
 * These states are enumerated by the SCRUBBER_[BOTH|CORE|NEITHER]_* defines
 * above.  One of those values is stored in
 * chpr_scrub_misc->chsm_core_state on each core.
 *
 * Also note that, for Panther, ecache_flush_line() will flush out the L2$
 * before the E$, so the L2$ will be scrubbed by the E$ scrubber.  No
 * additional code is necessary to scrub the L2$.
 *
 * For all cpu types, whenever a cpu or core is offlined, add it to
 * cpu_offline_set so the necessary scrubbers can still run.  This is still
 * necessary on Panther so the D$ scrubber can still run.
 */
/*ARGSUSED*/
int
cpu_scrub_cpu_setup(cpu_setup_t what, int cpuid, void *arg)
{
        processorid_t core_0_id;
        cpu_t *core_cpus[2];
        ch_scrub_misc_t *core_scrub[2];
        int old_state, i;
        int new_state = SCRUBBER_NEITHER_CORE_ONLINE;

        switch (what) {
        case CPU_ON:
        case CPU_INIT:
                CPUSET_DEL(cpu_offline_set, cpuid);
                break;
        case CPU_OFF:
                CPUSET_ADD(cpu_offline_set, cpuid);
                break;
        default:
                return (0);
        }

        if (!IS_PANTHER(cpunodes[cpuid].implementation)) {
                return (0);
        }

        /*
         * Update the chsm_enable[CACHE_SCRUBBER_INFO_E] value
         * if necessary
         */
        core_0_id = cmp_cpu_to_chip(cpuid);
        core_cpus[0] = cpu_get(core_0_id);
        core_cpus[1] = cpu_get_sibling_core(core_cpus[0]);

        for (i = 0; i < 2; i++) {
                if (core_cpus[i] == NULL) {
                        /*
                         * This may happen during DR - one core is offlined
                         * and completely unconfigured before the second
                         * core is offlined.  Give up and return quietly,
                         * since the second core should quickly be removed
                         * anyways.
                         */
                        return (0);
                }
                core_scrub[i] = CPU_PRIVATE_PTR(core_cpus[i], chpr_scrub_misc);
        }

        if (cpuid == (processorid_t)cmp_cpu_to_chip(cpuid)) {
                /* cpuid is core 0 */
                if (cpu_is_active(core_cpus[1])) {
                        new_state |= SCRUBBER_CORE_1_ONLINE;
                }
                if (what != CPU_OFF) {
                        new_state |= SCRUBBER_CORE_0_ONLINE;
                }
        } else {
                /* cpuid is core 1 */
                if (cpu_is_active(core_cpus[0])) {
                        new_state |= SCRUBBER_CORE_0_ONLINE;
                }
                if (what != CPU_OFF) {
                        new_state |= SCRUBBER_CORE_1_ONLINE;
                }
        }

        old_state = core_scrub[0]->chsm_core_state;

        if (old_state == new_state) {
                return (0);
        }

        if (old_state == SCRUBBER_CORE_1_ONLINE) {
                /*
                 * We need to move the scrubber state from core 1
                 * back to core 0.  This data is not protected by
                 * locks, but the worst that can happen is some
                 * lines are scrubbed multiple times.  chsm_oustanding is
                 * set to 0 to make sure an interrupt is scheduled the
                 * first time through do_scrub().
                 */
                core_scrub[0]->chsm_flush_index[CACHE_SCRUBBER_INFO_E] =
                    core_scrub[1]->chsm_flush_index[CACHE_SCRUBBER_INFO_E];
                core_scrub[0]->chsm_outstanding[CACHE_SCRUBBER_INFO_E] = 0;
        }

        switch (new_state) {
        case SCRUBBER_NEITHER_CORE_ONLINE:
        case SCRUBBER_BOTH_CORES_ONLINE:
        case SCRUBBER_CORE_0_ONLINE:
                core_scrub[1]->chsm_enable[CACHE_SCRUBBER_INFO_E] = 0;
                core_scrub[0]->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1;
                break;

        case SCRUBBER_CORE_1_ONLINE:
        default:
                /*
                 * We need to move the scrubber state from core 0
                 * to core 1.
                 */
                core_scrub[1]->chsm_flush_index[CACHE_SCRUBBER_INFO_E] =
                    core_scrub[0]->chsm_flush_index[CACHE_SCRUBBER_INFO_E];
                core_scrub[1]->chsm_outstanding[CACHE_SCRUBBER_INFO_E] = 0;

                core_scrub[0]->chsm_enable[CACHE_SCRUBBER_INFO_E] = 0;
                core_scrub[1]->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1;
                break;
        }

        core_scrub[0]->chsm_core_state = new_state;
        core_scrub[1]->chsm_core_state = new_state;
        return (0);
}

/*
 * Returns a pointer to the cpu structure of the argument's sibling core.
 * If no sibling core can be found, return NULL.
 */
static cpu_t *
cpu_get_sibling_core(cpu_t *cpup)
{
        cpu_t           *nextp;
        pg_t            *pg;
        pg_cpu_itr_t    i;

        if ((cpup == NULL) || (!cmp_cpu_is_cmp(cpup->cpu_id)))
                return (NULL);
        pg = (pg_t *)pghw_find_pg(cpup, PGHW_CHIP);
        if (pg == NULL)
                return (NULL);

        /*
         * Iterate over the CPUs in the chip PG looking
         * for a CPU that isn't cpup
         */
        PG_CPU_ITR_INIT(pg, i);
        while ((nextp = pg_cpu_next(&i)) != NULL) {
                if (nextp != cpup)
                        break;
        }

        if (nextp == NULL)
                return (NULL);

        return (nextp);
}