/*      $NetBSD: e500_intr.c,v 1.49 2024/09/15 19:08:34 andvar Exp $    */
/*-
 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects
 * Agency and which was developed by Matt Thomas of 3am Software Foundry.
 *
 * This material is based upon work supported by the Defense Advanced Research
 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under
 * Contract No. N66001-09-C-2073.
 * Approved for Public Release, Distribution Unlimited
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#define __INTR_PRIVATE

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: e500_intr.c,v 1.49 2024/09/15 19:08:34 andvar Exp $");

#ifdef _KERNEL_OPT
#include "opt_mpc85xx.h"
#include "opt_multiprocessor.h"
#endif

#include <sys/param.h>
#include <sys/proc.h>
#include <sys/intr.h>
#include <sys/cpu.h>
#include <sys/kmem.h>
#include <sys/atomic.h>
#include <sys/bus.h>
#include <sys/xcall.h>
#include <sys/ipi.h>
#include <sys/bitops.h>
#include <sys/interrupt.h>
#include <sys/systm.h>

#include <uvm/uvm_extern.h>

#ifdef __HAVE_FAST_SOFTINTS
#include <powerpc/softint.h>
#endif

#include <powerpc/spr.h>
#include <powerpc/booke/spr.h>

#include <powerpc/booke/cpuvar.h>
#include <powerpc/booke/e500reg.h>
#include <powerpc/booke/e500var.h>
#include <powerpc/booke/openpicreg.h>

#define IPL2CTPR(ipl)           ((ipl) + 15 - IPL_HIGH)
#define CTPR2IPL(ctpr)          ((ctpr) - (15 - IPL_HIGH))

#define IST_PERCPU_P(ist)       ((ist) >= IST_TIMER)

struct e500_intr_irq_info {
        bus_addr_t irq_vpr;
        bus_addr_t irq_dr;
        u_int irq_vector;
};

struct intr_source {
        int (*is_func)(void *);
        void *is_arg;
        int8_t is_ipl;
        uint8_t is_ist;
        uint8_t is_irq;
        uint8_t is_refcnt;
        bus_size_t is_vpr;
        bus_size_t is_dr;
        char is_source[INTRIDBUF];
        char is_xname[INTRDEVNAMEBUF];
};

#define INTR_SOURCE_INITIALIZER \
        { .is_func = e500_intr_spurious, .is_arg = NULL, \
        .is_irq = -1, .is_ipl = IPL_NONE, .is_ist = IST_NONE, \
        .is_source = "", .is_xname = "", }

struct e500_intr_name {
        uint8_t in_irq;
        const char in_name[15];
};

static const struct e500_intr_name e500_onchip_intr_names[] = {
        { ISOURCE_L2, "l2" },
        { ISOURCE_ECM, "ecm" },
        { ISOURCE_DDR, "ddr" },
        { ISOURCE_LBC, "lbc" },
        { ISOURCE_DMA_CHAN1, "dma-chan1" },
        { ISOURCE_DMA_CHAN2, "dma-chan2" },
        { ISOURCE_DMA_CHAN3, "dma-chan3" },
        { ISOURCE_DMA_CHAN4, "dma-chan4" },
        { ISOURCE_PCI1, "pci1" },
        { ISOURCE_PCIEX2, "pcie2" },
        { ISOURCE_PCIEX , "pcie1" },
        { ISOURCE_PCIEX3, "pcie3" },
        { ISOURCE_USB1, "usb1" },
        { ISOURCE_ETSEC1_TX, "etsec1-tx" },
        { ISOURCE_ETSEC1_RX, "etsec1-rx" },
        { ISOURCE_ETSEC3_TX, "etsec3-tx" },
        { ISOURCE_ETSEC3_RX, "etsec3-rx" },
        { ISOURCE_ETSEC3_ERR, "etsec3-err" },
        { ISOURCE_ETSEC1_ERR, "etsec1-err" },
        { ISOURCE_ETSEC2_TX, "etsec2-tx" },
        { ISOURCE_ETSEC2_RX, "etsec2-rx" },
        { ISOURCE_ETSEC4_TX, "etsec4-tx" },
        { ISOURCE_ETSEC4_RX, "etsec4-rx" },
        { ISOURCE_ETSEC4_ERR, "etsec4-err" },
        { ISOURCE_ETSEC2_ERR, "etsec2-err" },
        { ISOURCE_DUART, "duart" },
        { ISOURCE_I2C, "i2c" },
        { ISOURCE_PERFMON, "perfmon" },
        { ISOURCE_SECURITY1, "sec1" },
        { ISOURCE_GPIO, "gpio" },
        { ISOURCE_SRIO_EWPU, "srio-ewpu" },
        { ISOURCE_SRIO_ODBELL, "srio-odbell" },
        { ISOURCE_SRIO_IDBELL, "srio-idbell" },
        { ISOURCE_SRIO_OMU1, "srio-omu1" },
        { ISOURCE_SRIO_IMU1, "srio-imu1" },
        { ISOURCE_SRIO_OMU2, "srio-omu2" },
        { ISOURCE_SRIO_IMU2, "srio-imu2" },
        { ISOURCE_SECURITY2, "sec2" },
        { ISOURCE_SPI, "spi" },
        { ISOURCE_ETSEC1_PTP, "etsec1-ptp" },
        { ISOURCE_ETSEC2_PTP, "etsec2-ptp" },
        { ISOURCE_ETSEC3_PTP, "etsec3-ptp" },
        { ISOURCE_ETSEC4_PTP, "etsec4-ptp" },
        { ISOURCE_ESDHC, "esdhc" },
        { 0, "" },
};

const struct e500_intr_name default_external_intr_names[] = {
        { 0, "" },
};

static const struct e500_intr_name e500_msigroup_intr_names[] = {
        { 0, "msigroup0" },
        { 1, "msigroup1" },
        { 2, "msigroup2" },
        { 3, "msigroup3" },
        { 4, "msigroup4" },
        { 5, "msigroup5" },
        { 6, "msigroup6" },
        { 7, "msigroup7" },
        { 0, "" },
};

static const struct e500_intr_name e500_timer_intr_names[] = {
        { 0, "timer0" },
        { 1, "timer1" },
        { 2, "timer2" },
        { 3, "timer3" },
        { 0, "" },
};

static const struct e500_intr_name e500_ipi_intr_names[] = {
        { 0, "ipi0" },
        { 1, "ipi1" },
        { 2, "ipi2" },
        { 3, "ipi3" },
        { 0, "" },
};

static const struct e500_intr_name e500_mi_intr_names[] = {
        { 0, "mi0" },
        { 1, "mi1" },
        { 2, "mi2" },
        { 3, "mi3" },
        { 0, "" },
};

struct e500_intr_info {
        u_int ii_external_sources;
        uint32_t ii_onchip_bitmap[2];
        u_int ii_onchip_sources;
        u_int ii_msigroup_sources;
        u_int ii_ipi_sources;                   /* per-cpu */
        u_int ii_timer_sources;                 /* per-cpu */
        u_int ii_mi_sources;                    /* per-cpu */
        u_int ii_percpu_sources;
        const struct e500_intr_name *ii_external_intr_names;
        const struct e500_intr_name *ii_onchip_intr_names;
        u_int8_t ii_ist_vectors[IST_MAX+1];
};

static kmutex_t e500_intr_lock __cacheline_aligned;
static struct e500_intr_info e500_intr_info;

#define INTR_INFO_DECL(lc_chip, UC_CHIP)                                \
static const struct e500_intr_info lc_chip##_intr_info = {              \
        .ii_external_sources = UC_CHIP ## _EXTERNALSOURCES,             \
        .ii_onchip_bitmap = UC_CHIP ## _ONCHIPBITMAP,                   \
        .ii_onchip_sources = UC_CHIP ## _ONCHIPSOURCES,                 \
        .ii_msigroup_sources = UC_CHIP ## _MSIGROUPSOURCES,             \
        .ii_timer_sources = UC_CHIP ## _TIMERSOURCES,                   \
        .ii_ipi_sources = UC_CHIP ## _IPISOURCES,                       \
        .ii_mi_sources = UC_CHIP ## _MISOURCES,                         \
        .ii_percpu_sources = UC_CHIP ## _TIMERSOURCES                   \
            + UC_CHIP ## _IPISOURCES + UC_CHIP ## _MISOURCES,           \
        .ii_external_intr_names = lc_chip ## _external_intr_names,      \
        .ii_onchip_intr_names = lc_chip ## _onchip_intr_names,          \
        .ii_ist_vectors = {                                             \
                [IST_NONE]              = ~0,                           \
                [IST_EDGE]              = 0,                            \
                [IST_LEVEL_LOW]         = 0,                            \
                [IST_LEVEL_HIGH]        = 0,                            \
                [IST_PULSE]             = 0,                            \
                [IST_ONCHIP]            = UC_CHIP ## _EXTERNALSOURCES,  \
                [IST_MSIGROUP]          = UC_CHIP ## _EXTERNALSOURCES   \
                                            + UC_CHIP ## _ONCHIPSOURCES, \
                [IST_TIMER]             = UC_CHIP ## _EXTERNALSOURCES   \
                                            + UC_CHIP ## _ONCHIPSOURCES \
                                            + UC_CHIP ## _MSIGROUPSOURCES, \
                [IST_IPI]               = UC_CHIP ## _EXTERNALSOURCES   \
                                            + UC_CHIP ## _ONCHIPSOURCES \
                                            + UC_CHIP ## _MSIGROUPSOURCES \
                                            + UC_CHIP ## _TIMERSOURCES, \
                [IST_MI]                = UC_CHIP ## _EXTERNALSOURCES   \
                                            + UC_CHIP ## _ONCHIPSOURCES \
                                            + UC_CHIP ## _MSIGROUPSOURCES \
                                            + UC_CHIP ## _TIMERSOURCES  \
                                            + UC_CHIP ## _IPISOURCES,   \
                [IST_MAX]               = UC_CHIP ## _EXTERNALSOURCES   \
                                            + UC_CHIP ## _ONCHIPSOURCES \
                                            + UC_CHIP ## _MSIGROUPSOURCES \
                                            + UC_CHIP ## _TIMERSOURCES  \
                                            + UC_CHIP ## _IPISOURCES    \
                                            + UC_CHIP ## _MISOURCES,    \
        },                                                              \
}

#ifdef MPC8536
#define mpc8536_external_intr_names     default_external_intr_names
const struct e500_intr_name mpc8536_onchip_intr_names[] = {
        { ISOURCE_SATA2, "sata2" },
        { ISOURCE_USB2, "usb2" },
        { ISOURCE_USB3, "usb3" },
        { ISOURCE_SATA1, "sata1" },
        { 0, "" },
};

INTR_INFO_DECL(mpc8536, MPC8536);
#endif

#ifdef MPC8544
#define mpc8544_external_intr_names     default_external_intr_names
const struct e500_intr_name mpc8544_onchip_intr_names[] = {
        { 0, "" },
};

INTR_INFO_DECL(mpc8544, MPC8544);
#endif
#ifdef MPC8548
#define mpc8548_external_intr_names     default_external_intr_names
const struct e500_intr_name mpc8548_onchip_intr_names[] = {
        { ISOURCE_PCI1, "pci1" },
        { ISOURCE_PCI2, "pci2" },
        { 0, "" },
};

INTR_INFO_DECL(mpc8548, MPC8548);
#endif
#ifdef MPC8555
#define mpc8555_external_intr_names     default_external_intr_names
const struct e500_intr_name mpc8555_onchip_intr_names[] = {
        { ISOURCE_PCI2, "pci2" },
        { ISOURCE_CPM, "CPM" },
        { 0, "" },
};

INTR_INFO_DECL(mpc8555, MPC8555);
#endif
#ifdef MPC8568
#define mpc8568_external_intr_names     default_external_intr_names
const struct e500_intr_name mpc8568_onchip_intr_names[] = {
        { ISOURCE_QEB_LOW, "QEB low" },
        { ISOURCE_QEB_PORT, "QEB port" },
        { ISOURCE_QEB_IECC, "QEB iram ecc" },
        { ISOURCE_QEB_MUECC, "QEB ram ecc" },
        { ISOURCE_TLU1, "tlu1" },
        { ISOURCE_QEB_HIGH, "QEB high" },
        { 0, "" },
};

INTR_INFO_DECL(mpc8568, MPC8568);
#endif
#ifdef MPC8572
#define mpc8572_external_intr_names     default_external_intr_names
const struct e500_intr_name mpc8572_onchip_intr_names[] = {
        { ISOURCE_PCIEX3_MPC8572, "pcie3" },
        { ISOURCE_FEC, "fec" },
        { ISOURCE_PME_GENERAL, "pme" },
        { ISOURCE_TLU1, "tlu1" },
        { ISOURCE_TLU2, "tlu2" },
        { ISOURCE_PME_CHAN1, "pme-chan1" },
        { ISOURCE_PME_CHAN2, "pme-chan2" },
        { ISOURCE_PME_CHAN3, "pme-chan3" },
        { ISOURCE_PME_CHAN4, "pme-chan4" },
        { ISOURCE_DMA2_CHAN1, "dma2-chan1" },
        { ISOURCE_DMA2_CHAN2, "dma2-chan2" },
        { ISOURCE_DMA2_CHAN3, "dma2-chan3" },
        { ISOURCE_DMA2_CHAN4, "dma2-chan4" },
        { 0, "" },
};

INTR_INFO_DECL(mpc8572, MPC8572);
#endif

#ifdef P1025
#define p1025_external_intr_names       default_external_intr_names
const struct e500_intr_name p1025_onchip_intr_names[] = {
        { ISOURCE_PCIEX3_MPC8572, "pcie3" },
        { ISOURCE_ETSEC1_G1_TX, "etsec1-g1-tx" },
        { ISOURCE_ETSEC1_G1_RX, "etsec1-g1-rx" },
        { ISOURCE_ETSEC1_G1_ERR, "etsec1-g1-error" },
        { ISOURCE_ETSEC2_G1_TX, "etsec2-g1-tx" },
        { ISOURCE_ETSEC2_G1_RX, "etsec2-g1-rx" },
        { ISOURCE_ETSEC2_G1_ERR, "etsec2-g1-error" },
        { ISOURCE_ETSEC3_G1_TX, "etsec3-g1-tx" },
        { ISOURCE_ETSEC3_G1_RX, "etsec3-g1-rx" },
        { ISOURCE_ETSEC3_G1_ERR, "etsec3-g1-error" },
        { ISOURCE_QEB_MUECC, "qeb-low" },
        { ISOURCE_QEB_HIGH, "qeb-crit" },
        { ISOURCE_DMA2_CHAN1, "dma2-chan1" },
        { ISOURCE_DMA2_CHAN2, "dma2-chan2" },
        { ISOURCE_DMA2_CHAN3, "dma2-chan3" },
        { ISOURCE_DMA2_CHAN4, "dma2-chan4" },
        { 0, "" },
};

INTR_INFO_DECL(p1025, P1025);
#endif

#ifdef P2020
#define p20x0_external_intr_names       default_external_intr_names
const struct e500_intr_name p20x0_onchip_intr_names[] = {
        { ISOURCE_PCIEX3_MPC8572, "pcie3" },
        { ISOURCE_DMA2_CHAN1, "dma2-chan1" },
        { ISOURCE_DMA2_CHAN2, "dma2-chan2" },
        { ISOURCE_DMA2_CHAN3, "dma2-chan3" },
        { ISOURCE_DMA2_CHAN4, "dma2-chan4" },
        { 0, "" },
};

INTR_INFO_DECL(p20x0, P20x0);
#endif

#ifdef P1023
#define p1023_external_intr_names       default_external_intr_names
const struct e500_intr_name p1023_onchip_intr_names[] = {
        { ISOURCE_FMAN,            "fman" },
        { ISOURCE_MDIO,            "mdio" },
        { ISOURCE_QMAN0,           "qman0" },
        { ISOURCE_BMAN0,           "bman0" },
        { ISOURCE_QMAN1,           "qman1" },
        { ISOURCE_BMAN1,           "bman1" },
        { ISOURCE_QMAN2,           "qman2" },
        { ISOURCE_BMAN2,           "bman2" },
        { ISOURCE_SECURITY2_P1023, "sec2" },
        { ISOURCE_SEC_GENERAL,     "sec-general" },
        { ISOURCE_DMA2_CHAN1,      "dma2-chan1" },
        { ISOURCE_DMA2_CHAN2,      "dma2-chan2" },
        { ISOURCE_DMA2_CHAN3,      "dma2-chan3" },
        { ISOURCE_DMA2_CHAN4,      "dma2-chan4" },
        { 0, "" },
};

INTR_INFO_DECL(p1023, P1023);
#endif

static const char ist_names[][12] = {
        [IST_NONE] = "none",
        [IST_EDGE] = "edge",
        [IST_LEVEL_LOW] = "level-",
        [IST_LEVEL_HIGH] = "level+",
        [IST_PULSE] = "pulse",
        [IST_MSI] = "msi",
        [IST_ONCHIP] = "onchip",
        [IST_MSIGROUP] = "msigroup",
        [IST_TIMER] = "timer",
        [IST_IPI] = "ipi",
        [IST_MI] = "msgint",
};

static struct intr_source *e500_intr_sources;
static const struct intr_source *e500_intr_last_source;

static void     *e500_intr_establish(int, int, int, int (*)(void *), void *,
                    const char *);
static void     e500_intr_disestablish(void *);
static void     e500_intr_cpu_attach(struct cpu_info *ci);
static void     e500_intr_cpu_hatch(struct cpu_info *ci);
static void     e500_intr_cpu_send_ipi(cpuid_t, uintptr_t);
static void     e500_intr_init(void);
static void     e500_intr_init_precpu(void);
static const char *e500_intr_string(int, int, char *, size_t);
static const char *e500_intr_typename(int);
static void     e500_critintr(struct trapframe *tf);
static void     e500_decrintr(struct trapframe *tf);
static void     e500_extintr(struct trapframe *tf);
static void     e500_fitintr(struct trapframe *tf);
static void     e500_wdogintr(struct trapframe *tf);
static void     e500_spl0(void);
static int      e500_splraise(int);
static void     e500_splx(int);
static const char *e500_intr_all_name_lookup(int, int);

const struct intrsw e500_intrsw = {
        .intrsw_establish = e500_intr_establish,
        .intrsw_disestablish = e500_intr_disestablish,
        .intrsw_init = e500_intr_init,
        .intrsw_cpu_attach = e500_intr_cpu_attach,
        .intrsw_cpu_hatch = e500_intr_cpu_hatch,
        .intrsw_cpu_send_ipi = e500_intr_cpu_send_ipi,
        .intrsw_string = e500_intr_string,
        .intrsw_typename = e500_intr_typename,

        .intrsw_critintr = e500_critintr,
        .intrsw_decrintr = e500_decrintr,
        .intrsw_extintr = e500_extintr,
        .intrsw_fitintr = e500_fitintr,
        .intrsw_wdogintr = e500_wdogintr,

        .intrsw_splraise = e500_splraise,
        .intrsw_splx = e500_splx,
        .intrsw_spl0 = e500_spl0,

#ifdef __HAVE_FAST_SOFTINTS
        .intrsw_softint_init_md = powerpc_softint_init_md,
        .intrsw_softint_trigger = powerpc_softint_trigger,
#endif
};

static bool wdog_barked;

static inline uint32_t 
openpic_read(struct cpu_softc *cpu, bus_size_t offset)
{

        return bus_space_read_4(cpu->cpu_bst, cpu->cpu_bsh,
            OPENPIC_BASE + offset);
}

static inline void 
openpic_write(struct cpu_softc *cpu, bus_size_t offset, uint32_t val)
{

        return bus_space_write_4(cpu->cpu_bst, cpu->cpu_bsh,
            OPENPIC_BASE + offset, val);
}

static const char *
e500_intr_external_name_lookup(int irq)
{
        prop_array_t extirqs = board_info_get_object("external-irqs");
        prop_string_t irqname = prop_array_get(extirqs, irq);
        KASSERT(irqname != NULL);
        KASSERT(prop_object_type(irqname) == PROP_TYPE_STRING);

        return prop_string_value(irqname);
}

static const char *
e500_intr_name_lookup(const struct e500_intr_name *names, int irq)
{
        for (; names->in_name[0] != '\0'; names++) {
                if (names->in_irq == irq)
                        return names->in_name;
        }

        return NULL;
}

static const char *
e500_intr_onchip_name_lookup(int irq)
{
        const char *name;

        name = e500_intr_name_lookup(e500_intr_info.ii_onchip_intr_names, irq);
        if (name == NULL)
               name = e500_intr_name_lookup(e500_onchip_intr_names, irq);

        return name;
}

static inline void
e500_splset(struct cpu_info *ci, int ipl)
{
        struct cpu_softc * const cpu = ci->ci_softc;

#ifdef __HAVE_FAST_SOFTINTS /* XXX */
        KASSERT((curlwp->l_pflag & LP_INTR) == 0 || ipl != IPL_NONE);
#endif
        const u_int ctpr = IPL2CTPR(ipl);
        KASSERT(openpic_read(cpu, OPENPIC_CTPR) == IPL2CTPR(ci->ci_cpl));
        openpic_write(cpu, OPENPIC_CTPR, ctpr);
        KASSERT(openpic_read(cpu, OPENPIC_CTPR) == ctpr);
#ifdef DIAGNOSTIC
        cpu->cpu_spl_tb[ipl][ci->ci_cpl] = mftb();
#endif
        ci->ci_cpl = ipl;
}

static void
e500_spl0(void)
{
        wrtee(0);

        struct cpu_info * const ci = curcpu();

#ifdef __HAVE_FAST_SOFTINTS
        if (__predict_false(ci->ci_data.cpu_softints != 0)) {
                e500_splset(ci, IPL_HIGH);
                wrtee(PSL_EE);
                powerpc_softint(ci, IPL_NONE,
                    (vaddr_t)__builtin_return_address(0));
                wrtee(0);
        }
#endif /* __HAVE_FAST_SOFTINTS */
        e500_splset(ci, IPL_NONE);

        wrtee(PSL_EE);
}

static void
e500_splx(int ipl)
{
        struct cpu_info * const ci = curcpu();
        const int old_ipl = ci->ci_cpl;

        /* if we panicked because of watchdog, PSL_CE will be clear.  */
        KASSERT(wdog_barked || (mfmsr() & PSL_CE));

        if (ipl == old_ipl)
                return;

        if (__predict_false(ipl > old_ipl)) {
                printf("%s: %p: cpl=%u: ignoring splx(%u) to raise ipl\n",
                    __func__, __builtin_return_address(0), old_ipl, ipl);
                if (old_ipl == IPL_NONE)
                        console_debugger();
        }

        // const
        register_t msr = wrtee(0);
#ifdef __HAVE_FAST_SOFTINTS
        const u_int softints = ci->ci_data.cpu_softints & (IPL_SOFTMASK << ipl);
        if (__predict_false(softints != 0)) {
                e500_splset(ci, IPL_HIGH);
                wrtee(msr);
                powerpc_softint(ci, ipl,
                    (vaddr_t)__builtin_return_address(0));
                wrtee(0);
        }
#endif /* __HAVE_FAST_SOFTINTS */
        e500_splset(ci, ipl);
#if 1
        if (ipl < IPL_VM && old_ipl >= IPL_VM)
                msr = PSL_EE;
#endif
        wrtee(msr);
}

static int
e500_splraise(int ipl)
{
        struct cpu_info * const ci = curcpu();
        const int old_ipl = ci->ci_cpl;

        /* if we panicked because of watchdog, PSL_CE will be clear.  */
        KASSERT(wdog_barked || (mfmsr() & PSL_CE));

        if (old_ipl < ipl) {
                //const
                register_t msr = wrtee(0);
                e500_splset(ci, ipl);
#if 0
                if (old_ipl < IPL_VM && ipl >= IPL_VM)
                        msr = 0;
#endif
                wrtee(msr);
        }
#if 0
        else if (ipl == IPL_NONE) {
                panic("%s: %p: cpl=%u: attempt to splraise(IPL_NONE)",
                    __func__, __builtin_return_address(0), old_ipl);
        } else if (old_ipl > ipl) {
                printf("%s: %p: cpl=%u: ignoring splraise(%u) to lower ipl\n",
                    __func__, __builtin_return_address(0), old_ipl, ipl);
        }
#endif

        return old_ipl;
}

static int
e500_intr_spurious(void *arg)
{
        return 0;
}

static bool
e500_intr_irq_info_get(struct cpu_info *ci, u_int irq, int ipl, int ist,
        struct e500_intr_irq_info *ii)
{
        const struct e500_intr_info * const info = &e500_intr_info;
        bool ok;

#if DEBUG > 2
        printf("%s(%p,irq=%u,ipl=%u,ist=%u,%p)\n", __func__, ci, irq, ipl, ist, ii);
#endif

        if (ipl < IPL_VM || ipl > IPL_HIGH) {
#if DEBUG > 2
                printf("%s:%d ipl=%u\n", __func__, __LINE__, ipl);
#endif
                return false;
        }

        if (ist <= IST_NONE || ist >= IST_MAX) {
#if DEBUG > 2
                printf("%s:%d ist=%u\n", __func__, __LINE__, ist);
#endif
                return false;
        }

        ii->irq_vector = irq + info->ii_ist_vectors[ist];
        if (IST_PERCPU_P(ist) && ist != IST_IPI)
                ii->irq_vector += ci->ci_cpuid * info->ii_percpu_sources;

        switch (ist) {
        default:
                ii->irq_vpr = OPENPIC_EIVPR(irq);
                ii->irq_dr  = OPENPIC_EIDR(irq);
                ok = irq < info->ii_external_sources
                    && (ist == IST_EDGE
                        || ist == IST_LEVEL_LOW
                        || ist == IST_LEVEL_HIGH);
                break;
        case IST_PULSE:
                ok = false;
                break;
        case IST_ONCHIP:
                ii->irq_vpr = OPENPIC_IIVPR(irq);
                ii->irq_dr  = OPENPIC_IIDR(irq);
                ok = irq < 32 * __arraycount(info->ii_onchip_bitmap);
#if DEBUG > 2
                printf("%s: irq=%u: ok=%u\n", __func__, irq, ok);
#endif
                ok = ok && (info->ii_onchip_bitmap[irq/32] & (1 << (irq & 31)));
#if DEBUG > 2
                printf("%s: %08x%08x -> %08x%08x: ok=%u\n", __func__,
                    irq < 32 ? 0 : (1 << irq), irq < 32 ? (1 << irq) : 0,
                    info->ii_onchip_bitmap[1], info->ii_onchip_bitmap[0],
                    ok);
#endif
                break;
        case IST_MSIGROUP:
                ii->irq_vpr = OPENPIC_MSIVPR(irq);
                ii->irq_dr  = OPENPIC_MSIDR(irq);
                ok = irq < info->ii_msigroup_sources
                    && ipl == IPL_VM;
                break;
        case IST_TIMER:
                ii->irq_vpr = OPENPIC_GTVPR(ci->ci_cpuid, irq);
                ii->irq_dr  = OPENPIC_GTDR(ci->ci_cpuid, irq);
                ok = irq < info->ii_timer_sources;
#if DEBUG > 2
                printf("%s: IST_TIMER irq=%u: ok=%u\n", __func__, irq, ok);
#endif
                break;
        case IST_IPI:
                ii->irq_vpr = OPENPIC_IPIVPR(irq);
                ii->irq_dr  = OPENPIC_IPIDR(irq);
                ok = irq < info->ii_ipi_sources;
                break;
        case IST_MI:
                ii->irq_vpr = OPENPIC_MIVPR(irq);
                ii->irq_dr  = OPENPIC_MIDR(irq);
                ok = irq < info->ii_mi_sources;
                break;
        }

        return ok;
}

static const char *
e500_intr_string(int irq, int ist, char *buf, size_t len)
{
        struct cpu_info * const ci = curcpu();
        struct cpu_softc * const cpu = ci->ci_softc;
        struct e500_intr_irq_info ii;

        if (!e500_intr_irq_info_get(ci, irq, IPL_VM, ist, &ii))
                return NULL;

        strlcpy(buf, cpu->cpu_evcnt_intrs[ii.irq_vector].ev_name, len);
        return buf;
}

__CTASSERT(__arraycount(ist_names) == IST_MAX);

static const char *
e500_intr_typename(int ist)
{
        if (IST_NONE <= ist && ist < IST_MAX)
                return ist_names[ist];

        return NULL;
}

static void *
e500_intr_cpu_establish(struct cpu_info *ci, int irq, int ipl, int ist,
        int (*handler)(void *), void *arg, const char *xname)
{
        struct cpu_softc * const cpu = ci->ci_softc;
        struct e500_intr_irq_info ii;

        KASSERT(ipl >= IPL_VM && ipl <= IPL_HIGH);
        KASSERT(ist > IST_NONE && ist < IST_MAX && ist != IST_MSI);

        if (!e500_intr_irq_info_get(ci, irq, ipl, ist, &ii)) {
                printf("%s: e500_intr_irq_info_get(%p,%u,%u,%u,%p) failed\n",
                    __func__, ci, irq, ipl, ist, &ii);
                return NULL;
        }

        if (xname == NULL) {
                xname = e500_intr_all_name_lookup(irq, ist);
                if (xname == NULL)
                        xname = "unknown";
        }

        struct intr_source * const is = &e500_intr_sources[ii.irq_vector];
        mutex_enter(&e500_intr_lock);
        if (is->is_ipl != IPL_NONE) {
                /* XXX IPI0 is shared by all CPU. */
                if (is->is_ist != IST_IPI ||
                    is->is_irq != irq ||
                    is->is_ipl != ipl ||
                    is->is_ist != ist ||
                    is->is_func != handler ||
                    is->is_arg != arg) {
                        mutex_exit(&e500_intr_lock);
                        return NULL;
                }
        }

        is->is_func = handler;
        is->is_arg = arg;
        is->is_ipl = ipl;
        is->is_ist = ist;
        is->is_irq = irq;
        is->is_refcnt++;
        is->is_vpr = ii.irq_vpr;
        is->is_dr = ii.irq_dr;
        switch (ist) {
        case IST_EDGE:
        case IST_LEVEL_LOW:
        case IST_LEVEL_HIGH:
                snprintf(is->is_source, sizeof(is->is_source), "extirq %d",
                    irq);
                break;
        case IST_ONCHIP:
                snprintf(is->is_source, sizeof(is->is_source), "irq %d", irq);
                break;
        case IST_MSIGROUP:
                snprintf(is->is_source, sizeof(is->is_source), "msigroup %d",
                    irq);
                break;
        case IST_TIMER:
                snprintf(is->is_source, sizeof(is->is_source), "timer %d", irq);
                break;
        case IST_IPI:
                snprintf(is->is_source, sizeof(is->is_source), "ipi %d", irq);
                break;
        case IST_MI:
                snprintf(is->is_source, sizeof(is->is_source), "mi %d", irq);
                break;
        case IST_PULSE:
        default:
                panic("%s: invalid ist (%d)\n", __func__, ist);
        }
        strlcpy(is->is_xname, xname, sizeof(is->is_xname));

        uint32_t vpr = VPR_PRIORITY_MAKE(IPL2CTPR(ipl))
            | VPR_VECTOR_MAKE(((ii.irq_vector + 1) << 4) | ipl)
            | (ist == IST_LEVEL_LOW
                ? VPR_LEVEL_LOW
                : (ist == IST_LEVEL_HIGH
                    ? VPR_LEVEL_HIGH
                    : (ist == IST_ONCHIP
                      ? VPR_P_HIGH
                      : 0)));

        /*
         * All interrupts go to the primary except per-cpu interrupts which get
         * routed to the appropriate cpu.
         */
        uint32_t dr = openpic_read(cpu, ii.irq_dr);

        dr |= 1 << (IST_PERCPU_P(ist) ? ci->ci_cpuid : 0);

        /*
         * Update the vector/priority and destination registers keeping the
         * interrupt masked.
         */
        const register_t msr = wrtee(0);        /* disable interrupts */
        openpic_write(cpu, ii.irq_vpr, vpr | VPR_MSK);
        openpic_write(cpu, ii.irq_dr, dr);

        /*
         * Now unmask the interrupt.
         */
        openpic_write(cpu, ii.irq_vpr, vpr);

        wrtee(msr);                             /* re-enable interrupts */

        mutex_exit(&e500_intr_lock);

        return is;
}

static void *
e500_intr_establish(int irq, int ipl, int ist, int (*handler)(void *),
    void *arg, const char *xname)
{
        return e500_intr_cpu_establish(curcpu(), irq, ipl, ist, handler, arg,
            xname);
}

static void
e500_intr_disestablish(void *vis)
{
        struct cpu_softc * const cpu = curcpu()->ci_softc;
        struct intr_source * const is = vis;
        struct e500_intr_irq_info ii;

        KASSERT(e500_intr_sources <= is);
        KASSERT(is < e500_intr_last_source);
        KASSERT(!cpu_intr_p());

        bool ok = e500_intr_irq_info_get(curcpu(), is->is_irq, is->is_ipl,
            is->is_ist, &ii);
        (void)ok;       /* appease gcc */
        KASSERT(ok);
        KASSERT(is - e500_intr_sources == ii.irq_vector);

        mutex_enter(&e500_intr_lock);

        if (is->is_refcnt-- > 1) {
                mutex_exit(&e500_intr_lock);
                return;
        }

        /*
         * Mask the source using the mask (MSK) bit in the vector/priority reg.
         */
        uint32_t vpr = openpic_read(cpu, ii.irq_vpr);
        openpic_write(cpu, ii.irq_vpr, VPR_MSK | vpr);

        /*
         * Wait for the Activity (A) bit for the source to be cleared.
         */
        while (openpic_read(cpu, ii.irq_vpr) & VPR_A)
                ;

        /*
         * Now the source can be modified.
         */
        openpic_write(cpu, ii.irq_dr, 0);               /* stop delivery */
        openpic_write(cpu, ii.irq_vpr, VPR_MSK);        /* mask/reset it */

        *is = (struct intr_source)INTR_SOURCE_INITIALIZER;

        mutex_exit(&e500_intr_lock);
}

static void
e500_critintr(struct trapframe *tf)
{
        panic("%s: srr0/srr1=%#lx/%#lx", __func__, tf->tf_srr0, tf->tf_srr1);
}

static void
e500_decrintr(struct trapframe *tf)
{
        panic("%s: srr0/srr1=%#lx/%#lx", __func__, tf->tf_srr0, tf->tf_srr1);
}

static void
e500_fitintr(struct trapframe *tf)
{
        panic("%s: srr0/srr1=%#lx/%#lx", __func__, tf->tf_srr0, tf->tf_srr1);
}

static void
e500_wdogintr(struct trapframe *tf)
{
        struct cpu_info * const ci = curcpu();
        mtspr(SPR_TSR, TSR_ENW|TSR_WIS);
        wdog_barked = true;
        dump_splhist(ci, NULL);
        dump_trapframe(tf, NULL);
        panic("%s: tf=%p tb=%"PRId64" srr0/srr1=%#lx/%#lx"
            " cpl=%d idepth=%d, mtxcount=%d",
            __func__, tf, mftb(), tf->tf_srr0, tf->tf_srr1,
            ci->ci_cpl, ci->ci_idepth, ci->ci_mtx_count);
}

static void
e500_extintr(struct trapframe *tf)
{
        struct cpu_info * const ci = curcpu();
        struct cpu_softc * const cpu = ci->ci_softc;
        const int old_ipl = ci->ci_cpl;

        /* if we panicked because of watchdog, PSL_CE will be clear.  */
        KASSERT(wdog_barked || (mfmsr() & PSL_CE));

#if 0
//      printf("%s(%p): idepth=%d enter\n", __func__, tf, ci->ci_idepth);
        if ((register_t)tf >= (register_t)curlwp->l_addr + USPACE              
            || (register_t)tf < (register_t)curlwp->l_addr + NBPG) {            
                printf("%s(entry): pid %d.%d (%s): srr0/srr1=%#lx/%#lx: invalid tf addr %p\n",  
                    __func__, curlwp->l_proc->p_pid, curlwp->l_lid,
                    curlwp->l_proc->p_comm, tf->tf_srr0, tf->tf_srr1, tf);
        }       
#endif


        ci->ci_data.cpu_nintr++;
        tf->tf_cf.cf_idepth = ci->ci_idepth++;
        cpu->cpu_pcpls[ci->ci_idepth] = old_ipl;
#if 1
        if (mfmsr() & PSL_EE)
                panic("%s(%p): MSR[EE] is on (%#lx)!", __func__, tf, mfmsr());
        if (old_ipl == IPL_HIGH
            || IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR))
                panic("%s(%p): old_ipl(%u) == IPL_HIGH(%u) "
                    "|| old_ipl + %u != OPENPIC_CTPR (%u)",
                    __func__, tf, old_ipl, IPL_HIGH,
                    15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR));
#else
        if (old_ipl >= IPL_VM)
                panic("%s(%p): old_ipl(%u) >= IPL_VM(%u) CTPR=%u",
                    __func__, tf, old_ipl, IPL_VM, openpic_read(cpu, OPENPIC_CTPR));
#endif

        for (;;) {
                /*
                 * Find out the pending interrupt.
                 */
                KASSERTMSG((mfmsr() & PSL_EE) == 0,
                    "%s(%p): MSR[EE] left on (%#lx)!", __func__, tf, mfmsr());
                if (IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR))
                        panic("%s(%p): %d: old_ipl(%u) + %u != OPENPIC_CTPR (%u)",
                            __func__, tf, __LINE__, old_ipl, 
                            15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR));
                const uint32_t iack = openpic_read(cpu, OPENPIC_IACK);
                const int ipl = iack & 0xf;
                const int irq = (iack >> 4) - 1;
#if 0
                printf("%s: iack=%d ipl=%d irq=%d <%s>\n",
                    __func__, iack, ipl, irq,
                    (iack != IRQ_SPURIOUS ?
                        cpu->cpu_evcnt_intrs[irq].ev_name : "spurious"));
#endif
                if (IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR))
                        panic("%s(%p): %d: old_ipl(%u) + %u != OPENPIC_CTPR (%u)",
                            __func__, tf, __LINE__, old_ipl, 
                            15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR));
                if (iack == IRQ_SPURIOUS)
                        break;
                
                struct intr_source * const is = &e500_intr_sources[irq];
                if (__predict_true(is < e500_intr_last_source)) {
                        /*
                         * Timer interrupts get their argument overridden with
                         * the pointer to the trapframe.
                         */
                        KASSERTMSG(is->is_ipl == ipl,
                            "iack %#x: is %p: irq %d ipl %d != iack ipl %d",
                            iack, is, irq, is->is_ipl, ipl);
                        void *arg = (is->is_ist == IST_TIMER ? tf : is->is_arg);
                        if (is->is_ipl <= old_ipl)
                                panic("%s(%p): %s (%u): is->is_ipl (%u) <= old_ipl (%u)\n",
                                    __func__, tf,
                                    cpu->cpu_evcnt_intrs[irq].ev_name, irq,
                                    is->is_ipl, old_ipl);
                        KASSERT(is->is_ipl > old_ipl);
                        e500_splset(ci, is->is_ipl);    /* change IPL */
                        if (__predict_false(is->is_func == NULL)) {
                                aprint_error_dev(ci->ci_dev,
                                    "interrupt from unestablished irq %d\n",
                                    irq);
                        } else {
                                int (*func)(void *) = is->is_func;
                                wrtee(PSL_EE);
                                int rv = (*func)(arg);
                                wrtee(0);
#if DEBUG > 2
                                printf("%s: %s handler %p(%p) returned %d\n",
                                    __func__,
                                    cpu->cpu_evcnt_intrs[irq].ev_name,
                                    func, arg, rv);
#endif
                                if (rv == 0)
                                        cpu->cpu_evcnt_spurious_intr.ev_count++;
                        }
                        e500_splset(ci, old_ipl);       /* restore IPL */
                        cpu->cpu_evcnt_intrs[irq].ev_count++;
                } else {
                        aprint_error_dev(ci->ci_dev,
                            "interrupt from illegal irq %d\n", irq);
                        cpu->cpu_evcnt_spurious_intr.ev_count++;
                }
                /*
                 * If this is a nested interrupt, simply ack it and exit
                 * because the loop we interrupted will complete looking
                 * for interrupts.
                 */
                KASSERTMSG((mfmsr() & PSL_EE) == 0,
                    "%s(%p): MSR[EE] left on (%#lx)!", __func__, tf, mfmsr());
                if (IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR))
                        panic("%s(%p): %d: old_ipl(%u) + %u != OPENPIC_CTPR (%u)",
                            __func__, tf, __LINE__, old_ipl, 
                            15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR));

                openpic_write(cpu, OPENPIC_EOI, 0);
                if (IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR))
                        panic("%s(%p): %d: old_ipl(%u) + %u != OPENPIC_CTPR (%u)",
                            __func__, tf, __LINE__, old_ipl, 
                            15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR));
                if (ci->ci_idepth > 0)
                        break;
        }

        ci->ci_idepth--;

#ifdef __HAVE_FAST_SOFTINTS
        /*
         * Before exiting, deal with any softints that need to be dealt with.
         */
        const u_int softints = ci->ci_data.cpu_softints & (IPL_SOFTMASK << old_ipl);
        if (__predict_false(softints != 0)) {
                KASSERT(old_ipl < IPL_VM);
                e500_splset(ci, IPL_HIGH);      /* pop to high */
                wrtee(PSL_EE);                  /* reenable interrupts */
                powerpc_softint(ci, old_ipl,    /* deal with them */
                    tf->tf_srr0);
                wrtee(0);                       /* disable interrupts */
                e500_splset(ci, old_ipl);       /* and drop back */
        }
#endif /* __HAVE_FAST_SOFTINTS */
        KASSERT(ci->ci_cpl == old_ipl);

        /*
         * If we interrupted while power-saving and we need to exit idle,
         * we need to clear PSL_POW so we won't go back into power-saving.
         */
        if (__predict_false(tf->tf_srr1 & PSL_POW) && ci->ci_want_resched)
                tf->tf_srr1 &= ~PSL_POW;

//      printf("%s(%p): idepth=%d exit\n", __func__, tf, ci->ci_idepth);
}

static void
e500_intr_init(void)
{
        struct cpu_info * const ci = curcpu();
        struct cpu_softc * const cpu = ci->ci_softc;
        const uint32_t frr = openpic_read(cpu, OPENPIC_FRR);
        const u_int nirq = FRR_NIRQ_GET(frr) + 1;
//      const u_int ncpu = FRR_NCPU_GET(frr) + 1;
        struct intr_source *is;
        struct e500_intr_info * const ii = &e500_intr_info;

        const uint16_t svr = (mfspr(SPR_SVR) & ~0x80000) >> 16;
        switch (svr) {
#ifdef MPC8536
        case SVR_MPC8536v1 >> 16:
                *ii = mpc8536_intr_info;
                break;
#endif
#ifdef MPC8544
        case SVR_MPC8544v1 >> 16:
                *ii = mpc8544_intr_info;
                break;
#endif
#ifdef MPC8548
        case SVR_MPC8543v1 >> 16:
        case SVR_MPC8548v1 >> 16:
                *ii = mpc8548_intr_info;
                break;
#endif
#ifdef MPC8555
        case SVR_MPC8541v1 >> 16:
        case SVR_MPC8555v1 >> 16:
                *ii = mpc8555_intr_info;
                break;
#endif
#ifdef MPC8568
        case SVR_MPC8568v1 >> 16:
                *ii = mpc8568_intr_info;
                break;
#endif
#ifdef MPC8572
        case SVR_MPC8572v1 >> 16:
                *ii = mpc8572_intr_info;
                break;
#endif
#ifdef P1023
        case SVR_P1017v1 >> 16:
        case SVR_P1023v1 >> 16:
                *ii = p1023_intr_info;
                break;
#endif
#ifdef P1025
        case SVR_P1016v1 >> 16:
        case SVR_P1025v1 >> 16:
                *ii = p1025_intr_info;
                break;
#endif
#ifdef P2020
        case SVR_P2010v2 >> 16:
        case SVR_P2020v2 >> 16:
                *ii = p20x0_intr_info;
                break;
#endif
        default:
                panic("%s: don't know how to deal with SVR %#jx",
                    __func__, (uintmax_t)mfspr(SPR_SVR));
        }

        /*
         * Initialize interrupt handler lock
         */
        mutex_init(&e500_intr_lock, MUTEX_DEFAULT, IPL_HIGH);

        /*
         * We need to be in mixed mode.
         */
        openpic_write(cpu, OPENPIC_GCR, GCR_M);

        /*
         * Make we and the openpic both agree about the current SPL level.
         */
        e500_splset(ci, ci->ci_cpl);

        /*
         * Allow the required number of interrupt sources.
         */
        is = kmem_zalloc(nirq * sizeof(*is), KM_SLEEP);
        e500_intr_sources = is;
        e500_intr_last_source = is + nirq;

        /*
         * Initialize all the external interrupts as active low.
         */
        for (u_int irq = 0; irq < e500_intr_info.ii_external_sources; irq++) { 
                openpic_write(cpu, OPENPIC_EIVPR(irq),
                    VPR_VECTOR_MAKE(irq) | VPR_LEVEL_LOW);
        }
}

static void
e500_intr_init_precpu(void)
{
        struct cpu_info const *ci = curcpu();
        struct cpu_softc * const cpu = ci->ci_softc;
        bus_addr_t dr;

        /*
         * timer's DR is set to be delivered to cpu0 as initial value.
         */
        for (u_int irq = 0; irq < e500_intr_info.ii_timer_sources; irq++) { 
                dr = OPENPIC_GTDR(ci->ci_cpuid, irq);
                openpic_write(cpu, dr, 0);      /* stop delivery */
        }
}

static void
e500_idlespin(void)
{
        KASSERTMSG(curcpu()->ci_cpl == IPL_NONE,
            "%s: cpu%u: ci_cpl (%d) != 0", __func__, cpu_number(),
             curcpu()->ci_cpl);
        KASSERTMSG(CTPR2IPL(openpic_read(curcpu()->ci_softc, OPENPIC_CTPR)) == IPL_NONE,
            "%s: cpu%u: CTPR (%d) != IPL_NONE", __func__, cpu_number(),
             CTPR2IPL(openpic_read(curcpu()->ci_softc, OPENPIC_CTPR)));
        KASSERT(mfmsr() & PSL_EE);

        if (powersave > 0)
                mtmsr(mfmsr() | PSL_POW);
}

static void
e500_intr_cpu_attach(struct cpu_info *ci)
{
        struct cpu_softc * const cpu = ci->ci_softc;
        const char * const xname = device_xname(ci->ci_dev);

        const u_int32_t frr = openpic_read(cpu, OPENPIC_FRR);
        const u_int nirq = FRR_NIRQ_GET(frr) + 1;
//      const u_int ncpu = FRR_NCPU_GET(frr) + 1;

        const struct e500_intr_info * const info = &e500_intr_info;

        cpu->cpu_clock_gtbcr = OPENPIC_GTBCR(ci->ci_cpuid, E500_CLOCK_TIMER);

        cpu->cpu_evcnt_intrs =
            kmem_zalloc(nirq * sizeof(cpu->cpu_evcnt_intrs[0]), KM_SLEEP);

        struct evcnt *evcnt = cpu->cpu_evcnt_intrs;
        for (size_t j = 0; j < info->ii_external_sources; j++, evcnt++) {
                const char *name = e500_intr_external_name_lookup(j);
                evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, NULL, xname, name);
        }
        KASSERT(evcnt == cpu->cpu_evcnt_intrs + info->ii_ist_vectors[IST_ONCHIP]);
        for (size_t j = 0; j < info->ii_onchip_sources; j++, evcnt++) {
                if (info->ii_onchip_bitmap[j / 32] & __BIT(j & 31)) {
                        const char *name = e500_intr_onchip_name_lookup(j);
                        if (name != NULL) {
                                evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR,
                                    NULL, xname, name);
#ifdef DIAGNOSTIC
                        } else {
                                printf("%s: missing evcnt for onchip irq %zu\n",
                                    __func__, j);
#endif
                        }
                }
        }

        KASSERT(evcnt == cpu->cpu_evcnt_intrs + info->ii_ist_vectors[IST_MSIGROUP]);
        for (size_t j = 0; j < info->ii_msigroup_sources; j++, evcnt++) {
                evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR,
                    NULL, xname, e500_msigroup_intr_names[j].in_name);
        }

        KASSERT(evcnt == cpu->cpu_evcnt_intrs + info->ii_ist_vectors[IST_TIMER]);
        evcnt += ci->ci_cpuid * info->ii_percpu_sources;
        for (size_t j = 0; j < info->ii_timer_sources; j++, evcnt++) {
                evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR,
                    NULL, xname, e500_timer_intr_names[j].in_name);
        }

        for (size_t j = 0; j < info->ii_ipi_sources; j++, evcnt++) {
                evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR,
                    NULL, xname, e500_ipi_intr_names[j].in_name);
        }

        for (size_t j = 0; j < info->ii_mi_sources; j++, evcnt++) {
                evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR,
                    NULL, xname, e500_mi_intr_names[j].in_name);
        }

        ci->ci_idlespin = e500_idlespin;
}

static void
e500_intr_cpu_send_ipi(cpuid_t target, uint32_t ipimsg)
{
        struct cpu_info * const ci = curcpu();
        struct cpu_softc * const cpu = ci->ci_softc;
        uint32_t dstmask;

        if (target >= CPU_MAXNUM) {
                CPU_INFO_ITERATOR cii;
                struct cpu_info *dst_ci;

                KASSERT(target == IPI_DST_NOTME || target == IPI_DST_ALL);

                dstmask = 0;
                for (CPU_INFO_FOREACH(cii, dst_ci)) {
                        if (target == IPI_DST_ALL || ci != dst_ci) {
                                dstmask |= 1 << cpu_index(ci);
                                if (ipimsg)
                                        atomic_or_32(&dst_ci->ci_pending_ipis,
                                            ipimsg);
                        }
                }
        } else {
                struct cpu_info * const dst_ci = cpu_lookup(target);
                KASSERT(dst_ci != NULL);
                KASSERTMSG(target == cpu_index(dst_ci),
                    "%s: target (%lu) != cpu_index(cpu%u)",
                     __func__, target, cpu_index(dst_ci));
                dstmask = (1 << target);
                if (ipimsg)
                        atomic_or_32(&dst_ci->ci_pending_ipis, ipimsg);
        }

        openpic_write(cpu, OPENPIC_IPIDR(0), dstmask);
}

typedef void (*ipifunc_t)(void);

#ifdef __HAVE_PREEMPTION
static void
e500_ipi_kpreempt(void)
{
        poowerpc_softint_trigger(1 << IPL_NONE);
}
#endif

static void
e500_ipi_suspend(void)
{

#ifdef MULTIPROCESSOR
        cpu_pause(NULL);
#endif  /* MULTIPROCESSOR */
}

static void
e500_ipi_ast(void)
{
        curcpu()->ci_onproc->l_md.md_astpending = 1;
}

static const ipifunc_t e500_ipifuncs[] = {
        [ilog2(IPI_XCALL)] =    xc_ipi_handler,
        [ilog2(IPI_GENERIC)] =  ipi_cpu_handler,
        [ilog2(IPI_HALT)] =     e500_ipi_halt,
#ifdef __HAVE_PREEMPTION
        [ilog2(IPI_KPREEMPT)] = e500_ipi_kpreempt,
#endif
        [ilog2(IPI_TLB1SYNC)] = e500_tlb1_sync,
        [ilog2(IPI_SUSPEND)] =  e500_ipi_suspend,
        [ilog2(IPI_AST)] =      e500_ipi_ast,
};

static int
e500_ipi_intr(void *v)
{
        struct cpu_info * const ci = curcpu();

        ci->ci_ev_ipi.ev_count++;

        uint32_t pending_ipis = atomic_swap_32(&ci->ci_pending_ipis, 0);
        for (u_int ipi = 31; pending_ipis != 0; ipi--, pending_ipis <<= 1) {
                const u_int bits = __builtin_clz(pending_ipis);
                ipi -= bits;
                pending_ipis <<= bits;
                KASSERT(e500_ipifuncs[ipi] != NULL);
                (*e500_ipifuncs[ipi])();
        }
        
        return 1;
}

static void
e500_intr_cpu_hatch(struct cpu_info *ci)
{
        char iname[INTRIDBUF];

        /* Initialize percpu interrupts. */
        e500_intr_init_precpu();

        /*
         * Establish clock interrupt for this CPU.
         */
        snprintf(iname, sizeof(iname), "%s clock", device_xname(ci->ci_dev));
        if (e500_intr_cpu_establish(ci, E500_CLOCK_TIMER, IPL_CLOCK, IST_TIMER,
            e500_clock_intr, NULL, iname) == NULL)
                panic("%s: failed to establish clock interrupt!", __func__);

        /*
         * Establish the IPI interrupts for this CPU.
         */
        if (e500_intr_cpu_establish(ci, 0, IPL_VM, IST_IPI, e500_ipi_intr,
            NULL, "ipi") == NULL)
                panic("%s: failed to establish ipi interrupt!", __func__);

        /*
         * Enable watchdog interrupts.
         */
        uint32_t tcr = mfspr(SPR_TCR);
        tcr |= TCR_WIE;
        mtspr(SPR_TCR, tcr);
}

static const char *
e500_intr_all_name_lookup(int irq, int ist)
{
        const struct e500_intr_info * const info = &e500_intr_info;

        switch (ist) {
        default:
                if (irq < info->ii_external_sources &&
                    (ist == IST_EDGE ||
                     ist == IST_LEVEL_LOW ||
                     ist == IST_LEVEL_HIGH))
                        return e500_intr_name_lookup(
                            info->ii_external_intr_names, irq);
                break;

        case IST_PULSE:
                break;

        case IST_ONCHIP:
                if (irq < info->ii_onchip_sources)
                        return e500_intr_onchip_name_lookup(irq);
                break;

        case IST_MSIGROUP:
                if (irq < info->ii_msigroup_sources)
                        return e500_intr_name_lookup(e500_msigroup_intr_names,
                            irq);
                break;

        case IST_TIMER:
                if (irq < info->ii_timer_sources)
                        return e500_intr_name_lookup(e500_timer_intr_names,
                            irq);
                break;

        case IST_IPI:
                if (irq < info->ii_ipi_sources)
                        return e500_intr_name_lookup(e500_ipi_intr_names, irq);
                break;

        case IST_MI:
                if (irq < info->ii_mi_sources)
                        return e500_intr_name_lookup(e500_mi_intr_names, irq);
                break;
        }

        return NULL;
}

static void
e500_intr_get_affinity(struct intr_source *is, kcpuset_t *cpuset)
{
        struct cpu_info * const ci = curcpu();
        struct cpu_softc * const cpu = ci->ci_softc;
        struct e500_intr_irq_info ii;

        kcpuset_zero(cpuset);

        if (is->is_ipl != IPL_NONE && !IST_PERCPU_P(is->is_ist)) {
                if (e500_intr_irq_info_get(ci, is->is_irq, is->is_ipl,
                    is->is_ist, &ii)) {
                        uint32_t dr = openpic_read(cpu, ii.irq_dr);
                        while (dr != 0) {
                                u_int n = ffs(dr);
                                if (n-- == 0)
                                        break;
                                dr &= ~(1 << n);
                                kcpuset_set(cpuset, n);
                        }
                }
        }
}

static int
e500_intr_set_affinity(struct intr_source *is, const kcpuset_t *cpuset)
{
        struct cpu_info * const ci = curcpu();
        struct cpu_softc * const cpu = ci->ci_softc;
        struct e500_intr_irq_info ii;
        uint32_t ecpuset, tcpuset;

        KASSERT(mutex_owned(&cpu_lock));
        KASSERT(mutex_owned(&e500_intr_lock));
        KASSERT(!kcpuset_iszero(cpuset));

        kcpuset_export_u32(cpuset, &ecpuset, sizeof(ecpuset));
        tcpuset = ecpuset;
        while (tcpuset != 0) {
                u_int cpu_idx = ffs(tcpuset);
                if (cpu_idx-- == 0)
                        break;

                tcpuset &= ~(1 << cpu_idx);
                struct cpu_info * const newci = cpu_lookup(cpu_idx);
                if (newci == NULL)
                        return EINVAL;
                if ((newci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0)
                        return EINVAL;
        }

        if (!e500_intr_irq_info_get(ci, is->is_irq, is->is_ipl, is->is_ist,
            &ii))
                return ENXIO;

        /*
         * Update the vector/priority and destination registers keeping the
         * interrupt masked.
         */
        const register_t msr = wrtee(0);        /* disable interrupts */

        uint32_t vpr = openpic_read(cpu, ii.irq_vpr);
        openpic_write(cpu, ii.irq_vpr, vpr | VPR_MSK);

        /*
         * Wait for the Activity (A) bit for the source to be cleared.
         */
        while (openpic_read(cpu, ii.irq_vpr) & VPR_A)
                continue;

        /*
         * Update destination register
         */
        openpic_write(cpu, ii.irq_dr, ecpuset);

        /*
         * Now unmask the interrupt.
         */
        openpic_write(cpu, ii.irq_vpr, vpr);

        wrtee(msr);                             /* re-enable interrupts */

        return 0;
}

static bool
e500_intr_is_affinity_intrsource(struct intr_source *is,
    const kcpuset_t *cpuset)
{
        struct cpu_info * const ci = curcpu();
        struct cpu_softc * const cpu = ci->ci_softc;
        struct e500_intr_irq_info ii;
        bool result = false;

        if (is->is_ipl != IPL_NONE && !IST_PERCPU_P(is->is_ist)) {
                if (e500_intr_irq_info_get(ci, is->is_irq, is->is_ipl,
                    is->is_ist, &ii)) {
                        uint32_t dr = openpic_read(cpu, ii.irq_dr);
                        while (dr != 0 && !result) {
                                u_int n = ffs(dr);
                                if (n-- == 0)
                                        break;
                                dr &= ~(1 << n);
                                result = kcpuset_isset(cpuset, n);
                        }
                }
        }
        return result;
}

static struct intr_source *
e500_intr_get_source(const char *intrid)
{
        struct intr_source *is;

        mutex_enter(&e500_intr_lock);
        for (is = e500_intr_sources; is < e500_intr_last_source; ++is) {
                if (is->is_source[0] == '\0')
                        continue;

                if (!strncmp(intrid, is->is_source, sizeof(is->is_source) - 1))
                        break;
        }
        if (is == e500_intr_last_source)
                is = NULL;
        mutex_exit(&e500_intr_lock);
        return is;
}

uint64_t
interrupt_get_count(const char *intrid, u_int cpu_idx)
{
        struct cpu_info * const ci = cpu_lookup(cpu_idx);
        struct cpu_softc * const cpu = ci->ci_softc;
        struct intr_source *is;
        struct e500_intr_irq_info ii;

        is = e500_intr_get_source(intrid);
        if (is == NULL)
                return 0;

        if (e500_intr_irq_info_get(ci, is->is_irq, is->is_ipl, is->is_ist, &ii))
                return cpu->cpu_evcnt_intrs[ii.irq_vector].ev_count;
        return 0;
}

void
interrupt_get_assigned(const char *intrid, kcpuset_t *cpuset)
{
        struct intr_source *is;

        kcpuset_zero(cpuset);

        is = e500_intr_get_source(intrid);
        if (is == NULL)
                return;

        mutex_enter(&e500_intr_lock);
        e500_intr_get_affinity(is, cpuset);
        mutex_exit(&e500_intr_lock);
}

void
interrupt_get_available(kcpuset_t *cpuset)
{
        CPU_INFO_ITERATOR cii;
        struct cpu_info *ci;

        kcpuset_zero(cpuset);

        mutex_enter(&cpu_lock);
        for (CPU_INFO_FOREACH(cii, ci)) {
                if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) == 0)
                        kcpuset_set(cpuset, cpu_index(ci));
        }
        mutex_exit(&cpu_lock);
}

void
interrupt_get_devname(const char *intrid, char *buf, size_t len)
{
        struct intr_source *is;

        if (len == 0)
                return;

        buf[0] = '\0';

        is = e500_intr_get_source(intrid);
        if (is != NULL)
                strlcpy(buf, is->is_xname, len);
}

struct intrids_handler *
interrupt_construct_intrids(const kcpuset_t *cpuset)
{
        struct intr_source *is;
        struct intrids_handler *ii_handler;
        intrid_t *ids;
        int i, n;

        if (kcpuset_iszero(cpuset))
                return NULL;

        n = 0;
        mutex_enter(&e500_intr_lock);
        for (is = e500_intr_sources; is < e500_intr_last_source; ++is) {
                if (e500_intr_is_affinity_intrsource(is, cpuset))
                        ++n;
        }
        mutex_exit(&e500_intr_lock);

        const size_t alloc_size = sizeof(int) + sizeof(intrid_t) * n;
        ii_handler = kmem_zalloc(alloc_size, KM_SLEEP);
        ii_handler->iih_nids = n;
        if (n == 0)
                return ii_handler;

        ids = ii_handler->iih_intrids;
        mutex_enter(&e500_intr_lock);
        for (i = 0, is = e500_intr_sources;
             i < n && is < e500_intr_last_source;
             ++is) {
                if (!e500_intr_is_affinity_intrsource(is, cpuset))
                        continue;

                if (is->is_source[0] != '\0') {
                        strlcpy(ids[i], is->is_source, sizeof(ids[0]));
                        ++i;
                }
        }
        mutex_exit(&e500_intr_lock);

        return ii_handler;
}

void
interrupt_destruct_intrids(struct intrids_handler *ii_handler)
{
        size_t iih_size;

        if (ii_handler == NULL)
                return;

        iih_size = sizeof(int) + sizeof(intrid_t) * ii_handler->iih_nids;
        kmem_free(ii_handler, iih_size);
}

static int
interrupt_distribute_locked(struct intr_source *is, const kcpuset_t *newset,
    kcpuset_t *oldset)
{
        int error;

        KASSERT(mutex_owned(&cpu_lock));

        if (is->is_ipl == IPL_NONE || IST_PERCPU_P(is->is_ist))
                return EINVAL;

        mutex_enter(&e500_intr_lock);
        if (oldset != NULL)
                e500_intr_get_affinity(is, oldset);
        error = e500_intr_set_affinity(is, newset);
        mutex_exit(&e500_intr_lock);

        return error;
}

int
interrupt_distribute(void *ich, const kcpuset_t *newset, kcpuset_t *oldset)
{
        int error;

        mutex_enter(&cpu_lock);
        error = interrupt_distribute_locked(ich, newset, oldset);
        mutex_exit(&cpu_lock);

        return error;
}

int
interrupt_distribute_handler(const char *intrid, const kcpuset_t *newset,
    kcpuset_t *oldset)
{
        struct intr_source *is;
        int error;

        is = e500_intr_get_source(intrid);
        if (is != NULL) {
                mutex_enter(&cpu_lock);
                error = interrupt_distribute_locked(is, newset, oldset);
                mutex_exit(&cpu_lock);
        } else
                error = ENOENT;

        return error;
}