/*      $NetBSD: powerpc_machdep.c,v 1.89 2026/04/08 04:06:41 thorpej Exp $     */

/*
 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
 * Copyright (C) 1995, 1996 TooLs GmbH.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed by TooLs GmbH.
 * 4. The name of TooLs GmbH may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: powerpc_machdep.c,v 1.89 2026/04/08 04:06:41 thorpej Exp $");

#ifdef _KERNEL_OPT
#include "opt_altivec.h"
#include "opt_ddb.h"
#include "opt_modular.h"
#include "opt_multiprocessor.h"
#include "opt_ppcarch.h"
#include "opt_ppcopts.h"
#endif

#include <sys/param.h>
#include <sys/conf.h>
#include <sys/disklabel.h>
#include <sys/exec.h>
#include <sys/kauth.h>
#include <sys/pool.h>
#include <sys/proc.h>
#include <sys/signal.h>
#include <sys/sysctl.h>
#include <sys/ucontext.h>
#include <sys/cpu.h>
#include <sys/module.h>
#include <sys/device.h>
#include <sys/pcu.h>
#include <sys/atomic.h>
#include <sys/kmem.h>
#include <sys/xcall.h>
#include <sys/ipi.h>
#include <sys/kcore.h>

#include <dev/mm.h>

#include <powerpc/fpu.h>
#include <powerpc/pcb.h>
#include <powerpc/psl.h>
#include <powerpc/userret.h>
#if defined(ALTIVEC) || defined(PPC_HAVE_SPE)
#include <powerpc/altivec.h>
#endif
#include <powerpc/kcore.h>
#include <machine/powerpc.h>

#ifdef MULTIPROCESSOR
#include <powerpc/pic/ipivar.h>
#include <machine/cpu_counter.h>
#endif

#ifdef DDB
#include <machine/db_machdep.h>
#include <ddb/db_output.h>
#endif

int cpu_timebase;
int cpu_printfataltraps = 1;
#if !defined(PPC_IBM4XX)
extern int powersave;
#endif

/* exported variable to be filled in by the bootloaders */
char *booted_kernel;

const pcu_ops_t * const pcu_ops_md_defs[PCU_UNIT_COUNT] = {
        [PCU_FPU] = &fpu_ops,
#if defined(ALTIVEC) || defined(PPC_HAVE_SPE)
        [PCU_VEC] = &vec_ops,
#endif
};

#ifdef MULTIPROCESSOR
struct cpuset_info cpuset_info;
#endif

/*
 * Set set up registers on exec.
 */
void
setregs(struct lwp *l, struct exec_package *epp, vaddr_t stack)
{
        struct proc * const p = l->l_proc;
        struct trapframe * const tf = l->l_md.md_utf;
        struct pcb * const pcb = lwp_getpcb(l);
        struct ps_strings arginfo;
        vaddr_t func = epp->ep_entry;

        memset(tf, 0, sizeof *tf);
        tf->tf_fixreg[1] = -roundup(-stack + 8, 16);

        /*
         * XXX Machine-independent code has already copied arguments and
         * XXX environment to userland.  Get them back here.
         */
        (void)copyin_psstrings(p, &arginfo);

        /*
         * Set up arguments for _start():
         *      _start(argc, argv, envp, obj, cleanup, ps_strings);
         *
         * Notes:
         *      - obj and cleanup are the auxiliary and termination
         *        vectors.  They are fixed up by ld.elf_so.
         *      - ps_strings is a NetBSD extension, and will be
         *        ignored by executables which are strictly
         *        compliant with the SVR4 ABI.
         *
         * XXX We have to set both regs and retval here due to different
         * XXX calling convention in trap.c and init_main.c.
         */
        tf->tf_fixreg[3] = arginfo.ps_nargvstr;
        tf->tf_fixreg[4] = (register_t)arginfo.ps_argvstr;
        tf->tf_fixreg[5] = (register_t)arginfo.ps_envstr;
        tf->tf_fixreg[6] = 0;                   /* auxiliary vector */
        tf->tf_fixreg[7] = 0;                   /* termination vector */
        tf->tf_fixreg[8] = p->p_psstrp;         /* NetBSD extension */

#ifdef _LP64
        /*
         * For native ELF64, entry point to the function
         * descriptor which contains the real function address
         * and its TOC base address.
         */
        uintptr_t fdesc[3] = { [0] = func, [1] = 0, [2] = 0 };
        copyin((void *)func, fdesc, sizeof(fdesc));
        tf->tf_fixreg[2] = fdesc[1] + epp->ep_entryoffset;
        func = fdesc[0] + epp->ep_entryoffset;
#endif
        tf->tf_srr0 = func;
        tf->tf_srr1 = PSL_MBO | PSL_USERSET;
#ifdef ALTIVEC
        tf->tf_vrsave = 0;
#endif
        pcb->pcb_flags = PSL_FE_DFLT;

#if defined(PPC_BOOKE) || defined(PPC_IBM4XX)
        p->p_md.md_ss_addr[0] = p->p_md.md_ss_addr[1] = 0;
        p->p_md.md_ss_insn[0] = p->p_md.md_ss_insn[1] = 0;
#endif
}

/*
 * Machine dependent system variables.
 */
static int
sysctl_machdep_cacheinfo(SYSCTLFN_ARGS)
{
        struct sysctlnode node = *rnode;

        node.sysctl_data = &curcpu()->ci_ci;
        node.sysctl_size = sizeof(curcpu()->ci_ci);
        return (sysctl_lookup(SYSCTLFN_CALL(&node)));
}

#if !defined (PPC_IBM4XX)
static int
sysctl_machdep_powersave(SYSCTLFN_ARGS)
{
        struct sysctlnode node = *rnode;

        if (powersave < 0)
                node.sysctl_flags &= ~CTLFLAG_READWRITE;
        return (sysctl_lookup(SYSCTLFN_CALL(&node)));
}
#endif

static int
sysctl_machdep_booted_device(SYSCTLFN_ARGS)
{
        struct sysctlnode node;

        if (booted_device == NULL)
                return (EOPNOTSUPP);

        const char * const xname = device_xname(booted_device);

        node = *rnode;
        node.sysctl_data = __UNCONST(xname);
        node.sysctl_size = strlen(xname) + 1;
        return (sysctl_lookup(SYSCTLFN_CALL(&node)));
}

static int
sysctl_machdep_booted_kernel(SYSCTLFN_ARGS)
{
        struct sysctlnode node;

        if (booted_kernel == NULL || booted_kernel[0] == '\0')
                return (EOPNOTSUPP);

        node = *rnode;
        node.sysctl_data = booted_kernel;
        node.sysctl_size = strlen(booted_kernel) + 1;
        return (sysctl_lookup(SYSCTLFN_CALL(&node)));
}

SYSCTL_SETUP(sysctl_machdep_setup, "sysctl machdep subtree setup")
{

        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT,
                       CTLTYPE_NODE, "machdep", NULL,
                       NULL, 0, NULL, 0,
                       CTL_MACHDEP, CTL_EOL);

        /* Deprecated */
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
                       CTLTYPE_INT, "cachelinesize", NULL,
                       NULL, curcpu()->ci_ci.dcache_line_size, NULL, 0,
                       CTL_MACHDEP, CPU_CACHELINE, CTL_EOL);
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT,
                       CTLTYPE_INT, "timebase", NULL,
                       NULL, 0, &cpu_timebase, 0,
                       CTL_MACHDEP, CPU_TIMEBASE, CTL_EOL);
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
                       CTLTYPE_INT, "printfataltraps", NULL,
                       NULL, 0, &cpu_printfataltraps, 0,
                       CTL_MACHDEP, CPU_PRINTFATALTRAPS, CTL_EOL);
        /* Use this instead of CPU_CACHELINE */
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT,
                       CTLTYPE_STRUCT, "cacheinfo", NULL,
                       sysctl_machdep_cacheinfo, 0, NULL, 0,
                       CTL_MACHDEP, CPU_CACHEINFO, CTL_EOL);
#if !defined (PPC_IBM4XX)
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
                       CTLTYPE_INT, "powersave", NULL,
                       sysctl_machdep_powersave, 0, &powersave, 0,
                       CTL_MACHDEP, CPU_POWERSAVE, CTL_EOL);
#endif
#if defined(PPC_IBM4XX) || defined(PPC_BOOKE)
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
                       CTLTYPE_INT, "altivec", NULL,
                       NULL, 0, NULL, 0,
                       CTL_MACHDEP, CPU_ALTIVEC, CTL_EOL);
#else
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
                       CTLTYPE_INT, "altivec", NULL,
                       NULL, cpu_altivec, NULL, 0,
                       CTL_MACHDEP, CPU_ALTIVEC, CTL_EOL);
#endif
#ifdef PPC_BOOKE
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
                       CTLTYPE_INT, "execprot", NULL,
                       NULL, 1, NULL, 0,
                       CTL_MACHDEP, CPU_EXECPROT, CTL_EOL);
#endif
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT,
                       CTLTYPE_STRING, "booted_device", NULL,
                       sysctl_machdep_booted_device, 0, NULL, 0,
                       CTL_MACHDEP, CPU_BOOTED_DEVICE, CTL_EOL);
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT,
                       CTLTYPE_STRING, "booted_kernel", NULL,
                       sysctl_machdep_booted_kernel, 0, NULL, 0,
                       CTL_MACHDEP, CPU_BOOTED_KERNEL, CTL_EOL);
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
                       CTLTYPE_INT, "fpu_present", NULL,
                       NULL,
#if defined(PPC_HAVE_FPU)
                       1,
#else
                       0,
#endif
                       NULL, 0,
                       CTL_MACHDEP, CPU_FPU, CTL_EOL);
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
                       CTLTYPE_INT, "no_unaligned", NULL,
                       NULL,
#if defined(PPC_NO_UNALIGNED)
                       1,
#else
                       0,
#endif
                       NULL, 0,
                       CTL_MACHDEP, CPU_NO_UNALIGNED, CTL_EOL);
        sysctl_createv(clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
                       CTLTYPE_INT, "pvr", NULL,
                       NULL,
                       mfpvr(),
                       NULL, 0,
                       CTL_MACHDEP, CPU_PVR, CTL_EOL);
}

/*
 * Crash dump handling.
 */
u_int32_t dumpmag = 0x8fca0101;         /* magic number */
int dumpsize = 0;                       /* size of dump in pages */
long dumplo = -1;                       /* blocks */

/*
 * Calculate size of machine dependent kernel core dump headers.
 */
int
cpu_dumpsize(void)
{
        struct mem_region *mem, *avail;
        unsigned nreg;
        int size;

        nreg = 0;
        mem_regions(&mem, &avail);
        while (mem->size != 0) {
                nreg++;
                mem++;
        }

        size = ALIGN(sizeof(kcore_seg_t)) +
               ALIGN(sizeof(cpu_kcore_hdr_t)) +
               ALIGN(nreg * sizeof(phys_ram_seg_t));
        if (roundup(size, dbtob(1)) != dbtob(1)) {
                return -1;
        }

        return 1;
}

/*
 * This is called by main to set dumplo and dumpsize.
 */
void
cpu_dumpconf(void)
{
        struct mem_region *mem, *avail;
        u_long nblks, dumpblks; /* size of dump area */

        if (dumpdev == NODEV) {
                return;
        }

        nblks = bdev_size(dumpdev);
        if (nblks <= ctod(1)) {
                return;
        }

        dumpsize = 0;
        dumpblks = cpu_dumpsize();
        if (dumpblks < 0) {
                goto bad;
        }

        /* dumpsize is in page units, and doesn't include headers. */
        mem_regions(&mem, &avail);
        while (mem->size != 0) {
                dumpsize += mem->size / PAGE_SIZE;
                mem++;
        }
        dumpblks += ctod(dumpsize);

        /* If dump won't fit (incl. room for possible label), punt. */
        if (dumpblks > (nblks - ctod(1))) {
                goto bad;
        }

        /* Put dump at end of partition. */
        dumplo = nblks - dumpblks;

        return;

bad:
        dumpsize = 0;
}

/* 
 * Start a new LWP
 */
void
startlwp(void *arg)
{
        ucontext_t * const uc = arg;
        lwp_t * const l = curlwp;
        struct trapframe * const tf = l->l_md.md_utf;
        int error __diagused;

        error = cpu_setmcontext(l, &uc->uc_mcontext, uc->uc_flags);
        KASSERT(error == 0);

        kmem_free(uc, sizeof(ucontext_t));
        userret(l, tf);
}

/*
 * Process the tail end of a posix_spawn() for the child.
 */
void
cpu_spawn_return(struct lwp *l)
{
        struct trapframe * const tf = l->l_md.md_utf;

        userret(l, tf);
}

bool
cpu_intr_p(void)
{

        return curcpu()->ci_idepth >= 0;
}

void
cpu_idle(void)
{
        KASSERT(mfmsr() & PSL_EE);
        KASSERTMSG(curcpu()->ci_cpl == IPL_NONE,
            "ci_cpl = %d", curcpu()->ci_cpl);
        (*curcpu()->ci_idlespin)();
}

void
cpu_ast(struct lwp *l, struct cpu_info *ci)
{
        l->l_md.md_astpending = 0;      /* we are about to do it */
        if (l->l_pflag & LP_OWEUPC) {
                l->l_pflag &= ~LP_OWEUPC;
                ADDUPROF(l);
        }
}

void
cpu_need_resched(struct cpu_info *ci, struct lwp *l, int flags)
{
        KASSERT(kpreempt_disabled());

#ifdef __HAVE_PREEMPTION
        if ((flags & RESCHED_KPREEMPT) != 0) {
                if ((flags & RESCHED_REMOTE) != 0) {
                        cpu_send_ipi(cpu_index(ci), IPI_KPREEMPT);
                } else {
                        softint_trigger(SOFTINT_KPREEMPT);
                }
                return;
        }
#endif
        if ((flags & RESCHED_REMOTE) != 0) {
#if defined(MULTIPROCESSOR)
                cpu_send_ipi(cpu_index(ci), IPI_AST);
#endif
        } else {
                l->l_md.md_astpending = 1;      /* force call to cpu_ast() */
        }
}

void
cpu_need_proftick(lwp_t *l)
{
        l->l_pflag |= LP_OWEUPC;
        l->l_md.md_astpending = 1;
}

void
cpu_signotify(lwp_t *l)
{
        if (l->l_cpu != curcpu()) {
#if defined(MULTIPROCESSOR)
                cpu_send_ipi(cpu_index(l->l_cpu), IPI_AST);
#endif
        } else {
                l->l_md.md_astpending = 1;
        }
}

vaddr_t
cpu_lwp_pc(lwp_t *l)
{
        return l->l_md.md_utf->tf_srr0;
}

bool
cpu_clkf_usermode(const struct clockframe *cf)
{
        return (cf->cf_srr1 & PSL_PR) != 0;
}

vaddr_t
cpu_clkf_pc(const struct clockframe *cf)
{
        return cf->cf_srr0;
}

bool
cpu_clkf_intr(const struct clockframe *cf)
{
        return cf->cf_idepth > 0;
}

#ifdef MULTIPROCESSOR
/*
 * MD support for xcall(9) interface.
 */

void
xc_send_ipi(struct cpu_info *ci)
{
        KASSERT(kpreempt_disabled());
        KASSERT(curcpu() != ci);

        cpuid_t target = (ci != NULL ? cpu_index(ci) : IPI_DST_NOTME);

        /* Unicast: remote CPU. */
        /* Broadcast: all, but local CPU (caller will handle it). */
        cpu_send_ipi(target, IPI_XCALL);
}

void
cpu_ipi(struct cpu_info *ci)
{
        KASSERT(kpreempt_disabled());
        KASSERT(curcpu() != ci);

        cpuid_t target = (ci != NULL ? cpu_index(ci) : IPI_DST_NOTME);

        /* Unicast: remote CPU. */
        /* Broadcast: all, but local CPU (caller will handle it). */
        cpu_send_ipi(target, IPI_GENERIC);
}

/* XXX kcpuset_create(9), kcpuset_clone(9) couldn't use interrupt context */
typedef uint32_t __cpuset_t;
CTASSERT(MAXCPUS <= 32);

#define CPUSET_SINGLE(cpu)              ((__cpuset_t)1 << (cpu))

#define CPUSET_ADD(set, cpu)            atomic_or_32(&(set), CPUSET_SINGLE(cpu))
#define CPUSET_DEL(set, cpu)            atomic_and_32(&(set), ~CPUSET_SINGLE(cpu))
#define CPUSET_SUB(set1, set2)          atomic_and_32(&(set1), ~(set2))

#define CPUSET_EXCEPT(set, cpu)         ((set) & ~CPUSET_SINGLE(cpu))

#define CPUSET_HAS_P(set, cpu)          ((set) & CPUSET_SINGLE(cpu))
#define CPUSET_NEXT(set)                (ffs(set) - 1)

#define CPUSET_EMPTY_P(set)             ((set) == (__cpuset_t)0)
#define CPUSET_EQUAL_P(set1, set2)      ((set1) == (set2))
#define CPUSET_CLEAR(set)               ((set) = (__cpuset_t)0)
#define CPUSET_ASSIGN(set1, set2)       ((set1) = (set2))

#define CPUSET_EXPORT(kset, set)        kcpuset_export_u32((kset), &(set), sizeof(set))

/*
 * Send an inter-processor interrupt to CPUs in cpuset (excludes curcpu())
 */
static void
cpu_multicast_ipi(__cpuset_t cpuset, uint32_t msg)
{
        CPU_INFO_ITERATOR cii;
        struct cpu_info *ci;

        CPUSET_DEL(cpuset, cpu_index(curcpu()));
        if (CPUSET_EMPTY_P(cpuset))
                return;

        for (CPU_INFO_FOREACH(cii, ci)) {
                const int index = cpu_index(ci);
                if (CPUSET_HAS_P(cpuset, index)) {
                        CPUSET_DEL(cpuset, index);
                        cpu_send_ipi(index, msg);
                }
        }
}

static void
cpu_ipi_error(const char *s, kcpuset_t *succeeded, __cpuset_t expected)
{
        __cpuset_t cpuset;

        CPUSET_EXPORT(succeeded, cpuset);
        CPUSET_SUB(expected, cpuset);
        if (!CPUSET_EMPTY_P(expected)) {
                printf("Failed to %s:", s);
                do {
                        const int index = CPUSET_NEXT(expected);
                        CPUSET_DEL(expected, index);
                        printf(" cpu%d", index);
                } while (!CPUSET_EMPTY_P(expected));
                printf("\n");
        }
}

static int
cpu_ipi_wait(kcpuset_t *watchset, __cpuset_t mask)
{
        uint64_t tmout = curcpu()->ci_data.cpu_cc_freq; /* some finite amount of time */
        __cpuset_t cpuset;

        while (tmout--) {
                CPUSET_EXPORT(watchset, cpuset);
                if (cpuset == mask)
                        return 0;               /* success */
        }
        return 1;                               /* timed out */
}

/*
 * Halt this cpu.
 */
void
cpu_halt(void)
{
        struct cpuset_info * const csi = &cpuset_info;
        const cpuid_t index = cpu_index(curcpu());

        printf("cpu%ld: shutting down\n", index);
        kcpuset_set(csi->cpus_halted, index);
        spl0();                 /* allow interrupts e.g. further ipi ? */

        /* spin */
        for (;;)
                continue;
        /*NOTREACHED*/
}

/*
 * Halt all running cpus, excluding current cpu.
 */
void
cpu_halt_others(void)
{
        struct cpuset_info * const csi = &cpuset_info;
        const cpuid_t index = cpu_index(curcpu());
        __cpuset_t cpumask, cpuset, halted;

        KASSERT(kpreempt_disabled());

        CPUSET_EXPORT(csi->cpus_running, cpuset);
        CPUSET_DEL(cpuset, index);
        CPUSET_ASSIGN(cpumask, cpuset);
        CPUSET_EXPORT(csi->cpus_halted, halted);
        CPUSET_SUB(cpuset, halted);

        if (CPUSET_EMPTY_P(cpuset))
                return;

        cpu_multicast_ipi(cpuset, IPI_HALT);
        if (cpu_ipi_wait(csi->cpus_halted, cpumask))
                cpu_ipi_error("halt", csi->cpus_halted, cpumask);

        /*
         * TBD
         * Depending on available firmware methods, other cpus will
         * either shut down themselves, or spin and wait for us to
         * stop them.
         */
}

/*
 * Pause this cpu.
 */
void
cpu_pause(struct trapframe *tf)
{
        volatile struct cpuset_info * const csi = &cpuset_info;
        int s = splhigh();
        const cpuid_t index = cpu_index(curcpu());

        for (;;) {
                kcpuset_set(csi->cpus_paused, index);
                while (kcpuset_isset(csi->cpus_paused, index))
                        docritpollhooks();
                kcpuset_set(csi->cpus_resumed, index);
#ifdef DDB
                if (ddb_running_on_this_cpu_p())
                        cpu_Debugger();
                if (ddb_running_on_any_cpu_p())
                        continue;
#endif  /* DDB */
                break;
        }

        splx(s);
}

/*
 * Pause all running cpus, excluding current cpu.
 */
void
cpu_pause_others(void)
{
        struct cpuset_info * const csi = &cpuset_info;
        const cpuid_t index = cpu_index(curcpu());
        __cpuset_t cpuset;

        KASSERT(kpreempt_disabled());

        CPUSET_EXPORT(csi->cpus_running, cpuset);
        CPUSET_DEL(cpuset, index);

        if (CPUSET_EMPTY_P(cpuset))
                return;

        cpu_multicast_ipi(cpuset, IPI_SUSPEND);
        if (cpu_ipi_wait(csi->cpus_paused, cpuset))
                cpu_ipi_error("pause", csi->cpus_paused, cpuset);
}

/*
 * Resume a single cpu.
 */
void
cpu_resume(cpuid_t index)
{
        struct cpuset_info * const csi = &cpuset_info;
        __cpuset_t cpuset = CPUSET_SINGLE(index);

        kcpuset_zero(csi->cpus_resumed);
        kcpuset_clear(csi->cpus_paused, index);

        if (cpu_ipi_wait(csi->cpus_paused, cpuset))
                cpu_ipi_error("resume", csi->cpus_resumed, cpuset);
}

/*
 * Resume all paused cpus.
 */
void
cpu_resume_others(void)
{
        struct cpuset_info * const csi = &cpuset_info;
        __cpuset_t cpuset;

        kcpuset_zero(csi->cpus_resumed);
        CPUSET_EXPORT(csi->cpus_paused, cpuset);
        kcpuset_zero(csi->cpus_paused);

        if (cpu_ipi_wait(csi->cpus_resumed, cpuset))
                cpu_ipi_error("resume", csi->cpus_resumed, cpuset);
}

int
cpu_is_paused(int index)
{
        struct cpuset_info * const csi = &cpuset_info;

        return kcpuset_isset(csi->cpus_paused, index);
}

#ifdef DDB
void
cpu_debug_dump(void)
{
        struct cpuset_info * const csi = &cpuset_info;
        CPU_INFO_ITERATOR cii;
        struct cpu_info *ci;
        char running, hatched, paused, resumed, halted;

#ifdef _LP64
        db_printf("CPU CPUID STATE CPUINFO          CPL INT MTX IPIS\n");
#else
        db_printf("CPU CPUID STATE CPUINFO  CPL INT MTX IPIS\n");
#endif
        for (CPU_INFO_FOREACH(cii, ci)) {
                const cpuid_t index = cpu_index(ci);
                hatched = (kcpuset_isset(csi->cpus_hatched, index) ? 'H' : '-');
                running = (kcpuset_isset(csi->cpus_running, index) ? 'R' : '-');
                paused  = (kcpuset_isset(csi->cpus_paused,  index) ? 'P' : '-');
                resumed = (kcpuset_isset(csi->cpus_resumed, index) ? 'r' : '-');
                halted  = (kcpuset_isset(csi->cpus_halted,  index) ? 'h' : '-');
                db_printf("%3ld 0x%03x %c%c%c%c%c %p %3d %3d %3d 0x%08x\n",
                    index, ci->ci_cpuid,
                    running, hatched, paused, resumed, halted,
                    ci, ci->ci_cpl, ci->ci_idepth, ci->ci_mtx_count,
                    ci->ci_pending_ipis);
        }
}
#endif  /* DDB */
#endif /* MULTIPROCESSOR */

int
emulate_mxmsr(struct lwp *l, struct trapframe *tf, uint32_t opcode)
{

#define OPC_MFMSR_CODE          0x7c0000a6
#define OPC_MFMSR_MASK          0xfc1fffff
#define OPC_MFMSR_P(o)          (((o) & OPC_MFMSR_MASK) == OPC_MFMSR_CODE)

#define OPC_MTMSR_CODE          0x7c000124
#define OPC_MTMSR_MASK          0xfc1fffff
#define OPC_MTMSR_P(o)          (((o) & OPC_MTMSR_MASK) == OPC_MTMSR_CODE)

#define OPC_MXMSR_REG(o)        (((o) >> 21) & 0x1f)

        if (OPC_MFMSR_P(opcode)) {
                struct pcb * const pcb = lwp_getpcb(l);
                register_t msr = tf->tf_srr1 & PSL_USERSRR1;

                if (fpu_used_p(l))
                        msr |= PSL_FP;
#ifdef ALTIVEC
                if (vec_used_p(l))
                        msr |= PSL_VEC;
#endif

                msr |= (pcb->pcb_flags & PSL_FE_PREC);
                tf->tf_fixreg[OPC_MXMSR_REG(opcode)] = msr;
                return 1;
        }

        if (OPC_MTMSR_P(opcode)) {
                struct pcb * const pcb = lwp_getpcb(l);
                register_t msr = tf->tf_fixreg[OPC_MXMSR_REG(opcode)];

                /*
                 * Ignore the FP enable bit in the requested MSR.
                 * It might be set in the thread's actual MSR but the
                 * user code isn't allowed to change it.
                 */
                msr &= ~PSL_FP;
#ifdef ALTIVEC
                msr &= ~PSL_VEC;
#endif

                /*
                 * Don't let the user muck with bits he's not allowed to.
                 */
#ifdef PPC_HAVE_FPU
                if (!PSL_USEROK_P(msr))
#else
                if (!PSL_USEROK_P(msr & ~PSL_FE_PREC))
#endif
                        return 0;

                /*
                 * For now, only update the FP exception mode.
                 */
                pcb->pcb_flags &= ~PSL_FE_PREC;
                pcb->pcb_flags |= msr & PSL_FE_PREC;

#ifdef PPC_HAVE_FPU
                /*
                 * If we think we have the FPU, update SRR1 too.  If we're
                 * wrong userret() will take care of it.
                 */
                if (tf->tf_srr1 & PSL_FP) {
                        tf->tf_srr1 &= ~(PSL_FE0|PSL_FE1);
                        tf->tf_srr1 |= msr & (PSL_FE0|PSL_FE1);
                }
#endif
                return 1;
        }

        return 0;
}

bool
mm_md_direct_mapped_phys(paddr_t pa, vaddr_t *vap)
{
        if (atop(pa) < physmem) {
                *vap = pa;
                return true;
        }

        return false;
}

int
mm_md_physacc(paddr_t pa, vm_prot_t prot)
{

        return (atop(pa) < physmem) ? 0 : EFAULT;
}

int
mm_md_kernacc(void *va, vm_prot_t prot, bool *handled)
{
        if (atop((paddr_t)va) < physmem) {
                *handled = true;
                return 0;
        }

        if ((vaddr_t)va < VM_MIN_KERNEL_ADDRESS
            || (vaddr_t)va >= VM_MAX_KERNEL_ADDRESS)
                return EFAULT;

        *handled = false;
        return 0;
}