/*      $NetBSD: pmap.c,v 1.379 2025/12/24 20:37:04 andvar Exp $ */

/*
 * Copyright (c) 1996
 *      The President and Fellows of Harvard College. All rights reserved.
 * Copyright (c) 1992, 1993
 *      The Regents of the University of California.  All rights reserved.
 *
 * This software was developed by the Computer Systems Engineering group
 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
 * contributed to Berkeley.
 *
 * All advertising materials mentioning features or use of this software
 * must display the following acknowledgement:
 *      This product includes software developed by Harvard University.
 *      This product includes software developed by the University of
 *      California, Lawrence Berkeley Laboratory.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed by Aaron Brown and
 *      Harvard University.
 *      This product includes software developed by the University of
 *      California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *      @(#)pmap.c      8.4 (Berkeley) 2/5/94
 *
 */

/*
 * SPARC physical map management code.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.379 2025/12/24 20:37:04 andvar Exp $");

#include "opt_ddb.h"
#include "opt_kgdb.h"
#include "opt_sparc_arch.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/pool.h>
#include <sys/exec.h>
#include <sys/core.h>
#include <sys/kcore.h>
#include <sys/kernel.h>
#include <sys/atomic.h>

#include <sys/exec_aout.h>              /* for MID_* */

#include <uvm/uvm.h>

#include <machine/autoconf.h>
#include <machine/bsd_openprom.h>
#include <machine/oldmon.h>
#include <machine/cpu.h>
#include <machine/ctlreg.h>
#include <machine/kcore.h>
#include <machine/locore.h>

#include <sparc/sparc/asm.h>
#include <sparc/sparc/cache.h>
#include <sparc/sparc/vaddrs.h>
#include <sparc/sparc/cpuvar.h>

/*
 * The SPARCstation offers us the following challenges:
 *
 *   1. A virtual address cache.  This is, strictly speaking, not
 *      part of the architecture, but the code below assumes one.
 *      This is a write-through cache on the 4c and a write-back cache
 *      on others.
 *
 *   2. (4/4c only) An MMU that acts like a cache.  There is not enough
 *      space in the MMU to map everything all the time.  Instead, we need
 *      to load MMU with the `working set' of translations for each
 *      process. The sun4m does not act like a cache; tables are maintained
 *      in physical memory.
 *
 *   3. Segmented virtual and physical spaces.  The upper 12 bits of
 *      a virtual address (the virtual segment) index a segment table,
 *      giving a physical segment.  The physical segment selects a
 *      `Page Map Entry Group' (PMEG) and the virtual page number---the
 *      next 5 or 6 bits of the virtual address---select the particular
 *      `Page Map Entry' for the page.  We call the latter a PTE and
 *      call each Page Map Entry Group a pmeg (for want of a better name).
 *      Note that the sun4m has an unsegmented 36-bit physical space.
 *
 *      Since there are no valid bits in the segment table, the only way
 *      to have an invalid segment is to make one full pmeg of invalid PTEs.
 *      We use the last one (since the ROM does as well) (sun4/4c only)
 *
 *   4. Discontiguous physical pages.  The Mach VM expects physical pages
 *      to be in one sequential lump.
 *
 *   5. The MMU is always on: it is not possible to disable it.  This is
 *      mainly a startup hassle.
 */

struct pmap_stats {
        int     ps_unlink_pvfirst;      /* # of pv_unlinks on head */
        int     ps_unlink_pvsearch;     /* # of pv_unlink searches */
        int     ps_changeprots;         /* # of calls to changeprot */
        int     ps_enter_firstpv;       /* pv heads entered */
        int     ps_enter_secondpv;      /* pv nonheads entered */
        int     ps_useless_changewire;  /* useless wiring changes */
        int     ps_npg_prot_all;        /* # of active pages protected */
        int     ps_npg_prot_actual;     /* # pages actually affected */
        int     ps_npmeg_free;          /* # of free pmegs */
        int     ps_npmeg_locked;        /* # of pmegs on locked list */
        int     ps_npmeg_lru;           /* # of pmegs on lru list */
} pmap_stats;

#if defined(SUN4) || defined(SUN4C)
struct evcnt mmu_stolenpmegs_evcnt =
        EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"mmu","stln pmgs");
EVCNT_ATTACH_STATIC(mmu_stolenpmegs_evcnt);

struct evcnt mmu_pagein_evcnt =
        EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"mmu","pagein");
EVCNT_ATTACH_STATIC(mmu_pagein_evcnt);
#endif /* SUN4 || SUN4C */

#ifdef DEBUG
#define PDB_CREATE      0x0001
#define PDB_DESTROY     0x0002
#define PDB_REMOVE      0x0004
#define PDB_CHANGEPROT  0x0008
#define PDB_ENTER       0x0010
#define PDB_FOLLOW      0x0020
#define PDB_INITLOUD    0x0040

#define PDB_MMU_ALLOC   0x0100
#define PDB_MMU_STEAL   0x0200
#define PDB_CTX_ALLOC   0x0400
#define PDB_CTX_STEAL   0x0800
#define PDB_MMUREG_ALLOC        0x1000
#define PDB_MMUREG_STEAL        0x2000
#define PDB_CACHESTUFF  0x4000
#define PDB_SWITCHMAP   0x8000
#define PDB_SANITYCHK   0x10000
int     pmapdebug = 0;
#define DPRINTF(level, fmt, ...) do { \
        if (pmapdebug & (level)) \
                printf("%s:%d: " fmt "\n", __func__, __LINE__, ##__VA_ARGS__); \
} while (0)
#else
#define DPRINTF(level, fmt, ...) /* nothing */
#endif

/*
 * Bounds on managed physical addresses. Used by (MD) users
 * of uvm_pglistalloc() to provide search hints.
 */
paddr_t vm_first_phys = (paddr_t)-1;
paddr_t vm_last_phys = 0;
psize_t vm_num_phys;

#define PMAP_LOCK()     mutex_enter(&pmap_lock)
#define PMAP_UNLOCK()   mutex_exit(&pmap_lock)

/*
 * Flags in pvlist.pv_flags.  Note that PV_MOD must be 1 and PV_REF must be 2
 * since they must line up with the bits in the hardware PTEs (see pte.h).
 * SUN4M bits are at a slightly different location in the PTE.
 *
 * Note: the REF, MOD and ANC flag bits occur only in the head of a pvlist.
 * The NC bit is meaningful in each individual pv entry and reflects the
 * requested non-cacheability at the time the entry was made through
 * pv_link() or when subsequently altered by kvm_uncache() (but the latter
 * does not happen in kernels as of the time of this writing (March 2001)).
 */
#define PV_MOD          1       /* page modified */
#define PV_REF          2       /* page referenced */
#define PV_NC           4       /* page cannot be cached */
#define PV_REF4M        1       /* page referenced (SRMMU) */
#define PV_MOD4M        2       /* page modified (SRMMU) */
#define PV_ANC          0x10    /* page has incongruent aliases */

static struct pool pv_pool;

/*
 * pvhead(pte): find a VM page given a PTE entry.
 */
#if defined(SUN4) || defined(SUN4C)
static struct vm_page *
pvhead4_4c(u_int pte)
{
        paddr_t pa = (pte & PG_PFNUM) << PGSHIFT;

        return (PHYS_TO_VM_PAGE(pa));
}
#endif

#if defined(SUN4M) || defined(SUN4D)
static struct vm_page *
pvhead4m(u_int pte)
{
        paddr_t pa = (pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT;

        return (PHYS_TO_VM_PAGE(pa));
}
#endif

/*
 * Each virtual segment within each pmap is either valid or invalid.
 * It is valid if pm_npte[VA_VSEG(va)] is not 0.  This does not mean
 * it is in the MMU, however; that is true iff pm_segmap[VA_VSEG(va)]
 * does not point to the invalid PMEG.
 *
 * In the older SPARC architectures (sun4/sun4c), page tables are cached in
 * the MMU. The following discussion applies to these architectures:
 *
 * If a virtual segment is valid and loaded, the correct PTEs appear
 * in the MMU only.  If it is valid and unloaded, the correct PTEs appear
 * in the pm_pte[VA_VSEG(va)] only.  However, some effort is made to keep
 * the software copies consistent enough with the MMU so that libkvm can
 * do user address translations.  In particular, pv_changepte() and
 * pmap_enu() maintain consistency, while less critical changes are
 * not maintained.  pm_pte[VA_VSEG(va)] always points to space for those
 * PTEs.
 *
 * Each PMEG in the MMU is either free or contains PTEs corresponding to
 * some pmap and virtual segment.  If it contains some PTEs, it also contains
 * reference and modify bits that belong in the pv_table.  If we need
 * to steal a PMEG from some process (if we need one and none are free)
 * we must copy the ref and mod bits, and update pm_segmap in the other
 * pmap to show that its virtual segment is no longer in the MMU.
 *
 * There are 128 PMEGs in a small Sun-4, of which only a few dozen are
 * tied down permanently, leaving `about' 100 to be spread among
 * running processes.  These are managed as an LRU cache.  Before
 * calling the VM paging code for a user page fault, the fault handler
 * calls mmu_load(pmap, va) to try to get a set of PTEs put into the
 * MMU.  mmu_load will check the validity of the segment and tell whether
 * it did something.
 *
 * Since I hate the name PMEG I call this data structure an `mmu entry'.
 * Each mmuentry is on exactly one of three `usage' lists: free, LRU,
 * or locked.  The locked list is only used for kernel mappings that need
 * to be wired down.
 *
 *
 * In the sun4m architecture using the SPARC Reference MMU (SRMMU), three
 * levels of page tables are maintained in physical memory. We use the same
 * structures as with the 3-level old-style MMU (pm_regmap, pm_segmap,
 * rg_segmap, sg_pte, etc) to maintain kernel-edible page tables; we also
 * build a parallel set of physical tables that can be used by the MMU.
 * (XXX: This seems redundant, but is it necessary for the unified kernel?)
 *
 * If a virtual segment is valid, its entries will be in both parallel lists.
 * If it is not valid, then its entry in the kernel tables will be zero, and
 * its entry in the MMU tables will either be nonexistent or zero as well.
 *
 * The Reference MMU generally uses a Translation Look-aside Buffer (TLB)
 * to cache the result of recently executed page table walks. When
 * manipulating page tables, we need to ensure consistency of the
 * in-memory and TLB copies of the page table entries. This is handled
 * by flushing (and invalidating) a TLB entry when appropriate before
 * altering an in-memory page table entry.
 */
struct mmuentry {
        struct {
            struct mmuentry *prev, *next;
        }                       me_list;        /* usage list link */
        TAILQ_ENTRY(mmuentry)   me_pmchain;     /* pmap owner link */
        struct  pmap *me_pmap;          /* pmap, if in use */
        u_short me_vreg;                /* associated virtual region/segment */
        u_short me_vseg;                /* associated virtual region/segment */
        u_short me_cookie;              /* hardware SMEG/PMEG number */
#ifdef DIAGNOSTIC
        int *me_statp;/*XXX*/
#endif
};
struct mmuentry *mmusegments;   /* allocated in pmap_bootstrap */
struct mmuentry *mmuregions;    /* allocated in pmap_bootstrap */

#if defined(SUN4) || defined(SUN4C)
struct mmuentry segm_freelist, segm_lru, segm_locked;
#if defined(SUN4_MMU3L)
struct mmuentry region_freelist, region_lru, region_locked;
#endif
/*
 * We use a double linked list looping through its static head (which
 * always remains on the list), so we can remove any other member from
 * a list without knowing which list it is on.
 */
static void inline
mmuq_remove(struct mmuentry *e)
{
        e->me_list.next->me_list.prev = e->me_list.prev;
        e->me_list.prev->me_list.next = e->me_list.next;
}

static void inline
mmuq_init(struct mmuentry *e)
{
        memset(e, 0, sizeof(*e));
        e->me_list.next = e;
        e->me_list.prev = e;
}

static inline struct mmuentry *
mmuq_first(struct mmuentry *head)
{
        KASSERT(head->me_list.next != head);
        return head->me_list.next;
}

static inline bool
mmuq_empty(struct mmuentry *head)
{
        return head->me_list.next == head;
}

static inline void
mmuq_insert_tail(struct mmuentry *head, struct mmuentry *e)
{
        e->me_list.prev = head->me_list.prev;
        e->me_list.next = head;
        head->me_list.prev->me_list.next = e;
        head->me_list.prev = e;
}
#endif


int     seginval;               /* [4/4c] the invalid segment number */
int     reginval;               /* [4/3mmu] the invalid region number */

static kmutex_t pmap_lock;
static kmutex_t demap_lock;
static bool     lock_available = false; /* demap_lock has been initialized */

/*
 * (sun4/4c)
 * A context is simply a small number that dictates which set of 4096
 * segment map entries the MMU uses.  The Sun 4c has eight (SS1,IPC) or
 * sixteen (SS2,IPX) such sets. These are allotted in an `almost MRU' fashion.
 * (sun4m)
 * A context is simply a small number that indexes the context table, the
 * root-level page table mapping 4G areas. Each entry in this table points
 * to a 1st-level region table. A SPARC reference MMU will usually use 16
 * such contexts, but some offer as many as 64k contexts; the theoretical
 * maximum is 2^32 - 1, but this would create overlarge context tables.
 *
 * Each context is either free or attached to a pmap.
 *
 * Since the virtual address cache is tagged by context, when we steal
 * a context we have to flush (that part of) the cache.
 */
union ctxinfo {
        union   ctxinfo *c_nextfree;    /* free list (if free) */
        struct  pmap *c_pmap;           /* pmap (if busy) */
};

static kmutex_t ctx_lock;               /* lock for below, and {,de}activate */
union   ctxinfo *ctxinfo;               /* allocated at in pmap_bootstrap */
union   ctxinfo *ctx_freelist;          /* context free list */
int     ctx_kick;                       /* allocation rover when none free */
int     ctx_kickdir;                    /* ctx_kick roves both directions */
int     ncontext;                       /* sizeof ctx_freelist */

static void     ctx_alloc(struct pmap *);
static void     ctx_free(struct pmap *);

/*void *        vdumppages;     -* 32KB worth of reserved dump pages */

smeg_t          tregion;        /* [4/3mmu] Region for temporary mappings */

static struct pmap      kernel_pmap_store;      /* the kernel's pmap */
struct pmap *const kernel_pmap_ptr = &kernel_pmap_store; /* pmap_kernel() */
struct regmap   kernel_regmap_store[NKREG];     /* the kernel's regmap */
struct segmap   kernel_segmap_store[NKREG*NSEGRG];/* the kernel's segmaps */

#if defined(SUN4M) || defined(SUN4D)
u_int   *kernel_regtable_store;         /* 1k of storage to map the kernel */
u_int   *kernel_segtable_store;         /* 2k of storage to map the kernel */
u_int   *kernel_pagtable_store;         /* 128k of storage to map the kernel */

/*
 * Memory pools and back-end supplier for SRMMU page tables.
 * Share a pool between the level 2 and level 3 page tables,
 * since these are equal in size.
 */
static struct pool L1_pool;
static struct pool L23_pool;

static void *pgt_page_alloc(struct pool *, int);
static void  pgt_page_free(struct pool *, void *);

static struct pool_allocator pgt_page_allocator = {
        pgt_page_alloc, pgt_page_free, 0,
};

#endif /* SUN4M || SUN4D */

#if defined(SUN4) || defined(SUN4C)
/*
 * Memory pool for user and kernel PTE tables.
 */
static struct pool pte_pool;
#endif

struct  memarr *pmemarr;        /* physical memory regions */
int     npmemarr;               /* number of entries in pmemarr */

static paddr_t  avail_start;    /* first available physical page, other
                                   than the `etext gap' defined below */
static vaddr_t  etext_gap_start;/* start of gap between text & data */
static vaddr_t  etext_gap_end;  /* end of gap between text & data */
static vaddr_t  virtual_avail;  /* first free kernel virtual address */
static vaddr_t  virtual_end;    /* last free kernel virtual address */

static void pmap_page_upload(void);

int mmu_has_hole;

vaddr_t prom_vstart;    /* For /dev/kmem */
vaddr_t prom_vend;

/*
 * Memory pool for pmap structures.
 */
static struct pool_cache pmap_cache;
static int      pmap_pmap_pool_ctor(void *, void *, int);
static void     pmap_pmap_pool_dtor(void *, void *);
static struct pool segmap_pool;

#if defined(SUN4)
/*
 * [sun4]: segfixmask: on some systems (4/110) "getsegmap()" returns a
 * partly invalid value. getsegmap returns a 16 bit value on the sun4,
 * but only the first 8 or so bits are valid (the rest are *supposed* to
 * be zero. On the 4/110 the bits that are supposed to be zero are
 * all one instead. e.g. KERNBASE is usually mapped by pmeg number zero.
 * On a 4/300 getsegmap(KERNBASE) == 0x0000, but
 * on a 4/100 getsegmap(KERNBASE) == 0xff00
 *
 * This confuses mmu_reservemon() and causes it to not reserve the PROM's
 * pmegs. Then the PROM's pmegs get used during autoconfig and everything
 * falls apart!  (not very fun to debug, BTW.)
 *
 * solution: mask the invalid bits in the getsetmap macro.
 */

static u_int segfixmask = 0xffffffff; /* all bits valid to start */
#else
#define segfixmask 0xffffffff   /* It's in getsegmap's scope */
#endif

/*
 * pseudo-functions for mnemonic value
 */
#define getsegmap(va)           (CPU_ISSUN4C \
                                        ? lduba(va, ASI_SEGMAP) \
                                        : (lduha(va, ASI_SEGMAP) & segfixmask))
#define setsegmap(va, pmeg)     (CPU_ISSUN4C \
                                        ? stba(va, ASI_SEGMAP, pmeg) \
                                        : stha(va, ASI_SEGMAP, pmeg))

/* 3-level sun4 MMU only: */
#define getregmap(va)           ((unsigned)lduha((va)+2, ASI_REGMAP) >> 8)
#define setregmap(va, smeg)     stha((va)+2, ASI_REGMAP, (smeg << 8))


#if defined(SUN4M) || defined(SUN4D)
#if 0
#if VM_PROT_READ != 1 || VM_PROT_WRITE != 2 || VM_PROT_EXECUTE != 4
#error fix protection code translation table
#endif
#endif
/*
 * Translation table for kernel vs. PTE protection bits.
 */
const u_int protection_codes[2][8] = {
        /* kernel */
        {
        PPROT_N_RX,     /* VM_PROT_NONE    | VM_PROT_NONE  | VM_PROT_NONE */
        PPROT_N_RX,     /* VM_PROT_NONE    | VM_PROT_NONE  | VM_PROT_READ */
        PPROT_N_RWX,    /* VM_PROT_NONE    | VM_PROT_WRITE | VM_PROT_NONE */
        PPROT_N_RWX,    /* VM_PROT_NONE    | VM_PROT_WRITE | VM_PROT_READ */
        PPROT_N_RX,     /* VM_PROT_EXECUTE | VM_PROT_NONE  | VM_PROT_NONE */
        PPROT_N_RX,     /* VM_PROT_EXECUTE | VM_PROT_NONE  | VM_PROT_READ */
        PPROT_N_RWX,    /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_NONE */
        PPROT_N_RWX,    /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_READ */
        },

        /* user */
        {
        PPROT_N_RX,     /* VM_PROT_NONE    | VM_PROT_NONE  | VM_PROT_NONE */
        PPROT_R_R,      /* VM_PROT_NONE    | VM_PROT_NONE  | VM_PROT_READ */
        PPROT_RW_RW,    /* VM_PROT_NONE    | VM_PROT_WRITE | VM_PROT_NONE */
        PPROT_RW_RW,    /* VM_PROT_NONE    | VM_PROT_WRITE | VM_PROT_READ */
        PPROT_X_X,      /* VM_PROT_EXECUTE | VM_PROT_NONE  | VM_PROT_NONE */
        PPROT_RX_RX,    /* VM_PROT_EXECUTE | VM_PROT_NONE  | VM_PROT_READ */
        PPROT_RWX_RWX,  /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_NONE */
        PPROT_RWX_RWX,  /* VM_PROT_EXECUTE | VM_PROT_WRITE | VM_PROT_READ */
        }
};
#define pte_kprot4m(prot) (protection_codes[0][(prot)])
#define pte_uprot4m(prot) (protection_codes[1][(prot)])
#define pte_prot4m(pm, prot) \
        (protection_codes[(pm) == pmap_kernel() ? 0 : 1][(prot)])

void            setpte4m(vaddr_t va, int pte);
void            setpgt4m(int *ptep, int pte);
void            setpgt4m_va(vaddr_t, int *, int, int, int, u_int);
int             updatepte4m(vaddr_t, int *, int, int, int, u_int);
#endif /* SUN4M || SUN4D */

#if defined(MULTIPROCESSOR)
#define PMAP_SET_CPUSET(pmap, cpi)      \
        (pmap->pm_cpuset |= (1 << (cpi)->ci_cpuid))
#define PMAP_CLR_CPUSET(pmap, cpi)      \
        (pmap->pm_cpuset &= ~(1 << (cpi)->ci_cpuid))
#define PMAP_CPUSET(pmap)               (pmap->pm_cpuset)
#else
#define PMAP_SET_CPUSET(pmap, cpi)      /* nothing */
#define PMAP_CLR_CPUSET(pmap, cpi)      /* nothing */
#define PMAP_CPUSET(pmap)               1       /* XXX: 1 or 0? */
#endif /* MULTIPROCESSOR */


/* Function pointer messiness for supporting multiple sparc architectures
 * within a single kernel: notice that there are two versions of many of the
 * functions within this file/module, one for the sun4/sun4c and the other
 * for the sun4m. For performance reasons (since things like pte bits don't
 * map nicely between the two architectures), there are separate functions
 * rather than unified functions which test the cputyp variable. If only
 * one architecture is being used, then the non-suffixed function calls
 * are macro-translated into the appropriate xxx4_4c or xxx4m call. If
 * multiple architectures are defined, the calls translate to (*xxx_p),
 * i.e. they indirect through function pointers initialized as appropriate
 * to the run-time architecture in pmap_bootstrap. See also pmap.h.
 */

#if defined(SUN4M) || defined(SUN4D)
static void mmu_setup4m_L1(int, struct pmap *);
static void mmu_setup4m_L2(int, struct regmap *);
static void  mmu_setup4m_L3(int, struct segmap *);
/*static*/ void mmu_reservemon4m(struct pmap *);

/*static*/ void pmap_changeprot4m(pmap_t, vaddr_t, vm_prot_t, int);
/*static*/ void pmap_rmk4m(struct pmap *, vaddr_t, vaddr_t, int, int);
/*static*/ void pmap_rmu4m(struct pmap *, vaddr_t, vaddr_t, int, int);
/*static*/ int  pmap_enk4m(struct pmap *, vaddr_t, vm_prot_t,
                                int, struct vm_page *, int);
/*static*/ int  pmap_enu4m(struct pmap *, vaddr_t, vm_prot_t,
                                int, struct vm_page *, int);
/*static*/ void pv_changepte4m(struct vm_page *, int, int);
/*static*/ int  pv_syncflags4m(struct vm_page *);
/*static*/ int  pv_link4m(struct vm_page *, struct pmap *, vaddr_t, u_int *);
/*static*/ void pv_unlink4m(struct vm_page *, struct pmap *, vaddr_t);
#endif

#if defined(SUN4) || defined(SUN4C)
/*static*/ void mmu_reservemon4_4c(int *, int *);
/*static*/ void pmap_changeprot4_4c(pmap_t, vaddr_t, vm_prot_t, int);
/*static*/ void pmap_rmk4_4c(struct pmap *, vaddr_t, vaddr_t, int, int);
/*static*/ void pmap_rmu4_4c(struct pmap *, vaddr_t, vaddr_t, int, int);
/*static*/ int  pmap_enk4_4c(struct pmap *, vaddr_t, vm_prot_t,
                                  int, struct vm_page *, int);
/*static*/ int  pmap_enu4_4c(struct pmap *, vaddr_t, vm_prot_t,
                                  int, struct vm_page *, int);
/*static*/ void pv_changepte4_4c(struct vm_page *, int, int);
/*static*/ int  pv_syncflags4_4c(struct vm_page *);
/*static*/ int  pv_link4_4c(struct vm_page *, struct pmap *, vaddr_t, u_int *);
/*static*/ void pv_unlink4_4c(struct vm_page *, struct pmap *, vaddr_t);
#endif

#if !(defined(SUN4M) || defined(SUN4D)) && (defined(SUN4) || defined(SUN4C))
#define         pmap_rmk        pmap_rmk4_4c
#define         pmap_rmu        pmap_rmu4_4c

#elif (defined(SUN4M) || defined(SUN4D)) && !(defined(SUN4) || defined(SUN4C))
#define         pmap_rmk        pmap_rmk4m
#define         pmap_rmu        pmap_rmu4m

#else  /* must use function pointers */

/* function pointer declarations */
/* from pmap.h: */
bool            (*pmap_clear_modify_p)(struct vm_page *);
bool            (*pmap_clear_reference_p)(struct vm_page *);
int             (*pmap_enter_p)(pmap_t, vaddr_t, paddr_t, vm_prot_t, u_int);
bool            (*pmap_extract_p)(pmap_t, vaddr_t, paddr_t *);
bool            (*pmap_is_modified_p)(struct vm_page *);
bool            (*pmap_is_referenced_p)(struct vm_page *);
void            (*pmap_kenter_pa_p)(vaddr_t, paddr_t, vm_prot_t, u_int);
void            (*pmap_kremove_p)(vaddr_t, vsize_t);
void            (*pmap_kprotect_p)(vaddr_t, vsize_t, vm_prot_t);
void            (*pmap_page_protect_p)(struct vm_page *, vm_prot_t);
void            (*pmap_protect_p)(pmap_t, vaddr_t, vaddr_t, vm_prot_t);
/* local: */
void            (*pmap_rmk_p)(struct pmap *, vaddr_t, vaddr_t, int, int);
void            (*pmap_rmu_p)(struct pmap *, vaddr_t, vaddr_t, int, int);

#define         pmap_rmk        (*pmap_rmk_p)
#define         pmap_rmu        (*pmap_rmu_p)

#endif

/* --------------------------------------------------------------*/

/*
 * Next we have some sun4m/4d-specific routines which have no 4/4c
 * counterparts, or which are 4/4c macros.
 */

#if defined(SUN4M) || defined(SUN4D)
/*
 * SP versions of the tlb flush operations.
 *
 * Turn off traps to prevent register window overflows from writing
 * user windows to the wrong stack.  Cf. tlb_flush_page_real() &c.
 */
static void
sp_tlb_flush(int va, int ctx, int lvl)
{
        int opsr, octx;

        va &= ~0xfff;
        va |= lvl;

        /*
         * Turn off traps.
         *
         * Like setpsr((opsr = getpsr()) & ~PSR_ET); but we can shave
         * off one instruction b/c we never disable traps recursively,
         * so we can use the xor done by wrpsr itself to clear the
         * bit.
         *
         * XXX: Add to asm.h?  We can use this in cache.c too.
         */
        opsr = getpsr();        /* KDASSERT(opsr & PSR_ET); */
        __asm volatile ("wr %0, %1, %%psr"
                        :: "r"(opsr), "n"(PSR_ET) : "memory");
        __asm volatile ("nop; nop; nop");

        octx = getcontext4m();  /* save context */

        /* Do the TLB flush in "ctx" */
        setcontext4m(ctx);
        __asm volatile ("sta %%g0, [%0]%1" :: "r"(va), "n"(ASI_SRMMUFP));

        setcontext4m(octx);     /* restore context */
        setpsr(opsr);           /* turn traps on again */
}

static inline void
sp_tlb_flush_all(void)
{

        sta(ASI_SRMMUFP_LN, ASI_SRMMUFP, 0);
}

#if defined(MULTIPROCESSOR)
/*
 * The SMP versions of the tlb flush routines.  We only need to
 * do a cross call for these on sun4m (Mbus) systems. sun4d systems
 * have an Xbus which broadcasts TLB demaps in hardware.
 */

static inline void      smp_tlb_flush_page (int va, int ctx, u_int cpuset);
static inline void      smp_tlb_flush_segment (int va, int ctx, u_int cpuset);
static inline void      smp_tlb_flush_region (int va, int ctx, u_int cpuset);
static inline void      smp_tlb_flush_context (int ctx, u_int cpuset);
static inline void      smp_tlb_flush_all (void);

static inline void
smp_tlb_flush_page(int va, int ctx, u_int cpuset)
{

        if (CPU_ISSUN4D) {
                sp_tlb_flush(va, ctx, ASI_SRMMUFP_L3);
        } else
                FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L3, cpuset);
}

static inline void
smp_tlb_flush_segment(int va, int ctx, u_int cpuset)
{

        if (CPU_ISSUN4D) {
                sp_tlb_flush(va, ctx, ASI_SRMMUFP_L2);
        } else
                FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L2, cpuset);
}

static inline void
smp_tlb_flush_region(int va, int ctx, u_int cpuset)
{

        if (CPU_ISSUN4D) {
                sp_tlb_flush(va, ctx, ASI_SRMMUFP_L1);
        } else
                FXCALL3(sp_tlb_flush, ft_tlb_flush, va, ctx, ASI_SRMMUFP_L1, cpuset);
}

static inline void
smp_tlb_flush_context(int ctx, u_int cpuset)
{

        if (CPU_ISSUN4D) {
                sp_tlb_flush(0, ctx, ASI_SRMMUFP_L0);
        } else
                FXCALL3(sp_tlb_flush, ft_tlb_flush, 0, ctx, ASI_SRMMUFP_L0, cpuset);
}

static inline void
smp_tlb_flush_all(void)
{

        if (CPU_ISSUN4D) {
                sp_tlb_flush_all();
        } else
                XCALL0(sp_tlb_flush_all, CPUSET_ALL);
}
#endif /* MULTIPROCESSOR */

#if defined(MULTIPROCESSOR)
#define tlb_flush_page(va,ctx,s)        smp_tlb_flush_page(va,ctx,s)
#define tlb_flush_segment(va,ctx,s)     smp_tlb_flush_segment(va,ctx,s)
#define tlb_flush_region(va,ctx,s)      smp_tlb_flush_region(va,ctx,s)
#define tlb_flush_context(ctx,s)        smp_tlb_flush_context(ctx,s)
#define tlb_flush_all()                 smp_tlb_flush_all()
#else
#define tlb_flush_page(va,ctx,s)        sp_tlb_flush(va,ctx,ASI_SRMMUFP_L3)
#define tlb_flush_segment(va,ctx,s)     sp_tlb_flush(va,ctx,ASI_SRMMUFP_L2)
#define tlb_flush_region(va,ctx,s)      sp_tlb_flush(va,ctx,ASI_SRMMUFP_L1)
#define tlb_flush_context(ctx,s)        sp_tlb_flush(0,ctx,ASI_SRMMUFP_L0)
#define tlb_flush_all()                 sp_tlb_flush_all()
#endif /* MULTIPROCESSOR */

static u_int    VA2PA(void *);
static u_long   srmmu_bypass_read(u_long);

/*
 * VA2PA(addr) -- converts a virtual address to a physical address using
 * the MMU's currently-installed page tables. As a side effect, the address
 * translation used may cause the associated pte to be encached. The correct
 * context for VA must be set before this is called.
 *
 * This routine should work with any level of mapping, as it is used
 * during bootup to interact with the ROM's initial L1 mapping of the kernel.
 */
static u_int
VA2PA(void *addr)
{
        u_int pte;

        /*
         * We'll use that handy SRMMU flush/probe.
         * Try each level in turn until we find a valid pte. Otherwise panic.
         */

        pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L3, ASI_SRMMUFP);
        /* Unlock fault status; required on Hypersparc modules */
        (void)lda(SRMMU_SFSR, ASI_SRMMU);
        if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
            return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
                    ((u_int)addr & 0xfff));

        /* A `TLB Flush Entire' is required before any L0, L1 or L2 probe */
        tlb_flush_all_real();

        pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L2, ASI_SRMMUFP);
        if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
            return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
                    ((u_int)addr & 0x3ffff));
        pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L1, ASI_SRMMUFP);
        if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
            return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
                    ((u_int)addr & 0xffffff));
        pte = lda(((u_int)addr & ~0xfff) | ASI_SRMMUFP_L0, ASI_SRMMUFP);
        if ((pte & SRMMU_TETYPE) == SRMMU_TEPTE)
            return (((pte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
                    ((u_int)addr & 0xffffffff));

#ifdef DIAGNOSTIC
        panic("VA2PA: Asked to translate unmapped VA %p", addr);
#else
        return (0);
#endif
}

/*
 * Atomically update a PTE entry, coping with hardware updating the
 * PTE at the same time we are.  This is the procedure that is
 * recommended in the SuperSPARC user's manual.
 */
int
updatepte4m(vaddr_t va, int *pte, int bic, int bis, int ctx, u_int cpuset)
{
        int oldval, swapval;
        volatile int *vpte = (volatile int *)pte;
        bool can_lock = lock_available;

        /*
         * Can only be one of these happening in the system
         * at any one time.
         */
        if (__predict_true(can_lock))
                mutex_spin_enter(&demap_lock);

        /*
         * The idea is to loop swapping zero into the pte, flushing
         * it, and repeating until it stays zero.  At this point,
         * there should be no more hardware accesses to this PTE
         * so we can modify it without losing any mod/ref info.
         */
        oldval = 0;
        do {
                swapval = 0;
                swap(vpte, swapval);
                tlb_flush_page(va, ctx, cpuset);
                oldval |= swapval;
        } while (__predict_false(*vpte != 0));

        swapval = (oldval & ~bic) | bis;
        swap(vpte, swapval);

        if (__predict_true(can_lock))
                mutex_spin_exit(&demap_lock);

        return (oldval);
}

inline void
setpgt4m(int *ptep, int pte)
{

        kpreempt_disable();
        swap(ptep, pte);
        kpreempt_enable();
}

inline void
setpgt4m_va(vaddr_t va, int *ptep, int pte, int pageflush, int ctx,
            u_int cpuset)
{

#if defined(MULTIPROCESSOR)
        updatepte4m(va, ptep, 0xffffffff, pte, pageflush ? ctx : 0, cpuset);
#else
        kpreempt_disable();
        if (__predict_true(pageflush))
                tlb_flush_page(va, ctx, 0);
        setpgt4m(ptep, pte);
        kpreempt_enable();
#endif /* MULTIPROCESSOR */
}

/* Set the page table entry for va to pte. */
void
setpte4m(vaddr_t va, int pte)
{
        struct pmap *pm;
        struct regmap *rp;
        struct segmap *sp;

#ifdef DEBUG
        if (getcontext4m() != 0)
                panic("setpte4m: user context");
#endif

        pm = pmap_kernel();
        rp = &pm->pm_regmap[VA_VREG(va)];
        sp = &rp->rg_segmap[VA_VSEG(va)];

        tlb_flush_page(va, 0, CPUSET_ALL);
        setpgt4m(sp->sg_pte + VA_SUN4M_VPG(va), pte);
}

/*
 * Page table pool back-end.
 */
void *
pgt_page_alloc(struct pool *pp, int flags)
{
        int cacheit = (CACHEINFO.c_flags & CACHE_PAGETABLES) != 0;
        struct vm_page *pg;
        vaddr_t va;
        paddr_t pa;

        /* Allocate a page of physical memory */
        while ((pg = uvm_pagealloc(NULL, 0, NULL, 0)) == NULL &&
               (flags & PR_WAITOK) != 0) {
                uvm_wait("pgtpg");
        }
        if (pg == NULL) {
                KASSERT((flags & PR_WAITOK) == 0);
                return NULL;
        }

        /* Allocate virtual memory */
        va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY |
                ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK));
        if (va == 0) {
                KASSERT((flags & PR_WAITOK) == 0);
                uvm_pagefree(pg);
                return (NULL);
        }

        /*
         * On systems with a physical data cache we need to flush this page
         * from the cache if the pagetables cannot be cached.
         * On systems with a virtually indexed data cache, we only need
         * to map it non-cacheable, since the page is not currently mapped.
         */
        pa = VM_PAGE_TO_PHYS(pg);
        if (cacheit == 0)
                pcache_flush_page(pa, 1);

        /* Map the page */
        pmap_kenter_pa(va, pa | (cacheit ? 0 : PMAP_NC),
            VM_PROT_READ | VM_PROT_WRITE, 0);
        pmap_update(pmap_kernel());

        return ((void *)va);
}

void
pgt_page_free(struct pool *pp, void *v)
{
        vaddr_t va;
        paddr_t pa;
        bool rv __diagused;

        va = (vaddr_t)v;
        rv = pmap_extract(pmap_kernel(), va, &pa);
        KASSERT(rv);
        uvm_pagefree(PHYS_TO_VM_PAGE(pa));
        pmap_kremove(va, PAGE_SIZE);
        uvm_km_free(kernel_map, va, PAGE_SIZE, UVM_KMF_VAONLY);
}
#endif /* SUN4M || SUN4D */

/*----------------------------------------------------------------*/

/*
 * The following three macros are to be used in sun4/sun4c code only.
 */
#if defined(SUN4_MMU3L)
#define CTX_USABLE(pm,rp) (                                     \
                ((pm)->pm_ctx != NULL &&                        \
                 (!HASSUN4_MMU3L || (rp)->rg_smeg != reginval)) \
)
#else
#define CTX_USABLE(pm,rp)       ((pm)->pm_ctx != NULL )
#endif

#define GAP_WIDEN(pm,vr) do if (CPU_HAS_SUNMMU) {               \
        if (vr + 1 == pm->pm_gap_start)                         \
                pm->pm_gap_start = vr;                          \
        if (vr == pm->pm_gap_end)                               \
                pm->pm_gap_end = vr + 1;                        \
} while (0)

#define GAP_SHRINK(pm,vr) do if (CPU_HAS_SUNMMU) {                      \
        int x;                                                          \
        x = pm->pm_gap_start + (pm->pm_gap_end - pm->pm_gap_start) / 2; \
        if (vr > x) {                                                   \
                if (vr < pm->pm_gap_end)                                \
                        pm->pm_gap_end = vr;                            \
        } else {                                                        \
                if (vr >= pm->pm_gap_start && x != pm->pm_gap_start)    \
                        pm->pm_gap_start = vr + 1;                      \
        }                                                               \
} while (0)


static void get_phys_mem(void **);
#if 0 /* not used */
void    kvm_iocache(char *, int);
#endif

#ifdef DEBUG
void    pm_check(char *, struct pmap *);
void    pm_check_k(char *, struct pmap *);
void    pm_check_u(char *, struct pmap *);
#endif

/*
 * During the PMAP bootstrap, we can use a simple translation to map a
 * kernel virtual address to a psysical memory address (this is arranged
 * in locore).  Usually, KERNBASE maps to physical address 0. This is always
 * the case on sun4 and sun4c machines. On sun4m machines -- if no memory is
 * installed in the bank corresponding to physical address 0 -- the PROM may
 * elect to load us at some other address, presumably at the start of
 * the first memory bank that is available. We set the up the variable
 * `va2pa_offset' to hold the physical address corresponding to KERNBASE.
 */

static u_long va2pa_offset;
#define PMAP_BOOTSTRAP_VA2PA(v) ((paddr_t)((u_long)(v) - va2pa_offset))
#define PMAP_BOOTSTRAP_PA2VA(p) ((vaddr_t)((u_long)(p) + va2pa_offset))

/*
 * Grab physical memory list.
 * While here, compute `physmem'.
 */
void
get_phys_mem(void **top)
{
        struct memarr *mp;
        char *p;
        int i;

        /* Load the memory descriptor array at the current kernel top */
        p = (void *)ALIGN(*top);
        pmemarr = (struct memarr *)p;
        npmemarr = prom_makememarr(pmemarr, 1000, MEMARR_AVAILPHYS);

        /* Update kernel top */
        p += npmemarr * sizeof(struct memarr);
        *top = p;

        for (physmem = 0, mp = pmemarr, i = npmemarr; --i >= 0; mp++)
                physmem += btoc(mp->len);
}


/*
 * Support functions for vm_page_bootstrap().
 */

/*
 * How much virtual space does this kernel have?
 * (After mapping kernel text, data, etc.)
 */
void
pmap_virtual_space(vaddr_t *v_start, vaddr_t *v_end)
{

        *v_start = virtual_avail;
        *v_end   = virtual_end;
}

#ifdef PMAP_GROWKERNEL
vaddr_t
pmap_growkernel(vaddr_t eva)
{
        struct regmap *rp;
        struct segmap *sp;
        int vr, evr, M, N, i;
        struct vm_page *pg;
        vaddr_t va;

        if (eva <= virtual_end)
                return (virtual_end);

        /* For now, only implemented for sun4/sun4c */
        KASSERT(CPU_HAS_SUNMMU);

        /*
         * Map in the next region(s)
         */

        /* Get current end-of-kernel */
        vr = virtual_end >> RGSHIFT;
        evr = (eva + NBPRG - 1) >> RGSHIFT;
        eva = evr << RGSHIFT;

        if (eva > VM_MAX_KERNEL_ADDRESS)
                panic("growkernel: grown too large: %lx", eva);

        /*
         * Divide a region in N blocks of M segments, where each segment
         * block can have its PTEs mapped by one page.
         * N should come out to 1 for 8K pages and to 4 for 4K pages.
         */
        M = NBPG / (NPTESG * sizeof(int));
        N = (NBPRG/NBPSG) / M;

        while (vr < evr) {
                rp = &pmap_kernel()->pm_regmap[vr];
                for (i = 0; i < N; i++) {
                        sp = &rp->rg_segmap[i * M];
                        va = (vaddr_t)sp->sg_pte;
                        pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
                        if (pg == NULL)
                                panic("growkernel: out of memory");
                        pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
                                        VM_PROT_READ | VM_PROT_WRITE, 0);
                }
        }

        virtual_end = eva;
        return (eva);
}
#endif

/*
 * Helper routine that hands off available physical pages to the VM system.
 */
static void
pmap_page_upload(void)
{
        int     n;
        paddr_t pstart, pend;

        /* First, the `etext gap' */
        pstart = PMAP_BOOTSTRAP_VA2PA(etext_gap_start);
        pend = PMAP_BOOTSTRAP_VA2PA(etext_gap_end);

#ifdef DIAGNOSTIC
        if (avail_start <= pstart)
                panic("pmap_page_upload: etext gap overlap: %lx < %lx",
                        (u_long)avail_start, (u_long)pstart);
#endif
        if (etext_gap_start < etext_gap_end) {
                vm_first_phys = pstart;
                uvm_page_physload(
                        atop(pstart),
                        atop(pend),
                        atop(pstart),
                        atop(pend), VM_FREELIST_DEFAULT);
        }

        for (n = 0; n < npmemarr; n++) {

                pstart = pmemarr[n].addr;
                pend = pstart + pmemarr[n].len;

                /* Update vm_{first_last}_phys */
                if (vm_first_phys > pstart)
                        vm_first_phys = pstart;
                if (vm_last_phys < pend)
                        vm_last_phys = pend;

                /*
                 * Exclude any memory allocated for the kernel as computed
                 * by pmap_bootstrap(), i.e. the range
                 *      [KERNBASE_PA, avail_start>.
                 * Note that this will also exclude the `etext gap' range
                 * already uploaded above.
                 */
                if (pstart < PMAP_BOOTSTRAP_VA2PA(KERNBASE)) {
                        /*
                         * This segment starts below the kernel load address.
                         * Chop it off at the pstart of the kernel.
                         */
                        paddr_t chop = PMAP_BOOTSTRAP_VA2PA(KERNBASE);

                        if (pend < chop)
                                chop = pend;
#ifdef DEBUG
                        prom_printf("bootstrap gap: pstart %lx, chop %lx, pend %lx\n",
                                pstart, chop, pend);
#endif
                        uvm_page_physload(
                                atop(pstart),
                                atop(chop),
                                atop(pstart),
                                atop(chop),
                                VM_FREELIST_DEFAULT);

                        /*
                         * Adjust the start address to reflect the
                         * uploaded portion of this segment.
                         */
                        pstart = chop;
                }

                /* Skip the current kernel address range */
                if (pstart <= avail_start && avail_start < pend)
                        pstart = avail_start;

                if (pstart == pend)
                        continue;

                /* Upload (the rest of) this segment */
                uvm_page_physload(
                        atop(pstart),
                        atop(pend),
                        atop(pstart),
                        atop(pend), VM_FREELIST_DEFAULT);
        }

#if defined(MULTIPROCESSOR)
        {
                CPU_INFO_ITERATOR cpunum;
                struct cpu_info *cpi;

                for (CPU_INFO_FOREACH(cpunum, cpi)) {
                        if (cpi->ci_free_sva1)
                                uvm_page_physload(atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva1)),
                                                  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva1)),
                                                  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva1)),
                                                  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva1)),
                                                  VM_FREELIST_DEFAULT);
                        if (cpi->ci_free_sva2)
                                uvm_page_physload(atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva2)),
                                                  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva2)),
                                                  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_sva2)),
                                                  atop(PMAP_BOOTSTRAP_VA2PA(cpi->ci_free_eva2)),
                                                  VM_FREELIST_DEFAULT);
                }
        }
#endif
}

/*
 * This routine is used by mmrw() to validate access to `/dev/mem'.
 */
int
pmap_pa_exists(paddr_t pa)
{
        int nmem;
        struct memarr *mp;

        for (mp = pmemarr, nmem = npmemarr; --nmem >= 0; mp++) {
                if (pa >= mp->addr && pa < mp->addr + mp->len)
                        return 1;
        }

        return 0;
}

/* update pv_flags given a valid pte */
#define MR4_4C(pte) (((pte) >> PG_M_SHIFT) & (PV_MOD | PV_REF))
#define MR4M(pte) (((pte) >> PG_M_SHIFT4M) & (PV_MOD4M | PV_REF4M))

/*----------------------------------------------------------------*/

/*
 * Agree with the monitor ROM as to how many MMU entries are
 * to be reserved, and map all of its segments into all contexts.
 *
 * Unfortunately, while the Version 0 PROM had a nice linked list of
 * taken virtual memory, the Version 2 PROM provides instead a convoluted
 * description of *free* virtual memory.  Rather than invert this, we
 * resort to two magic constants from the PROM vector description file.
 */
#if defined(SUN4) || defined(SUN4C)
void
mmu_reservemon4_4c(int *nrp, int *nsp)
{
        u_int va = 0, eva = 0;
        int mmuseg, i, nr, ns, vr;
        int *pte;
#if defined(SUN4_MMU3L)
        int mmureg, lastvr = 0;
#endif
        struct regmap *rp;

#if defined(SUN4)
        if (CPU_ISSUN4) {
                prom_vstart = va = OLDMON_STARTVADDR;
                prom_vend = eva = OLDMON_ENDVADDR;
        }
#endif
#if defined(SUN4C)
        if (CPU_ISSUN4C) {
                prom_vstart = va = OPENPROM_STARTVADDR;
                prom_vend = eva = OPENPROM_ENDVADDR;
        }
#endif
        ns = *nsp;
        nr = *nrp;
        while (va < eva) {
                vr = VA_VREG(va);
                rp = &pmap_kernel()->pm_regmap[vr];

#if defined(SUN4_MMU3L)
                if (HASSUN4_MMU3L && vr != lastvr) {
                        lastvr = vr;
                        mmureg = getregmap(va);
                        if (mmureg < nr)
                                rp->rg_smeg = nr = mmureg;
                        /*
                         * On 3-level MMU machines, we distribute regions,
                         * rather than segments, amongst the contexts.
                         */
                        for (i = ncontext; --i > 0;)
                                prom_setcontext(i, (void *)va, mmureg);
                }
#endif
                mmuseg = getsegmap(va);
                if (mmuseg < ns)
                        ns = mmuseg;

                if (!HASSUN4_MMU3L)
                        for (i = ncontext; --i > 0;)
                                prom_setcontext(i, (void *)va, mmuseg);

                if (mmuseg == seginval) {
                        va += NBPSG;
                        continue;
                }
                /*
                 * Another PROM segment. Enter into region map.
                 * Assume the entire segment is valid.
                 */
                rp->rg_nsegmap += 1;
                rp->rg_segmap[VA_VSEG(va)].sg_pmeg = mmuseg;
                rp->rg_segmap[VA_VSEG(va)].sg_npte = NPTESG;
                pte = rp->rg_segmap[VA_VSEG(va)].sg_pte;

                /* PROM maps its memory user-accessible: fix it. */
                for (i = NPTESG; --i >= 0; va += NBPG, pte++) {
                        *pte = getpte4(va) | PG_S;
                        setpte4(va, *pte);
                }
        }
        *nsp = ns;
        *nrp = nr;
        return;
}
#endif

#if defined(SUN4M) || defined(SUN4D) /* SRMMU versions of above */

u_long
srmmu_bypass_read(u_long paddr)
{
        unsigned long v;

        if (cpuinfo.mxcc) {
                /*
                 * We're going to have to use MMU passthrough. If we're on
                 * a Viking SuperSPARC with a MultiCache Controller, we
                 * need to set the AC (Alternate Cacheable) bit in the MMU's
                 * control register in order to not by-pass the cache.
                 */

                unsigned long s = lda(SRMMU_PCR, ASI_SRMMU);

                /* set MMU AC bit */
                sta(SRMMU_PCR, ASI_SRMMU, s | VIKING_PCR_AC);
                v = lda(paddr, ASI_BYPASS);
                sta(SRMMU_PCR, ASI_SRMMU, s);
        } else
                v = lda(paddr, ASI_BYPASS);

        return (v);
}


/*
 * Take the monitor's initial page table layout, convert it to 3rd-level pte's
 * (it starts out as a L1 mapping), and install it along with a set of kernel
 * mapping tables as the kernel's initial page table setup. Also create and
 * enable a context table. I suppose we also want to block user-mode access
 * to the new kernel/ROM mappings.
 */

/*
 * mmu_reservemon4m(): Copies the existing (ROM) page tables to kernel space,
 * converting any L1/L2 PTEs to L3 PTEs. Does *not* copy the L1 entry mapping
 * the kernel at KERNBASE since we don't want to map 16M of physical
 * memory for the kernel. Thus the kernel must be installed later!
 * Also installs ROM mappings into the kernel pmap.
 * NOTE: This also revokes all user-mode access to the mapped regions.
 */
void
mmu_reservemon4m(struct pmap *kpmap)
{
        unsigned int rom_ctxtbl;
        int te;

#if !(defined(PROM_AT_F0) || defined(MSIIEP))
        prom_vstart = OPENPROM_STARTVADDR;
        prom_vend = OPENPROM_ENDVADDR;
#else /* OBP3/OFW in JavaStations */
        prom_vstart = 0xf0000000;
#if defined(MSIIEP)
        prom_vend = 0xf0800000;
#else
        prom_vend = 0xf0080000;
#endif
#endif

        /*
         * XXX: although the sun4m can handle 36 bits of physical
         * address space, we assume that all these page tables, etc
         * are in the lower 4G (32-bits) of address space, i.e. out of I/O
         * space. Eventually this should be changed to support the 36 bit
         * physical addressing, in case some crazed ROM designer decides to
         * stick the pagetables up there. In that case, we should use MMU
         * transparent mode, (i.e. ASI 0x20 to 0x2f) to access
         * physical memory.
         */

        rom_ctxtbl = (lda(SRMMU_CXTPTR,ASI_SRMMU) << SRMMU_PPNPASHIFT);

        te = srmmu_bypass_read(rom_ctxtbl);     /* i.e. context 0 */

        switch (te & SRMMU_TETYPE) {
        case SRMMU_TEINVALID:
                cpuinfo.ctx_tbl[0] = SRMMU_TEINVALID;
                panic("mmu_reservemon4m: no existing L0 mapping! "
                      "(How are we running?");
                break;
        case SRMMU_TEPTE:
                panic("mmu_reservemon4m: can't handle ROM 4G page size");
                /* XXX: Should make this work, however stupid it is */
                break;
        case SRMMU_TEPTD:
                mmu_setup4m_L1(te, kpmap);
                break;
        default:
                panic("mmu_reservemon4m: unknown pagetable entry type");
        }
}

/* regtblptd - PTD for region table to be remapped */
void
mmu_setup4m_L1(int regtblptd, struct pmap *kpmap)
{
        unsigned int regtblrover;
        int i;
        unsigned int te;
        struct regmap *rp;
        int j, k;

        /*
         * Here we scan the region table to copy any entries which appear.
         * We are only concerned with regions in kernel space and above
         * (i.e. regions VA_VREG(KERNBASE)+1 to 0xff). We ignore the first
         * region (at VA_VREG(KERNBASE)), since that is the 16MB L1 mapping
         * that the ROM used to map the kernel in initially. Later, we will
         * rebuild a new L3 mapping for the kernel and install it before
         * switching to the new pagetables.
         */
        regtblrover =
                ((regtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT) +
                (VA_VREG(KERNBASE)+1) * sizeof(long);   /* kernel only */

        for (i = VA_VREG(KERNBASE) + 1; i < SRMMU_L1SIZE;
             i++, regtblrover += sizeof(long)) {

                /* The region we're dealing with */
                rp = &kpmap->pm_regmap[i];

                te = srmmu_bypass_read(regtblrover);
                switch(te & SRMMU_TETYPE) {
                case SRMMU_TEINVALID:
                        break;

                case SRMMU_TEPTE:
#ifdef DEBUG
                        prom_printf("mmu_setup4m_L1: "
                               "converting region 0x%x from L1->L3\n", i);
#endif
                        /*
                         * This region entry covers 64MB of memory -- or
                         * (NSEGRG * NPTESG) pages -- which we must convert
                         * into a 3-level description.
                         */

                        for (j = 0; j < SRMMU_L2SIZE; j++) {
                                struct segmap *sp = &rp->rg_segmap[j];

                                for (k = 0; k < SRMMU_L3SIZE; k++) {
                                        setpgt4m(&sp->sg_pte[k],
                                                (te & SRMMU_L1PPNMASK) |
                                                (j << SRMMU_L2PPNSHFT) |
                                                (k << SRMMU_L3PPNSHFT) |
                                                (te & SRMMU_PGBITSMSK) |
                                                ((te & SRMMU_PROT_MASK) |
                                                 PPROT_U2S_OMASK) |
                                                SRMMU_TEPTE);
                                }
                        }
                        break;

                case SRMMU_TEPTD:
                        mmu_setup4m_L2(te, rp);
                        break;

                default:
                        panic("mmu_setup4m_L1: unknown pagetable entry type");
                }
        }
}

void
mmu_setup4m_L2(int segtblptd, struct regmap *rp)
{
        unsigned int segtblrover;
        int i, k;
        unsigned int te;
        struct segmap *sp;

        segtblrover = (segtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT;
        for (i = 0; i < SRMMU_L2SIZE; i++, segtblrover += sizeof(long)) {

                sp = &rp->rg_segmap[i];

                te = srmmu_bypass_read(segtblrover);
                switch(te & SRMMU_TETYPE) {
                case SRMMU_TEINVALID:
                        break;

                case SRMMU_TEPTE:
#ifdef DEBUG
                        prom_printf("mmu_setup4m_L2: converting L2 entry at segment 0x%x to L3\n",i);
#endif
                        /*
                         * This segment entry covers 256KB of memory -- or
                         * (NPTESG) pages -- which we must convert
                         * into a 3-level description.
                         */
                        for (k = 0; k < SRMMU_L3SIZE; k++) {
                                setpgt4m(&sp->sg_pte[k],
                                        (te & SRMMU_L1PPNMASK) |
                                        (te & SRMMU_L2PPNMASK) |
                                        (k << SRMMU_L3PPNSHFT) |
                                        (te & SRMMU_PGBITSMSK) |
                                        ((te & SRMMU_PROT_MASK) |
                                         PPROT_U2S_OMASK) |
                                        SRMMU_TEPTE);
                        }
                        break;

                case SRMMU_TEPTD:
                        mmu_setup4m_L3(te, sp);
                        break;

                default:
                        panic("mmu_setup4m_L2: unknown pagetable entry type");
                }
        }
}

void
mmu_setup4m_L3(int pagtblptd, struct segmap *sp)
{
        unsigned int pagtblrover;
        int i;
        unsigned int te;

        pagtblrover = (pagtblptd & ~SRMMU_TETYPE) << SRMMU_PPNPASHIFT;
        for (i = 0; i < SRMMU_L3SIZE; i++, pagtblrover += sizeof(long)) {
                te = srmmu_bypass_read(pagtblrover);
                switch(te & SRMMU_TETYPE) {
                case SRMMU_TEINVALID:
                        break;
                case SRMMU_TEPTE:
                        setpgt4m(&sp->sg_pte[i], te | PPROT_U2S_OMASK);
                        pmap_kernel()->pm_stats.resident_count++;
                        break;
                case SRMMU_TEPTD:
                        panic("mmu_setup4m_L3: PTD found in L3 page table");
                default:
                        panic("mmu_setup4m_L3: unknown pagetable entry type");
                }
        }
}
#endif /* defined SUN4M || defined SUN4D */

/*----------------------------------------------------------------*/

#if defined(SUN4) || defined(SUN4C)
/*
 * MMU management.
 */
static int      me_alloc(struct mmuentry *, struct pmap *, int, int);
static void     me_free(struct pmap *, u_int);
#if defined(SUN4_MMU3L)
static int      region_alloc(struct mmuentry *, struct pmap *, int);
static void     region_free(struct pmap *, u_int);
#endif


/*
 * Allocate an MMU entry (i.e., a PMEG).
 * If necessary, steal one from someone else.
 * Put it on the tail of the given queue
 * (which is either the LRU list or the locked list).
 * The locked list is not actually ordered, but this is easiest.
 * Also put it on the given (new) pmap's chain,
 * enter its pmeg number into that pmap's segmap,
 * and store the pmeg's new virtual segment number (me->me_vseg).
 *
 * This routine is large and complicated, but it must be fast
 * since it implements the dynamic allocation of MMU entries.
 */

static inline int
me_alloc(struct mmuentry *mh, struct pmap *newpm, int newvreg, int newvseg)
{
        struct mmuentry *me;
        struct pmap *pm;
        int i, va, *ptep, pte;
        int ctx;
        struct regmap *rp;
        struct segmap *sp;

        /* try free list first */
        if (!mmuq_empty(&segm_freelist)) {
                me = mmuq_first(&segm_freelist);
                mmuq_remove(me);
#ifdef DEBUG
                if (me->me_pmap != NULL)
                        panic("me_alloc: freelist entry has pmap");
                DPRINTF(PDB_MMU_ALLOC,
                    "me_alloc: got pmeg %d", me->me_cookie);
#endif
                mmuq_insert_tail(mh, me);

                /* onto on pmap chain; pmap is already locked, if needed */
                TAILQ_INSERT_TAIL(&newpm->pm_seglist, me, me_pmchain);
#ifdef DIAGNOSTIC
                pmap_stats.ps_npmeg_free--;
                if (mh == &segm_locked) {
                        pmap_stats.ps_npmeg_locked++;
                        me->me_statp = &pmap_stats.ps_npmeg_locked;
                } else {
                        pmap_stats.ps_npmeg_lru++;
                        me->me_statp = &pmap_stats.ps_npmeg_lru;
                }
#endif

                /* into pmap segment table, with backpointers */
                me->me_pmap = newpm;
                me->me_vseg = newvseg;
                me->me_vreg = newvreg;

                return (me->me_cookie);
        }

        /* no luck, take head of LRU list */
        if (mmuq_empty(&segm_lru))
                panic("me_alloc: all pmegs gone");

        me = mmuq_first(&segm_lru);
        pm = me->me_pmap;
        DPRINTF(PDB_MMU_ALLOC | PDB_MMU_STEAL,
            "me_alloc: stealing pmeg 0x%x from pmap %p", me->me_cookie, pm);

        mmu_stolenpmegs_evcnt.ev_count++;

        /*
         * Remove from LRU list, and insert at end of new list
         * (probably the LRU list again, but so what?).
         */
        mmuq_remove(me);
        mmuq_insert_tail(mh, me);

#ifdef DIAGNOSTIC
        if (mh == &segm_locked) {
                pmap_stats.ps_npmeg_lru--;
                pmap_stats.ps_npmeg_locked++;
                me->me_statp = &pmap_stats.ps_npmeg_locked;
        } else {
                me->me_statp = &pmap_stats.ps_npmeg_lru;
        }
#endif

        rp = &pm->pm_regmap[me->me_vreg];
        sp = &rp->rg_segmap[me->me_vseg];
        ptep = sp->sg_pte;

#ifdef DEBUG
        if (sp->sg_pmeg != me->me_cookie)
                panic("me_alloc: wrong sg_pmeg (%d != %d)",
                                sp->sg_pmeg, me->me_cookie);
#endif

        /*
         * The PMEG must be mapped into some context so that we can
         * read its PTEs.  Use its current context if it has one;
         * if not, and since context 0 is reserved for the kernel,
         * the simplest method is to switch to 0 and map the PMEG
         * to virtual address 0---which, being a user space address,
         * is by definition not in use.
         *
         * XXX do not have to flush cache immediately
         */
        ctx = getcontext4();

        /*
         * Even if we're stealing a PMEG from ourselves (i.e. if pm==newpm),
         * we must make sure there are no user register windows in the CPU
         * for the following reasons:
         * (1) if we have a write-allocate cache and the segment we are
         *     stealing contains stack pages, an interrupt during the
         *     interval that starts at cache_flush_segment() below and ends
         *     when the segment is finally removed from the MMU, may cause
         *     dirty cache lines to reappear.
         * (2) when re-wiring this PMEG for use by another segment (e.g.
         *     in mmu_pagein()) a window exists where the PTEs in this PMEG
         *     point at arbitrary pages allocated to this address space.
         *     Again, a register window flush at this point is likely to
         *     cause data corruption in case the segment being rewired
         *     contains stack virtual addresses.
         */
        write_user_windows();
        if (CTX_USABLE(pm,rp)) {
                setcontext4(pm->pm_ctxnum);
                va = VSTOVA(me->me_vreg, me->me_vseg);
#ifdef DEBUG
                if (getsegmap(va) != me->me_cookie)
                        panic("me_alloc: wrong pmeg in MMU (%d != %d)",
                                getsegmap(va), me->me_cookie);
#endif
                cache_flush_segment(me->me_vreg, me->me_vseg, pm->pm_ctxnum);
        } else {
                va = 0;
                setcontext4(0);
                if (HASSUN4_MMU3L)
                        setregmap(va, tregion);
                setsegmap(va, me->me_cookie);
                /*
                 * No cache flush needed: it happened earlier when
                 * the old context was taken.
                 */
        }

        /*
         * Record reference and modify bits for each page,
         * and copy PTEs into kernel memory so that they can
         * be reloaded later.
         */
        i = NPTESG;
        do {
                int swbits = *ptep & PG_MBZ;
                pte = getpte4(va);
                if ((pte & (PG_V | PG_TYPE)) == (PG_V | PG_OBMEM)) {
                        struct vm_page *pg;
                        if ((pg = pvhead4_4c(pte)) != NULL)
                                VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(pte);
                }
                *ptep++ = swbits | (pte & ~(PG_U|PG_M));
                va += NBPG;
        } while (--i > 0);

        /* update segment tables */
        if (CTX_USABLE(pm,rp)) {
                va = VSTOVA(me->me_vreg,me->me_vseg);
                if (pm != pmap_kernel() || HASSUN4_MMU3L)
                        setsegmap(va, seginval);
                else {
                        /* Unmap segment from all contexts */
                        for (i = ncontext; --i >= 0;) {
                                setcontext4(i);
                                setsegmap(va, seginval);
                        }
                }
        }
        sp->sg_pmeg = seginval;

        /* off old pmap chain */
        TAILQ_REMOVE(&pm->pm_seglist, me, me_pmchain);
        setcontext4(ctx);

        /* onto new pmap chain; new pmap is already locked, if needed */
        TAILQ_INSERT_TAIL(&newpm->pm_seglist, me, me_pmchain);

        /* into new segment table, with backpointers */
        me->me_pmap = newpm;
        me->me_vseg = newvseg;
        me->me_vreg = newvreg;

        return (me->me_cookie);
}

/*
 * Free an MMU entry.
 *
 * Assumes the corresponding pmap is already locked.
 * Caller must update hardware.
 */
static inline void
me_free(struct pmap *pm, u_int pmeg)
{
        struct mmuentry *me = &mmusegments[pmeg];
#ifdef DEBUG
        struct regmap *rp;
        int i, va, tpte, ctx;
#endif

#ifdef DEBUG
        rp = &pm->pm_regmap[me->me_vreg];
        DPRINTF(PDB_MMU_ALLOC,
           "me_free: freeing pmeg %d from pmap %p", me->me_cookie, pm);
        if (me->me_cookie != pmeg)
                panic("me_free: wrong mmuentry");
        if (pm != me->me_pmap)
                panic("me_free: pm != me_pmap");
        if (rp->rg_segmap[me->me_vseg].sg_pmeg != pmeg &&
            rp->rg_segmap[me->me_vseg].sg_pmeg != seginval)
                panic("me_free: wrong sg_pmeg (%d != %d)",
                        rp->rg_segmap[me->me_vseg].sg_pmeg, pmeg);

        /* check for spurious mappings (using temp. mapping in context 0) */
        ctx = getcontext4();
        setcontext4(0);
        if (HASSUN4_MMU3L)
                setregmap(0, tregion);
        setsegmap(0, me->me_cookie);
        va = 0;
        i = NPTESG;
        do {
                tpte = getpte4(va);
                if ((tpte & PG_V) == PG_V)
                        panic("me_free: segment not clean (pte=%x)", tpte);
                va += NBPG;
        } while (--i > 0);
        setcontext4(ctx);
#endif /* DEBUG */

        /* take mmu entry off pmap chain */
        TAILQ_REMOVE(&pm->pm_seglist, me, me_pmchain);

        /* off LRU or lock chain */
        mmuq_remove(me);
#ifdef DIAGNOSTIC
        if (me->me_statp == NULL)
                panic("me_statp");
        (*me->me_statp)--;
        me->me_statp = NULL;
#endif

        /* no associated pmap; on free list */
        me->me_pmap = NULL;
        mmuq_insert_tail(&segm_freelist, me);
#ifdef DIAGNOSTIC
        pmap_stats.ps_npmeg_free++;
#endif
}

#if defined(SUN4_MMU3L)

/* XXX - Merge with segm_alloc/segm_free ? */

int
region_alloc(struct mmuentry *mh, struct pmap *newpm, int newvr)
{
        struct mmuentry *me;
        struct pmap *pm;
        int ctx;
        struct regmap *rp;

        /* try free list first */
        if (!mmuq_empty(&region_freelist)) {
                me = mmuq_first(&region_freelist);
                mmuq_remove(me);
#ifdef DEBUG
                if (me->me_pmap != NULL)
                        panic("region_alloc: freelist entry has pmap");
                DPRINTF(PDB_MMUREG_ALLOC,
                    "region_alloc: got smeg 0x%x", me->me_cookie);
#endif
                mmuq_insert_tail(mh, me);

                /* onto on pmap chain; pmap is already locked, if needed */
                TAILQ_INSERT_TAIL(&newpm->pm_reglist, me, me_pmchain);

                /* into pmap segment table, with backpointers */
                me->me_pmap = newpm;
                me->me_vreg = newvr;

                return (me->me_cookie);
        }

        /* no luck, take head of LRU list */
        if (mmuq_empty(&region_lru))
                panic("region_alloc: all smegs gone");

        me = mmuq_first(&region_lru);

        pm = me->me_pmap;
        if (pm == NULL)
                panic("region_alloc: LRU entry has no pmap");
        if (pm == pmap_kernel())
                panic("region_alloc: stealing from kernel");
        DPRINTF(PDB_MMUREG_ALLOC | PDB_MMUREG_STEAL,
            "region_alloc: stealing smeg 0x%x from pmap %p",
            me->me_cookie, pm);

        /*
         * Remove from LRU list, and insert at end of new list
         * (probably the LRU list again, but so what?).
         */
        mmuq_remove(me);
        mmuq_insert_tail(mh, me);

        rp = &pm->pm_regmap[me->me_vreg];
        ctx = getcontext4();

        /* Flush register windows; see comment in me_alloc() */
        write_user_windows();
        if (pm->pm_ctx) {
                setcontext4(pm->pm_ctxnum);
                cache_flush_region(me->me_vreg, pm->pm_ctxnum);
        }

        /* update region tables */
        if (pm->pm_ctx)
                setregmap(VRTOVA(me->me_vreg), reginval);
        rp->rg_smeg = reginval;

        /* off old pmap chain */
        TAILQ_REMOVE(&pm->pm_reglist, me, me_pmchain);
        setcontext4(ctx);       /* done with old context */

        /* onto new pmap chain; new pmap is already locked, if needed */
        TAILQ_INSERT_TAIL(&newpm->pm_reglist, me, me_pmchain);

        /* into new segment table, with backpointers */
        me->me_pmap = newpm;
        me->me_vreg = newvr;

        return (me->me_cookie);
}

/*
 * Free an MMU entry.
 * Assumes the corresponding pmap is already locked.
 * Caller must update hardware.
 */
void
region_free(struct pmap *pm, u_int smeg)
{
        struct mmuentry *me = &mmuregions[smeg];

        DPRINTF(PDB_MMUREG_ALLOC,
            "region_free: freeing smeg 0x%x from pmap %p", me->me_cookie, pm);
#ifdef DEBUG
        if (me->me_cookie != smeg)
                panic("region_free: wrong mmuentry");
        if (pm != me->me_pmap)
                panic("region_free: pm != me_pmap");
#endif

        /* take mmu entry off pmap chain */
        TAILQ_REMOVE(&pm->pm_reglist, me, me_pmchain);

        /* off LRU or lock chain */
        mmuq_remove(me);

        /* no associated pmap; on free list */
        me->me_pmap = NULL;
        mmuq_insert_tail(&region_freelist, me);
}

static void
mmu_pagein_reg(struct pmap *pm, struct regmap *rp, vaddr_t va,
                int vr, struct mmuentry *mh)
{
        int i, s, smeg;

        va = VA_ROUNDDOWNTOREG(va);
        rp->rg_smeg = smeg = region_alloc(mh, pm, vr);

        s = splvm();
        if (pm == pmap_kernel()) {
                /* Map region into all contexts */
                int ctx = getcontext4();
                i = ncontext - 1;
                do {
                        setcontext4(i);
                        setregmap(va, smeg);
                } while (--i >= 0);
                setcontext4(ctx);
        } else
                setregmap(va, smeg);

        /* Load PMEGs into this region */
        for (i = 0; i < NSEGRG; i++) {
                setsegmap(va, rp->rg_segmap[i].sg_pmeg);
                va += NBPSG;
        }
        splx(s);
}
#endif /* SUN4_MMU3L */

static void
mmu_pmeg_lock(int pmeg)
{
        struct mmuentry *me = &mmusegments[pmeg];

        mmuq_remove(me);
        mmuq_insert_tail(&segm_locked, me);
#ifdef DIAGNOSTIC
        (*me->me_statp)--;
        pmap_stats.ps_npmeg_locked++;
        me->me_statp = &pmap_stats.ps_npmeg_locked;
#endif
}

static void
mmu_pmeg_unlock(int pmeg)
{
        struct mmuentry *me = &mmusegments[pmeg];

        mmuq_remove(me);
        mmuq_insert_tail(&segm_lru, me);
#ifdef DIAGNOSTIC
        (*me->me_statp)--;
        pmap_stats.ps_npmeg_lru++;
        me->me_statp = &pmap_stats.ps_npmeg_lru;
#endif
}

static void
mmu_pagein_seg(struct pmap *pm, struct segmap *sp, vaddr_t va,
                int vr, int vs, struct mmuentry *mh)
{
        int s, i, pmeg, *pte;

        mmu_pagein_evcnt.ev_count++;

        va = VA_ROUNDDOWNTOSEG(va);
        s = splvm();            /* paranoid */
        sp->sg_pmeg = pmeg = me_alloc(mh, pm, vr, vs);
        if (pm != pmap_kernel() || HASSUN4_MMU3L)
                setsegmap(va, pmeg);
        else {
                /* Map kernel address into all contexts */
                int ctx = getcontext4();
                i = ncontext - 1;
                do {
                        setcontext4(i);
                        setsegmap(va, pmeg);
                } while (--i >= 0);
                setcontext4(ctx);
        }

        /* reload segment: write PTEs into a the MMU */
        pte = sp->sg_pte;
        i = NPTESG;
        do {
                setpte4(va, *pte++ & ~PG_MBZ);
                va += NBPG;
        } while (--i > 0);
        splx(s);
}

/*
 * `Page in' (load or inspect) an MMU entry; called on page faults.
 * Returns 1 if we reloaded the segment, -1 if the segment was
 * already loaded and the page was marked valid (in which case the
 * fault must be a bus error or something), or 0 (segment loaded but
 * PTE not valid, or segment not loaded at all).
 */
int
mmu_pagein(struct pmap *pm, vaddr_t va, int prot)
{
        int vr, vs, bits;
        struct regmap *rp;
        struct segmap *sp;

        PMAP_LOCK();

        if (prot != VM_PROT_NONE)
                bits = PG_V | ((prot & VM_PROT_WRITE) ? PG_W : 0);
        else
                bits = 0;

        vr = VA_VREG(va);
        vs = VA_VSEG(va);
        rp = &pm->pm_regmap[vr];

        /* return 0 if we have no PMEGs to load */
        if (rp->rg_nsegmap == 0) {
                PMAP_UNLOCK();
                return (0);
        }

#ifdef DIAGNOSTIC
        if (rp->rg_segmap == NULL)
                panic("pagein: no segmap");
#endif

#if defined(SUN4_MMU3L)
        if (HASSUN4_MMU3L && rp->rg_smeg == reginval)
                mmu_pagein_reg(pm, rp, va, vr, &region_lru);
#endif
        sp = &rp->rg_segmap[vs];

        /* return 0 if we have no PTEs to load */
        if (sp->sg_npte == 0) {
                PMAP_UNLOCK();
                return (0);
        }

        /* return -1 if the fault is `hard', 0 if not */
        if (sp->sg_pmeg != seginval) {
                PMAP_UNLOCK();
                return (bits && (getpte4(va) & bits) == bits ? -1 : 0);
        }

        mmu_pagein_seg(pm, sp, va, vr, vs, &segm_lru);
        PMAP_UNLOCK();
        return (1);
}
#endif /* SUN4 or SUN4C */

/*
 * Allocate a context.  If necessary, steal one from someone else.
 * Changes hardware context number and loads segment map.
 *
 * This routine is only ever called from locore.s just after it has
 * saved away the previous process, so there are no active user windows.
 */
static void
ctx_alloc(struct pmap *pm)
{
        union ctxinfo *c;
        int cnum, i = 0, doflush;
        struct regmap *rp;
        int gap_start, gap_end;
        vaddr_t va;
#if defined(SUN4M) || defined(SUN4D)
        struct cpu_info *cpi;
#endif

        KASSERT(mutex_owned(&ctx_lock));

/*XXX-GCC!*/gap_start=gap_end=0;
#ifdef DEBUG
        if (pm->pm_ctx)
                panic("ctx_alloc pm_ctx");
#endif
        DPRINTF(PDB_CTX_ALLOC,
            "ctx_alloc[%d](%p)", cpu_number(), pm);

        if (CPU_HAS_SUNMMU) {
                gap_start = pm->pm_gap_start;
                gap_end = pm->pm_gap_end;
        }

        if ((c = ctx_freelist) != NULL) {
                ctx_freelist = c->c_nextfree;
                cnum = c - ctxinfo;
                doflush = 0;
        } else {
                if ((ctx_kick += ctx_kickdir) >= ncontext) {
                        ctx_kick = ncontext - 1;
                        ctx_kickdir = -1;
                } else if (ctx_kick < 1) {
                        ctx_kick = 1;
                        ctx_kickdir = 1;
                }
                c = &ctxinfo[cnum = ctx_kick];
#ifdef DEBUG
                if (c->c_pmap == NULL)
                        panic("ctx_alloc cu_pmap");
#endif
                DPRINTF(PDB_CTX_ALLOC | PDB_CTX_STEAL,
                    "ctx_alloc[%d]: steal context %d from %p",
                    cpu_number(), cnum, c->c_pmap);

                c->c_pmap->pm_ctx = NULL;
                c->c_pmap->pm_ctxnum = 0;
                doflush = (CACHEINFO.c_vactype != VAC_NONE);
                if (CPU_HAS_SUNMMU) {
                        if (gap_start < c->c_pmap->pm_gap_start)
                                gap_start = c->c_pmap->pm_gap_start;
                        if (gap_end > c->c_pmap->pm_gap_end)
                                gap_end = c->c_pmap->pm_gap_end;
                }
        }

        c->c_pmap = pm;
        pm->pm_ctx = c;
        pm->pm_ctxnum = cnum;

        if (CPU_HAS_SUNMMU) {

                /*
                 * Write pmap's region (3-level MMU) or segment table into
                 * the MMU.
                 *
                 * Only write those entries that actually map something in
                 * this context by maintaining a pair of region numbers in
                 * between which the pmap has no valid mappings.
                 *
                 * If a context was just allocated from the free list, trust
                 * that all its pmeg numbers are `seginval'. We make sure this
                 * is the case initially in pmap_bootstrap(). Otherwise, the
                 * context was freed by calling ctx_free() in pmap_release(),
                 * which in turn is supposedly called only when all mappings
                 * have been removed.
                 *
                 * On the other hand, if the context had to be stolen from
                 * another pmap, we possibly shrink the gap to be the
                 * disjuction of the new and the previous map.
                 */

                setcontext4(cnum);
                if (doflush)
                        cache_flush_context(cnum);

                rp = pm->pm_regmap;
                for (va = 0, i = NUREG; --i >= 0; ) {
                        if (VA_VREG(va) >= gap_start) {
                                va = VRTOVA(gap_end);
                                i -= gap_end - gap_start;
                                rp += gap_end - gap_start;
                                if (i < 0)
                                        break;
                                /* mustn't re-enter this branch */
                                gap_start = NUREG;
                        }
                        if (HASSUN4_MMU3L) {
                                setregmap(va, rp++->rg_smeg);
                                va += NBPRG;
                        } else {
                                int j;
                                struct segmap *sp = rp->rg_segmap;
                                for (j = NSEGRG; --j >= 0; va += NBPSG)
                                        setsegmap(va,
                                                  sp?sp++->sg_pmeg:seginval);
                                rp++;
                        }
                }

        } else if (CPU_HAS_SRMMU) {

#if defined(SUN4M) || defined(SUN4D)
                /*
                 * Reload page and context tables to activate the page tables
                 * for this context.
                 *
                 * The gap stuff isn't really needed in the sun4m architecture,
                 * since we don't have to worry about excessive mappings (all
                 * mappings exist since the page tables must be complete for
                 * the mmu to be happy).
                 *
                 * If a context was just allocated from the free list, trust
                 * that all of its mmu-edible page tables are zeroed out
                 * (except for those associated with the kernel). We make
                 * sure this is the case initially in pmap_bootstrap() and
                 * pmap_init() (?).
                 * Otherwise, the context was freed by calling ctx_free() in
                 * pmap_release(), which in turn is supposedly called only
                 * when all mappings have been removed.
                 *
                 * XXX: Do we have to flush cache after reloading ctx tbl?
                 */

                /*
                 * We need to flush the cache only when stealing a context
                 * from another pmap. In that case it's Ok to switch the
                 * context and leave it set, since the context table
                 * will have a valid region table entry for this context
                 * number.
                 *
                 * Otherwise, we switch to the new context after loading
                 * the context table entry with the new pmap's region.
                 */
                if (doflush) {
                        cache_flush_context(cnum);
                }

                /*
                 * The context allocated to a process is the same on all CPUs.
                 * Here we install the per-CPU region table in each CPU's
                 * context table slot.
                 *
                 * Note on multi-threaded processes: a context must remain
                 * valid as long as any thread is still running on a CPU.
                 */
                for (CPU_INFO_FOREACH(i, cpi)) {
                        setpgt4m(&cpi->ctx_tbl[cnum],
                                 (pm->pm_reg_ptps_pa[i] >> SRMMU_PPNPASHIFT) |
                                        SRMMU_TEPTD);
                }

                /* And finally switch to the new context */
                (*cpuinfo.pure_vcache_flush)();
                setcontext4m(cnum);
#endif /* SUN4M || SUN4D */
        }
}

/*
 * Give away a context.
 */
static void
ctx_free(struct pmap *pm)
{
        union ctxinfo *c;
        int ctx;
#if defined(SUN4M) || defined(SUN4D)
        struct cpu_info *cpi;
#endif

        KASSERT(mutex_owned(&ctx_lock));

        c = pm->pm_ctx;
        ctx = pm->pm_ctxnum;
        pm->pm_ctx = NULL;
        pm->pm_ctxnum = 0;
#if defined(SUN4) || defined(SUN4C)
        if (CPU_HAS_SUNMMU) {
                int octx = getcontext4();
                setcontext4(ctx);
                cache_flush_context(ctx);
                setcontext4(octx);
        }
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)
        if (CPU_HAS_SRMMU) {
                CPU_INFO_ITERATOR i;

                __USE(i);

                cache_flush_context(ctx);
                tlb_flush_context(ctx, PMAP_CPUSET(pm));
                for (CPU_INFO_FOREACH(i, cpi)) {
                        setpgt4m(&cpi->ctx_tbl[ctx], SRMMU_TEINVALID);
                }
        }
#endif

        c->c_nextfree = ctx_freelist;
        ctx_freelist = c;
}


/*----------------------------------------------------------------*/

/*
 * pvlist functions.
 */

/*
 * Walk the given pv list, and for each PTE, set or clear some bits
 * (e.g., PG_W or PG_NC).
 *
 * This routine flushes the cache for any page whose PTE changes,
 * as long as the process has a context; this is overly conservative.
 * It also copies ref and mod bits to the pvlist, on the theory that
 * this might save work later.  (XXX should test this theory)
 */

#if defined(SUN4) || defined(SUN4C)

void
pv_changepte4_4c(struct vm_page *pg, int bis, int bic)
{
        int pte, *ptep;
        struct pvlist *pv;
        struct pmap *pm;
        int va, vr, vs;
        int ctx, s;
        struct regmap *rp;
        struct segmap *sp;

        pv = VM_MDPAGE_PVHEAD(pg);

        write_user_windows();           /* paranoid? */
        s = splvm();                    /* paranoid? */
        if (pv->pv_pmap == NULL) {
                splx(s);
                return;
        }
        ctx = getcontext4();
        for (; pv != NULL; pv = pv->pv_next) {
                pm = pv->pv_pmap;
                va = pv->pv_va;
                vr = VA_VREG(va);
                vs = VA_VSEG(va);
                rp = &pm->pm_regmap[vr];
                sp = &rp->rg_segmap[vs];
                ptep = &sp->sg_pte[VA_VPG(va)];

                if (sp->sg_pmeg == seginval) {
                        /* not in hardware: just fix software copy */
                        *ptep = (*ptep | bis) & ~bic;
                } else {
                        /* in hardware: fix hardware copy */
                        if (CTX_USABLE(pm,rp)) {
                                setcontext4(pm->pm_ctxnum);
                                /* XXX should flush only when necessary */
                                pte = getpte4(va);
                                /*
                                 * XXX: always flush cache; conservative, but
                                 * needed to invalidate cache tag protection
                                 * bits and when disabling caching.
                                 */
                                cache_flush_page(va, pm->pm_ctxnum);
                        } else {
                                /* Make temp map in ctx 0 to access the PTE */
                                setcontext4(0);
                                if (HASSUN4_MMU3L)
                                        setregmap(0, tregion);
                                setsegmap(0, sp->sg_pmeg);
                                va = VA_VPG(va) << PGSHIFT;
                                pte = getpte4(va);
                        }
                        if (pte & PG_V)
                                VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(pte);
                        pte = (pte | bis) & ~bic;
                        setpte4(va, pte);
                        *ptep = (*ptep & PG_MBZ) | pte;
                }
        }
        setcontext4(ctx);
        splx(s);
}

/*
 * Sync ref and mod bits in pvlist (turns off same in hardware PTEs).
 * Returns the new flags.
 *
 * This is just like pv_changepte, but we never add or remove bits,
 * hence never need to adjust software copies.
 */
int
pv_syncflags4_4c(struct vm_page *pg)
{
        struct pvlist *pv;
        struct pmap *pm;
        int pte, va, vr, vs, pmeg, flags;
        int ctx, s;
        struct regmap *rp;
        struct segmap *sp;

        pv = VM_MDPAGE_PVHEAD(pg);

        s = splvm();                    /* paranoid? */
        if (pv->pv_pmap == NULL) {
                /* Page not mapped; pv_flags is already up to date */
                splx(s);
                return (0);
        }
        ctx = getcontext4();
        flags = pv->pv_flags;
        for (; pv != NULL; pv = pv->pv_next) {
                pm = pv->pv_pmap;
                va = pv->pv_va;
                vr = VA_VREG(va);
                vs = VA_VSEG(va);
                rp = &pm->pm_regmap[vr];
                sp = &rp->rg_segmap[vs];
                if ((pmeg = sp->sg_pmeg) == seginval)
                        continue;
                if (CTX_USABLE(pm,rp)) {
                        setcontext4(pm->pm_ctxnum);
                        /* XXX should flush only when necessary */
                        pte = getpte4(va);
                        if (pte & PG_M)
                                cache_flush_page(va, pm->pm_ctxnum);
                } else {
                        /* Make temp map in ctx 0 to access the PTE */
                        setcontext4(0);
                        if (HASSUN4_MMU3L)
                                setregmap(0, tregion);
                        setsegmap(0, pmeg);
                        va = VA_VPG(va) << PGSHIFT;
                        pte = getpte4(va);
                }
                if (pte & (PG_M|PG_U) && pte & PG_V) {
                        flags |= MR4_4C(pte);
                        pte &= ~(PG_M|PG_U);
                        setpte4(va, pte);
                }
        }

        VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
        setcontext4(ctx);
        splx(s);
        return (flags);
}

/*
 * pv_unlink is a helper function for pmap_remove.
 * It takes a pointer to the pv_table head for some physical address
 * and removes the appropriate (pmap, va) entry.
 *
 * Once the entry is removed, if the pv_table head has the cache
 * inhibit bit set, see if we can turn that off; if so, walk the
 * pvlist and turn off PG_NC in each PTE.  (The pvlist is by
 * definition nonempty, since it must have at least two elements
 * in it to have PV_NC set, and we only remove one here.)
 */
/*static*/ void
pv_unlink4_4c(struct vm_page *pg, struct pmap *pm, vaddr_t va)
{
        struct pvlist *pv0, *npv;

        pv0 = VM_MDPAGE_PVHEAD(pg);
        npv = pv0->pv_next;

        /*
         * First entry is special (sigh).
         */
        if (pv0->pv_pmap == pm && pv0->pv_va == va) {
                pmap_stats.ps_unlink_pvfirst++;
                if (npv != NULL) {
                        /*
                         * Shift next entry into the head.
                         * Make sure to retain the REF, MOD and ANC flags.
                         */
                        pv0->pv_next = npv->pv_next;
                        pv0->pv_pmap = npv->pv_pmap;
                        pv0->pv_va = npv->pv_va;
                        pv0->pv_flags &= ~PV_NC;
                        pv0->pv_flags |= (npv->pv_flags & PV_NC);
                        pool_put(&pv_pool, npv);
                } else {
                        /*
                         * No mappings left; we still need to maintain
                         * the REF and MOD flags. since pmap_is_modified()
                         * can still be called for this page.
                         */
                        pv0->pv_pmap = NULL;
                        pv0->pv_flags &= ~(PV_NC|PV_ANC);
                        return;
                }
        } else {
                struct pvlist *prev;

                pmap_stats.ps_unlink_pvsearch++;
                for (prev = pv0;; prev = npv, npv = npv->pv_next) {
                        if (npv == NULL) {
                                panic("pv_unlink: pm %p is missing on pg %p",
                                        pm, pg);
                        }
                        if (npv->pv_pmap == pm && npv->pv_va == va)
                                break;
                }
                prev->pv_next = npv->pv_next;
                pool_put(&pv_pool, npv);
        }
        if ((pv0->pv_flags & (PV_NC|PV_ANC)) == PV_ANC) {
                /*
                 * Not cached: check whether we can fix that now.
                 */
                va = pv0->pv_va;
                for (npv = pv0->pv_next; npv != NULL; npv = npv->pv_next)
                        if (BADALIAS(va, npv->pv_va) ||
                            (npv->pv_flags & PV_NC) != 0)
                                return;
                pv0->pv_flags &= ~PV_ANC;
                pv_changepte4_4c(pg, 0, PG_NC);
        }
}

/*
 * pv_link is the inverse of pv_unlink, and is used in pmap_enter.
 * It returns PG_NC if the (new) pvlist says that the address cannot
 * be cached.
 */
/*static*/ int
pv_link4_4c(struct vm_page *pg, struct pmap *pm, vaddr_t va,
            unsigned int *pteprotop)
{
        struct pvlist *pv0, *pv, *npv;
        int nc = (*pteprotop & PG_NC) != 0 ? PV_NC : 0;

        pv0 = VM_MDPAGE_PVHEAD(pg);

        if (pv0->pv_pmap == NULL) {
                /* no pvlist entries yet */
                pmap_stats.ps_enter_firstpv++;
                pv0->pv_next = NULL;
                pv0->pv_pmap = pm;
                pv0->pv_va = va;
                pv0->pv_flags |= nc;
                return (0);
        }

        /*
         * Allocate the new PV entry now, and, if that fails, bail out
         * before changing the cacheable state of the existing mappings.
         */
        npv = pool_get(&pv_pool, PR_NOWAIT);
        if (npv == NULL)
                return (ENOMEM);

        pmap_stats.ps_enter_secondpv++;

        /*
         * Before entering the new mapping, see if
         * it will cause old mappings to become aliased
         * and thus need to be `discached'.
         */
        if (pv0->pv_flags & PV_ANC) {
                /* already uncached, just stay that way */
                *pteprotop |= PG_NC;
                goto link_npv;
        }

        for (pv = pv0; pv != NULL; pv = pv->pv_next) {
                if ((pv->pv_flags & PV_NC) != 0) {
                        *pteprotop |= PG_NC;
#ifdef DEBUG
                        /* Check currently illegal condition */
                        if (nc == 0)
                                printf("pv_link: proc %s, va=0x%lx: "
                                "unexpected uncached mapping at 0x%lx\n",
                                    curproc ? curproc->p_comm : "--",
                                    va, pv->pv_va);
#endif
                }
                if (BADALIAS(va, pv->pv_va)) {
                        DPRINTF(PDB_CACHESTUFF,
                            "pv_link: badalias: proc %s, 0x%lx<=>0x%lx, pg %p",
                            curproc ? curproc->p_comm : "--",
                            va, pv->pv_va, pg);
                        /* Mark list head `uncached due to aliases' */
                        pv0->pv_flags |= PV_ANC;
                        pv_changepte4_4c(pg, PG_NC, 0);
                        *pteprotop |= PG_NC;
                        break;
                }
        }

link_npv:
        npv->pv_next = pv0->pv_next;
        npv->pv_pmap = pm;
        npv->pv_va = va;
        npv->pv_flags = nc;
        pv0->pv_next = npv;
        return (0);
}

#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)    /* SRMMU versions of above */
/*
 * Walk the given pv list, and for each PTE, set or clear some bits
 * (e.g., PG_W or PG_NC).
 *
 * This routine flushes the cache for any page whose PTE changes,
 * as long as the process has a context; this is overly conservative.
 * It also copies ref and mod bits to the pvlist, on the theory that
 * this might save work later.  (XXX should test this theory)
 *
 * Called with PV lock and pmap main lock held.
 */
void
pv_changepte4m(struct vm_page *pg, int bis, int bic)
{
        struct pvlist *pv;
        struct pmap *pm;
        vaddr_t va;
        struct regmap *rp;
        struct segmap *sp;

        pv = VM_MDPAGE_PVHEAD(pg);
        if (pv->pv_pmap == NULL)
                return;

        for (; pv != NULL; pv = pv->pv_next) {
                pm = pv->pv_pmap;
                /* XXXSMP: should lock pm */
                va = pv->pv_va;
                rp = &pm->pm_regmap[VA_VREG(va)];
                sp = &rp->rg_segmap[VA_VSEG(va)];

                if (pm->pm_ctx) {
                        /*
                         * XXX: always flush cache; conservative, but
                         * needed to invalidate cache tag protection
                         * bits and when disabling caching.
                         */
                        cache_flush_page(va, pm->pm_ctxnum);
                }

                KASSERT((sp->sg_pte[VA_SUN4M_VPG(va)] & SRMMU_TETYPE) ==
                        SRMMU_TEPTE);
                VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(updatepte4m(va,
                    &sp->sg_pte[VA_SUN4M_VPG(va)], bic, bis, pm->pm_ctxnum,
                    PMAP_CPUSET(pm)));
        }
}

/*
 * Sync ref and mod bits in pvlist. If page has been ref'd or modified,
 * update ref/mod bits in pvlist, and clear the hardware bits.
 *
 * Return the new flags.
 */
int
pv_syncflags4m(struct vm_page *pg)
{
        struct pvlist *pv;
        struct pmap *pm;
        int va, flags;
        int s;
        struct regmap *rp;
        struct segmap *sp;
        int tpte;

        s = splvm();
        PMAP_LOCK();
        pv = VM_MDPAGE_PVHEAD(pg);
        if (pv->pv_pmap == NULL) {
                /* Page not mapped; pv_flags is already up to date */
                flags = 0;
                goto out;
        }

        flags = pv->pv_flags;
        for (; pv != NULL; pv = pv->pv_next) {
                pm = pv->pv_pmap;
                va = pv->pv_va;
                rp = &pm->pm_regmap[VA_VREG(va)];
                sp = &rp->rg_segmap[VA_VSEG(va)];

                tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
                if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE &&
                    (tpte & (SRMMU_PG_R|SRMMU_PG_M)) != 0) {
                        /*
                         * Flush cache if modified to make sure the PTE
                         * M bit will be set again on the next write access.
                         */
                        if (pm->pm_ctx && (tpte & SRMMU_PG_M) == SRMMU_PG_M)
                                cache_flush_page(va, pm->pm_ctxnum);

                        flags |= MR4M(updatepte4m(va,
                                        &sp->sg_pte[VA_SUN4M_VPG(va)],
                                        SRMMU_PG_M | SRMMU_PG_R,
                                        0, pm->pm_ctxnum, PMAP_CPUSET(pm)));
                }
        }

        VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
out:
        PMAP_UNLOCK();
        splx(s);
        return (flags);
}

/*
 * Should be called with pmap already locked.
 */
void
pv_unlink4m(struct vm_page *pg, struct pmap *pm, vaddr_t va)
{
        struct pvlist *pv0, *npv;

        pv0 = VM_MDPAGE_PVHEAD(pg);

        npv = pv0->pv_next;
        /*
         * First entry is special (sigh).
         */
        if (pv0->pv_pmap == pm && pv0->pv_va == va) {
                pmap_stats.ps_unlink_pvfirst++;
                if (npv != NULL) {
                        /*
                         * Shift next entry into the head.
                         * Make sure to retain the REF, MOD and ANC flags
                         * on the list head.
                         */
                        pv0->pv_next = npv->pv_next;
                        pv0->pv_pmap = npv->pv_pmap;
                        pv0->pv_va = npv->pv_va;
                        pv0->pv_flags &= ~PV_NC;
                        pv0->pv_flags |= (npv->pv_flags & PV_NC);
                        pool_put(&pv_pool, npv);
                } else {
                        /*
                         * No mappings left; we need to maintain
                         * the REF and MOD flags, since pmap_is_modified()
                         * can still be called for this page.
                         */
                        pv0->pv_pmap = NULL;
                        pv0->pv_flags &= ~(PV_NC|PV_ANC);
                        return;
                }
        } else {
                struct pvlist *prev;

                pmap_stats.ps_unlink_pvsearch++;
                for (prev = pv0;; prev = npv, npv = npv->pv_next) {
                        if (npv == NULL) {
                                panic("pv_unlink: pm %p is missing on pg %p",
                                        pm, pg);
                                return;
                        }
                        if (npv->pv_pmap == pm && npv->pv_va == va)
                                break;
                }
                prev->pv_next = npv->pv_next;
                pool_put(&pv_pool, npv);
        }

        if ((pv0->pv_flags & (PV_NC|PV_ANC)) == PV_ANC) {

                /*
                 * Not cached: check whether we can fix that now.
                 */
                va = pv0->pv_va;
                for (npv = pv0->pv_next; npv != NULL; npv = npv->pv_next)
                        if (BADALIAS(va, npv->pv_va) ||
                            (npv->pv_flags & PV_NC) != 0)
                                return;
                DPRINTF(PDB_CACHESTUFF,
                    "pv_unlink: alias ok: proc %s, va 0x%lx, pg %p",
                    curproc ? curproc->p_comm : "--", va, pg);
                pv0->pv_flags &= ~PV_ANC;
                pv_changepte4m(pg, SRMMU_PG_C, 0);
        }
}

/*
 * pv_link is the inverse of pv_unlink, and is used in pmap_enter.
 * May turn off the cacheable bit in the pte prototype for the new mapping.
 * Called with pm locked.
 */
/*static*/ int
pv_link4m(struct vm_page *pg, struct pmap *pm, vaddr_t va,
          unsigned int *pteprotop)
{
        struct pvlist *pv0, *pv, *npv;
        int nc = (*pteprotop & SRMMU_PG_C) == 0 ? PV_NC : 0;
        int error = 0;

        pv0 = VM_MDPAGE_PVHEAD(pg);

        if (pv0->pv_pmap == NULL) {
                /* no pvlist entries yet */
                pmap_stats.ps_enter_firstpv++;
                pv0->pv_next = NULL;
                pv0->pv_pmap = pm;
                pv0->pv_va = va;
                pv0->pv_flags |= nc;
                goto out;
        }

        /*
         * Allocate the new PV entry now, and, if that fails, bail out
         * before changing the cacheable state of the existing mappings.
         */
        npv = pool_get(&pv_pool, PR_NOWAIT);
        if (npv == NULL) {
                error = ENOMEM;
                goto out;
        }

        pmap_stats.ps_enter_secondpv++;

        /*
         * See if the new mapping will cause old mappings to
         * become aliased and thus need to be `discached'.
         */
        if ((pv0->pv_flags & PV_ANC) != 0) {
                /* already uncached, just stay that way */
                *pteprotop &= ~SRMMU_PG_C;
                goto link_npv;
        }

        for (pv = pv0; pv != NULL; pv = pv->pv_next) {
                if ((pv->pv_flags & PV_NC) != 0) {
                        *pteprotop &= ~SRMMU_PG_C;
#ifdef DEBUG
                        /* Check currently illegal condition */
                        if (nc == 0)
                                printf("pv_link: proc %s, va=0x%lx: "
                                "unexpected uncached mapping at 0x%lx\n",
                                    curproc ? curproc->p_comm : "--",
                                    va, pv->pv_va);
#endif
                }
                if (BADALIAS(va, pv->pv_va)) {
                        DPRINTF(PDB_CACHESTUFF,
                            "pv_link: badalias: proc %s, 0x%lx<=>0x%lx, pg %p",
                                curproc ? curproc->p_comm : "--",
                                va, pv->pv_va, pg);
                        /* Mark list head `uncached due to aliases' */
                        pv0->pv_flags |= PV_ANC;
                        pv_changepte4m(pg, 0, SRMMU_PG_C);
                        *pteprotop &= ~SRMMU_PG_C;
                        break;
                }
        }

link_npv:
        /* Now link in the new PV entry */
        npv->pv_next = pv0->pv_next;
        npv->pv_pmap = pm;
        npv->pv_va = va;
        npv->pv_flags = nc;
        pv0->pv_next = npv;

out:
        return (error);
}
#endif

/*
 * Uncache all entries on behalf of kvm_uncache(). In addition to
 * removing the cache bit from the PTE, we are also setting PV_NC
 * in each entry to stop pv_unlink() from re-caching (i.e. when a
 * a bad alias is going away).
 */
static void
pv_uncache(struct vm_page *pg)
{
        struct pvlist *pv;
        int s;

        s = splvm();
        PMAP_LOCK();

        for (pv = VM_MDPAGE_PVHEAD(pg); pv != NULL; pv = pv->pv_next)
                pv->pv_flags |= PV_NC;

#if defined(SUN4M) || defined(SUN4D)
        if (CPU_HAS_SRMMU)
                pv_changepte4m(pg, 0, SRMMU_PG_C);
#endif
#if defined(SUN4) || defined(SUN4C)
        if (CPU_HAS_SUNMMU)
                pv_changepte4_4c(pg, PG_NC, 0);
#endif
        PMAP_UNLOCK();
        splx(s);
}

/*
 * Walk the given list and flush the cache for each (MI) page that is
 * potentially in the cache. Called only if vactype != VAC_NONE.
 */
#if defined(SUN4) || defined(SUN4C)
static void
pv_flushcache4_4c(struct vm_page *pg)
{
        struct pvlist *pv;
        struct pmap *pm;
        int s, ctx;

        pv = VM_MDPAGE_PVHEAD(pg);

        write_user_windows();   /* paranoia? */
        s = splvm();            /* XXX extreme paranoia */
        if ((pm = pv->pv_pmap) != NULL) {
                ctx = getcontext4();
                for (;;) {
                        if (pm->pm_ctx) {
                                setcontext4(pm->pm_ctxnum);
                                cache_flush_page(pv->pv_va, pm->pm_ctxnum);
                        }
                        pv = pv->pv_next;
                        if (pv == NULL)
                                break;
                        pm = pv->pv_pmap;
                }
                setcontext4(ctx);
        }
        splx(s);
}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)
static void
pv_flushcache4m(struct vm_page *pg)
{
        struct pvlist *pv;
        struct pmap *pm;
        int s;

        pv = VM_MDPAGE_PVHEAD(pg);

        s = splvm();            /* XXX extreme paranoia */
        if ((pm = pv->pv_pmap) != NULL) {
                for (;;) {
                        if (pm->pm_ctx) {
                                cache_flush_page(pv->pv_va, pm->pm_ctxnum);
                        }
                        pv = pv->pv_next;
                        if (pv == NULL)
                                break;
                        pm = pv->pv_pmap;
                }
        }
        splx(s);
}
#endif /* SUN4M || SUN4D */

/*----------------------------------------------------------------*/

/*
 * At last, pmap code.
 */

#if defined(SUN4) && (defined(SUN4C) || defined(SUN4M) || defined(SUN4D))
int nptesg;
#endif

#if defined(SUN4M) || defined(SUN4D)
static void pmap_bootstrap4m(void *);
#endif
#if defined(SUN4) || defined(SUN4C)
static void pmap_bootstrap4_4c(void *, int, int, int);
#endif

/*
 * Bootstrap the system enough to run with VM enabled.
 *
 * nsegment is the number of mmu segment entries (``PMEGs'');
 * nregion is the number of mmu region entries (``SMEGs'');
 * nctx is the number of contexts.
 */
void
pmap_bootstrap(int nctx, int nregion, int nsegment)
{
        void *p;

        uvmexp.pagesize = NBPG;
        uvm_md_init();

#if defined(SUN4) && (defined(SUN4C) || defined(SUN4M) || defined(SUN4D))
        /* In this case NPTESG is a variable */
        nptesg = (NBPSG >> pgshift);
#endif

        /*
         * Grab physical memory list.
         */
        p = kernel_top;
        get_phys_mem(&p);

        /*
         * The data segment in sparc ELF images is aligned to a 64KB
         * (the maximum page size defined by the ELF/sparc ABI) boundary.
         * This results in a unused portion of physical memory in between
         * the text/rodata and the data segment. We pick up that gap
         * here to remove it from the kernel map and give it to the
         * VM manager later.
         */
        etext_gap_start = (vaddr_t)(etext + NBPG - 1) & ~PGOFSET;
        etext_gap_end = (vaddr_t)kernel_data_start & ~PGOFSET;

        if (CPU_HAS_SRMMU) {
#if defined(SUN4M) || defined(SUN4D)
                pmap_bootstrap4m(p);
#endif
        } else if (CPU_HAS_SUNMMU) {
#if defined(SUN4) || defined(SUN4C)
                pmap_bootstrap4_4c(p, nctx, nregion, nsegment);
#endif
        }

        pmap_page_upload();
        mutex_init(&pmap_lock, MUTEX_DEFAULT, IPL_NONE);
        mutex_init(&demap_lock, MUTEX_DEFAULT, IPL_VM);
        mutex_init(&ctx_lock, MUTEX_DEFAULT, IPL_SCHED);
        lock_available = true;
}

#if defined(SUN4) || defined(SUN4C)
void
pmap_bootstrap4_4c(void *top, int nctx, int nregion, int nsegment)
{
        union ctxinfo *ci;
        struct mmuentry *mmuseg;
#if defined(SUN4_MMU3L)
        struct mmuentry *mmureg;
#endif
        struct regmap *rp;
        struct segmap *sp;
        int i, j;
        int npte, zseg, vr, vs;
        int startscookie, scookie;
#if defined(SUN4_MMU3L)
        int startrcookie = 0, rcookie = 0;
#endif
        int *kptes;
        int lastpage;
        vaddr_t va;
        vaddr_t p;

        /*
         * Compute `va2pa_offset'.
         * Use `kernel_text' to probe the MMU translation since
         * the pages at KERNBASE might not be mapped.
         */
        va2pa_offset = (vaddr_t)kernel_text -
                        ((getpte4(kernel_text) & PG_PFNUM) << PGSHIFT);

        ncontext = nctx;

        switch (cputyp) {
        case CPU_SUN4C:
                mmu_has_hole = 1;
                break;
        case CPU_SUN4:
                if (cpuinfo.cpu_type != CPUTYP_4_400) {
                        mmu_has_hole = 1;
                        break;
                }
        }

#if defined(SUN4)
        /*
         * set up the segfixmask to mask off invalid bits
         */
        segfixmask =  nsegment - 1; /* assume nsegment is a power of 2 */
#ifdef DIAGNOSTIC
        if (((nsegment & segfixmask) | (nsegment & ~segfixmask)) != nsegment) {
                printf("pmap_bootstrap: unsuitable number of segments (%d)\n",
                        nsegment);
                callrom();
        }
#endif
#endif

#if defined(SUN4M) || defined(SUN4D) /* We're in a dual-arch kernel.
                                        Setup 4/4c fn. ptrs */
        pmap_clear_modify_p     =       pmap_clear_modify4_4c;
        pmap_clear_reference_p  =       pmap_clear_reference4_4c;
        pmap_enter_p            =       pmap_enter4_4c;
        pmap_extract_p          =       pmap_extract4_4c;
        pmap_is_modified_p      =       pmap_is_modified4_4c;
        pmap_is_referenced_p    =       pmap_is_referenced4_4c;
        pmap_kenter_pa_p        =       pmap_kenter_pa4_4c;
        pmap_kremove_p          =       pmap_kremove4_4c;
        pmap_kprotect_p         =       pmap_kprotect4_4c;
        pmap_page_protect_p     =       pmap_page_protect4_4c;
        pmap_protect_p          =       pmap_protect4_4c;
        pmap_rmk_p              =       pmap_rmk4_4c;
        pmap_rmu_p              =       pmap_rmu4_4c;
#endif /* defined SUN4M || defined SUN4D */

        p = (vaddr_t)top;

        /*
         * Last segment is the `invalid' one (one PMEG of pte's with !pg_v).
         * It will never be used for anything else.
         */
        seginval = --nsegment;

#if defined(SUN4_MMU3L)
        if (HASSUN4_MMU3L)
                reginval = --nregion;
#endif

        /*
         * Allocate and initialise mmu entries and context structures.
         */
#if defined(SUN4_MMU3L)
        mmuregions = mmureg = (struct mmuentry *)p;
        p += nregion * sizeof(struct mmuentry);
        memset(mmuregions, 0, nregion * sizeof(struct mmuentry));
#endif
        mmusegments = mmuseg = (struct mmuentry *)p;
        p += nsegment * sizeof(struct mmuentry);
        memset(mmusegments, 0, nsegment * sizeof(struct mmuentry));

        pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p;
        p += nctx * sizeof *ci;

        /* Initialize MMU resource queues */
#if defined(SUN4_MMU3L)
        mmuq_init(&region_freelist);
        mmuq_init(&region_lru);
        mmuq_init(&region_locked);
#endif
        mmuq_init(&segm_freelist);
        mmuq_init(&segm_lru);
        mmuq_init(&segm_locked);


        /*
         * Initialize the kernel pmap.
         */
        /* kernel_pmap_store.pm_ctxnum = 0; */
        kernel_pmap_store.pm_refcount = 1;
#if defined(SUN4_MMU3L)
        TAILQ_INIT(&kernel_pmap_store.pm_reglist);
#endif
        TAILQ_INIT(&kernel_pmap_store.pm_seglist);

        /*
         * Allocate memory for kernel PTEs
         * XXX Consider allocating memory for only a few regions
         * and use growkernel() to allocate more as needed.
         */
        kptes = (int *)p;
        p += NKREG * NSEGRG * NPTESG * sizeof(int);
        memset(kptes, 0, NKREG * NSEGRG * NPTESG * sizeof(int));

        /*
         * Set up pm_regmap for kernel to point NUREG *below* the beginning
         * of kernel regmap storage. Since the kernel only uses regions
         * above NUREG, we save storage space and can index kernel and
         * user regions in the same way.
         */
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Warray-bounds"
        kernel_pmap_store.pm_regmap = kernel_regmap_store - NUREG;
#pragma GCC diagnostic pop
        for (i = NKREG; --i >= 0;) {
#if defined(SUN4_MMU3L)
                kernel_regmap_store[i].rg_smeg = reginval;
#endif
                kernel_regmap_store[i].rg_segmap =
                        &kernel_segmap_store[i * NSEGRG];
                for (j = NSEGRG; --j >= 0;) {
                        sp = &kernel_segmap_store[i * NSEGRG + j];
                        sp->sg_pmeg = seginval;
                        sp->sg_pte = &kptes[(i * NSEGRG + j) * NPTESG];
                }
        }

        /*
         * Preserve the monitor ROM's reserved VM region, so that
         * we can use L1-A or the monitor's debugger.  As a side
         * effect we map the ROM's reserved VM into all contexts
         * (otherwise L1-A crashes the machine!).
         */

        mmu_reservemon4_4c(&nregion, &nsegment);

#if defined(SUN4_MMU3L)
        /* Reserve one region for temporary mappings */
        if (HASSUN4_MMU3L)
                tregion = --nregion;
#endif

        /*
         * Set up the `constants' for the call to vm_init()
         * in main().  All pages beginning at p (rounded up to
         * the next whole page) and continuing through the number
         * of available pages are free, but they start at a higher
         * virtual address.  This gives us two mappable MD pages
         * for pmap_zero_page and pmap_copy_page, and one MI page
         * for /dev/mem, all with no associated physical memory.
         */
        p = (p + NBPG - 1) & ~PGOFSET;

        avail_start = PMAP_BOOTSTRAP_VA2PA(p);

        i = p;
        cpuinfo.vpage[0] = (void *)p, p += NBPG;
        cpuinfo.vpage[1] = (void *)p, p += NBPG;
        p = (vaddr_t)reserve_dumppages((void *)p);

        virtual_avail = p;
        virtual_end = VM_MAX_KERNEL_ADDRESS;

        p = i;                  /* retract to first free phys */


        /*
         * All contexts are free except the kernel's.
         *
         * XXX sun4c could use context 0 for users?
         */
        ci->c_pmap = pmap_kernel();
        ctx_freelist = ci + 1;
        for (i = 1; i < ncontext; i++) {
                ci++;
                ci->c_nextfree = ci + 1;
        }
        ci->c_nextfree = NULL;
        ctx_kick = 0;
        ctx_kickdir = -1;

        /*
         * Init mmu entries that map the kernel physical addresses.
         *
         * All the other MMU entries are free.
         *
         * THIS ASSUMES THE KERNEL IS MAPPED BY A CONTIGUOUS RANGE OF
         * MMU SEGMENTS/REGIONS DURING THE BOOT PROCESS
         */

        /* Compute the number of segments used by the kernel */
        zseg = (((p + NBPSG - 1) & ~SGOFSET) - KERNBASE) >> SGSHIFT;
        lastpage = VA_VPG(p);
        if (lastpage == 0)
                /*
                 * If the page bits in p are 0, we filled the last segment
                 * exactly; if not, it is the last page filled in the
                 * last segment.
                 */
                lastpage = NPTESG;

        p = KERNBASE;                   /* first va */
        vs = VA_VSEG(KERNBASE);         /* first virtual segment */
        vr = VA_VREG(KERNBASE);         /* first virtual region */
        rp = &pmap_kernel()->pm_regmap[vr];

        /* Get region/segment where kernel addresses start */
#if defined(SUN4_MMU3L)
        if (HASSUN4_MMU3L)
                startrcookie = rcookie = getregmap(p);
        mmureg = &mmuregions[rcookie];
#endif

        startscookie = scookie = getsegmap(p);
        mmuseg = &mmusegments[scookie];
        zseg += scookie;        /* First free segment */

        for (;;) {

                /*
                 * Distribute each kernel region/segment into all contexts.
                 * This is done through the monitor ROM, rather than
                 * directly here: if we do a setcontext we will fault,
                 * as we are not (yet) mapped in any other context.
                 */

                if ((vs % NSEGRG) == 0) {
                        /* Entering a new region */
                        if (VA_VREG(p) > vr) {
#ifdef DEBUG
                                printf("note: giant kernel!\n");
#endif
                                vr++, rp++;
                        }
#if defined(SUN4_MMU3L)
                        if (HASSUN4_MMU3L) {
                                for (i = 1; i < nctx; i++)
                                        prom_setcontext(i, (void *)p, rcookie);

                                mmuq_insert_tail(&region_locked,
                                                  mmureg);
                                TAILQ_INSERT_TAIL(&pmap_kernel()->pm_reglist,
                                                  mmureg, me_pmchain);
#ifdef DIAGNOSTIC
                                mmuseg->me_statp = NULL;
#endif
                                mmureg->me_cookie = rcookie;
                                mmureg->me_pmap = pmap_kernel();
                                mmureg->me_vreg = vr;
                                rp->rg_smeg = rcookie;
                                mmureg++;
                                rcookie++;
                        }
#endif /* SUN4_MMU3L */
                }

#if defined(SUN4_MMU3L)
                if (!HASSUN4_MMU3L)
#endif
                        for (i = 1; i < nctx; i++)
                                prom_setcontext(i, (void *)p, scookie);

                /* set up the mmu entry */
                mmuq_insert_tail(&segm_locked, mmuseg);
#ifdef DIAGNOSTIC
                mmuseg->me_statp = &pmap_stats.ps_npmeg_locked;
#endif
                TAILQ_INSERT_TAIL(&pmap_kernel()->pm_seglist, mmuseg, me_pmchain);
                pmap_stats.ps_npmeg_locked++;
                mmuseg->me_cookie = scookie;
                mmuseg->me_pmap = pmap_kernel();
                mmuseg->me_vreg = vr;
                mmuseg->me_vseg = vs % NSEGRG;
                sp = &rp->rg_segmap[vs % NSEGRG];
                sp->sg_pmeg = scookie;
                npte = ++scookie < zseg ? NPTESG : lastpage;
                sp->sg_npte = npte;
                sp->sg_nwired = npte;
                pmap_kernel()->pm_stats.resident_count += npte;
                rp->rg_nsegmap += 1;
                for (i = 0; i < npte; i++)
                        sp->sg_pte[i] = getpte4(p + i * NBPG) | PG_WIRED;
                mmuseg++;
                vs++;
                if (scookie < zseg) {
                        p += NBPSG;
                        continue;
                }

                /*
                 * Unmap the pages, if any, that are not part of
                 * the final segment.
                 */
                for (p += npte << PGSHIFT; npte < NPTESG; npte++, p += NBPG)
                        setpte4(p, 0);

#if defined(SUN4_MMU3L)
                if (HASSUN4_MMU3L) {
                        /*
                         * Unmap the segments, if any, that are not part of
                         * the final region.
                         */
                        for (i = rp->rg_nsegmap; i < NSEGRG; i++, p += NBPSG)
                                setsegmap(p, seginval);

                        /*
                         * Unmap any kernel regions that we aren't using.
                         */
                        for (i = 0; i < nctx; i++) {
                                setcontext4(i);
                                for (va = p;
                                     va < (OPENPROM_STARTVADDR & ~(NBPRG - 1));
                                     va += NBPRG)
                                        setregmap(va, reginval);
                        }

                } else
#endif
                {
                        /*
                         * Unmap any kernel segments that we aren't using.
                         */
                        for (i = 0; i < nctx; i++) {
                                setcontext4(i);
                                for (va = p;
                                     va < (OPENPROM_STARTVADDR & ~(NBPSG - 1));
                                     va += NBPSG)
                                        setsegmap(va, seginval);
                        }
                }
                break;
        }

#if defined(SUN4_MMU3L)
        if (HASSUN4_MMU3L)
                for (rcookie = 0; rcookie < nregion; rcookie++) {
                        if (rcookie == startrcookie)
                                /* Kernel must fit in one region! */
                                rcookie++;
                        mmureg = &mmuregions[rcookie];
                        mmureg->me_cookie = rcookie;
                        mmuq_insert_tail(&region_freelist, mmureg);
#ifdef DIAGNOSTIC
                        mmuseg->me_statp = NULL;
#endif
                }
#endif /* SUN4_MMU3L */

        for (scookie = 0; scookie < nsegment; scookie++) {
                if (scookie == startscookie)
                        /* Skip static kernel image */
                        scookie = zseg;
                mmuseg = &mmusegments[scookie];
                mmuseg->me_cookie = scookie;
                mmuq_insert_tail(&segm_freelist, mmuseg);
                pmap_stats.ps_npmeg_free++;
#ifdef DIAGNOSTIC
                mmuseg->me_statp = NULL;
#endif
        }

        /* Erase all spurious user-space segmaps */
        for (i = 1; i < ncontext; i++) {
                setcontext4(i);
                if (HASSUN4_MMU3L)
                        for (p = 0, j = NUREG; --j >= 0; p += NBPRG)
                                setregmap(p, reginval);
                else
                        for (p = 0, vr = 0; vr < NUREG; vr++) {
                                if (VA_INHOLE(p)) {
                                        p = MMU_HOLE_END;
                                        vr = VA_VREG(p);
                                }
                                for (j = NSEGRG; --j >= 0; p += NBPSG)
                                        setsegmap(p, seginval);
                        }
        }
        setcontext4(0);

        /*
         * write protect & encache kernel text;
         * set red zone at kernel base;
         * enable cache on message buffer and cpuinfo.
         */

        /* Enable cache on message buffer and cpuinfo */
        for (p = KERNBASE; p < (vaddr_t)trapbase; p += NBPG)
                setpte4(p, getpte4(p) & ~PG_NC);

        /* Enable cache and write protect kernel text */
        for (p = (vaddr_t)trapbase; p < (vaddr_t)etext; p += NBPG)
                setpte4(p, getpte4(p) & ~(PG_NC|PG_W));

        /*
         * Unmap the `etext gap'; it'll be made available
         * to the VM manager.
         */
        for (p = etext_gap_start; p < etext_gap_end; p += NBPG) {
                rp = &pmap_kernel()->pm_regmap[VA_VREG(p)];
                sp = &rp->rg_segmap[VA_VSEG(p)];
                sp->sg_nwired--;
                sp->sg_npte--;
                pmap_kernel()->pm_stats.resident_count--;
                sp->sg_pte[VA_VPG(p)] = 0;
                setpte4(p, 0);
        }

        /* Enable cache on data & bss */
        for (p = etext_gap_end; p < virtual_avail; p += NBPG)
                setpte4(p, getpte4(p) & ~PG_NC);

        cpus[0] = (struct cpu_info *)CPUINFO_VA;
}
#endif

#if defined(SUN4M) || defined(SUN4D)    /* SRMMU version of pmap_bootstrap */
/*
 * Bootstrap the system enough to run with VM enabled on a sun4m machine.
 *
 * Switches from ROM to kernel page tables, and sets up initial mappings.
 */
static void
pmap_bootstrap4m(void *top)
{
        int i, j;
        vaddr_t p, q;
        union ctxinfo *ci;
        int reg, seg;
        unsigned int ctxtblsize;
        vaddr_t pagetables_start, pagetables_end;
        paddr_t pagetables_start_pa;
        vaddr_t va;
#if defined(MULTIPROCESSOR)
        vsize_t off;
        size_t cpuinfo_len = sizeof(struct cpu_info);
        uint8_t *cpuinfo_data;
        int align = PAGE_SIZE;
        vaddr_t sva, cpuinfo_va;
        vsize_t sz;
#endif

        /*
         * Compute `va2pa_offset'.
         * Use `kernel_text' to probe the MMU translation since
         * the pages at KERNBASE might not be mapped.
         */
        va2pa_offset = (vaddr_t)kernel_text - VA2PA(kernel_text);

        ncontext = cpuinfo.mmu_ncontext;

#if defined(SUN4) || defined(SUN4C) /* setup SRMMU fn. ptrs for dual-arch
                                       kernel */
        pmap_clear_modify_p     =       pmap_clear_modify4m;
        pmap_clear_reference_p  =       pmap_clear_reference4m;
        pmap_enter_p            =       pmap_enter4m;
        pmap_extract_p          =       pmap_extract4m;
        pmap_is_modified_p      =       pmap_is_modified4m;
        pmap_is_referenced_p    =       pmap_is_referenced4m;
        pmap_kenter_pa_p        =       pmap_kenter_pa4m;
        pmap_kremove_p          =       pmap_kremove4m;
        pmap_kprotect_p         =       pmap_kprotect4m;
        pmap_page_protect_p     =       pmap_page_protect4m;
        pmap_protect_p          =       pmap_protect4m;
        pmap_rmk_p              =       pmap_rmk4m;
        pmap_rmu_p              =       pmap_rmu4m;
#endif /* defined SUN4/SUN4C */

        /*
         * p points to top of kernel mem
         */
        p = (vaddr_t)top;

        p = (p + NBPG - 1) & ~PGOFSET;
        DPRINTF(PDB_INITLOUD, "initial p=%lx", p);

        /*
         * Initialize the kernel pmap.
         */
        /* kernel_pmap_store.pm_ctxnum = 0; */
        kernel_pmap_store.pm_refcount = 1;

        /*
         * Set up pm_regmap for kernel to point NUREG *below* the beginning
         * of kernel regmap storage. Since the kernel only uses regions
         * above NUREG, we save storage space and can index kernel and
         * user regions in the same way.
         */
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Warray-bounds"
        kernel_pmap_store.pm_regmap = kernel_regmap_store - NUREG;
#pragma GCC diagnostic pop
        memset(kernel_regmap_store, 0, sizeof kernel_regmap_store);
        memset(kernel_segmap_store, 0, sizeof kernel_segmap_store);
        for (i = NKREG; --i >= 0;) {
                kernel_regmap_store[i].rg_segmap =
                        &kernel_segmap_store[i * NSEGRG];
                kernel_regmap_store[i].rg_seg_ptps = NULL;
                for (j = NSEGRG; --j >= 0;)
                        kernel_segmap_store[i * NSEGRG + j].sg_pte = NULL;
        }

        /* Allocate kernel region pointer tables */
        pmap_kernel()->pm_reg_ptps = (int **)(q = p);
        DPRINTF(PDB_INITLOUD, "kernel region pointer tables p=%lx", p);
        p += sparc_ncpus * sizeof(int **);
        memset((void *)q, 0, (u_int)p - (u_int)q);

        pmap_kernel()->pm_reg_ptps_pa = (int *)(q = p);
        DPRINTF(PDB_INITLOUD, "kernel region pointer tables pa p=%lx", p);
        p += sparc_ncpus * sizeof(int *);
        memset((void *)q, 0, (u_int)p - (u_int)q);

        /* Allocate context administration */
        pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p;
        DPRINTF(PDB_INITLOUD, "context administration p=%lx", p);
        p += ncontext * sizeof *ci;
        memset((void *)ci, 0, (u_int)p - (u_int)ci);

        /*
         * Set up the `constants' for the call to vm_init()
         * in main().  All pages beginning at p (rounded up to
         * the next whole page) and continuing through the number
         * of available pages are free.
         */
        p = (p + NBPG - 1) & ~PGOFSET;
        DPRINTF(PDB_INITLOUD, "align p=%lx", p);

        /*
         * Reserve memory for MMU pagetables. Some of these have severe
         * alignment restrictions. We allocate in a sequence that
         * minimizes alignment gaps.
         */

        pagetables_start = p;
        pagetables_start_pa = PMAP_BOOTSTRAP_VA2PA(p);

        /*
         * Allocate context table.
         * To keep supersparc happy, minimum alignment is on a 4K boundary.
         */
        ctxtblsize = uimax(ncontext,1024) * sizeof(int);
        cpuinfo.ctx_tbl = (int *)roundup((u_int)p, ctxtblsize);
        cpuinfo.ctx_tbl_pa = PMAP_BOOTSTRAP_VA2PA(cpuinfo.ctx_tbl);
        p = (u_int)cpuinfo.ctx_tbl + ctxtblsize;
        DPRINTF(PDB_INITLOUD, "post ctx table p=%lx", p);

#if defined(MULTIPROCESSOR)
        /*
         * Make sure all smp_tlb_flush*() routines for kernel pmap are
         * broadcast to all CPU's.
         */
        pmap_kernel()->pm_cpuset = CPUSET_ALL;
#endif

        /*
         * Reserve memory for segment and page tables needed to map the entire
         * kernel. This takes (2K + NKREG * 16K) of space, but unfortunately
         * is necessary since pmap_enter() *must* be able to enter a kernel
         * mapping without delay.
         */
        p = (vaddr_t) roundup(p, SRMMU_L1SIZE * sizeof(u_int));
        DPRINTF(PDB_INITLOUD, "roundup kernel_regtable_store p=%lx", p);
        qzero((void *)p, SRMMU_L1SIZE * sizeof(u_int));
        kernel_regtable_store = (u_int *)p;
        p += SRMMU_L1SIZE * sizeof(u_int);
        DPRINTF(PDB_INITLOUD, "L1 pages p=%lx", p);

        p = (vaddr_t) roundup(p, SRMMU_L2SIZE * sizeof(u_int));
        DPRINTF(PDB_INITLOUD, "roundup kernel_segtable_store p=%lx", p);
        qzero((void *)p, (SRMMU_L2SIZE * sizeof(u_int)) * NKREG);
        kernel_segtable_store = (u_int *)p;
        p += (SRMMU_L2SIZE * sizeof(u_int)) * NKREG;
        DPRINTF(PDB_INITLOUD, "L2 pages p=%lx", p);

        p = (vaddr_t) roundup(p, SRMMU_L3SIZE * sizeof(u_int));
        DPRINTF(PDB_INITLOUD, "roundup kernel_pagtable_store p=%lx", p);
        /* zero it: all will be SRMMU_TEINVALID */
        qzero((void *)p, ((SRMMU_L3SIZE * sizeof(u_int)) * NSEGRG) * NKREG);
        kernel_pagtable_store = (u_int *)p;
        p += ((SRMMU_L3SIZE * sizeof(u_int)) * NSEGRG) * NKREG;
        DPRINTF(PDB_INITLOUD, "L3 pages p=%lx", p);

        /* Round to next page and mark end of pre-wired kernel space */
        p = (p + NBPG - 1) & ~PGOFSET;
        DPRINTF(PDB_INITLOUD, "align p=%lx", p);
        pagetables_end = p;

#if defined(MULTIPROCESSOR)
        /*
         * Allocate aligned KVA.  `cpuinfo' resides at a fixed virtual
         * address. Since we need to access an other CPU's cpuinfo
         * structure occasionally, this must be done at a virtual address
         * that's cache congruent to the fixed address CPUINFO_VA.
         *
         * NOTE: we're using the cache properties of the boot CPU to
         * determine the alignment (XXX).
         */
        sz = sizeof(struct cpu_info);
        if (sparc_ncpus > 1) {
                if (CACHEINFO.c_totalsize > align) {
                        /* Need a power of two */
                        while (align <= CACHEINFO.c_totalsize)
                                align <<= 1;
                        align >>= 1;
                }

                sz = (sz + PAGE_SIZE - 1) & -PAGE_SIZE;
                cpuinfo_len = sz + align - PAGE_SIZE;

                /* Grab as much space as we need */
                DPRINTF(PDB_INITLOUD, "cpuinfo=%lx", p);
                cpuinfo_data = (uint8_t *)p;
                p += (cpuinfo_len * sparc_ncpus);
        } else
                cpuinfo_data = (uint8_t *)CPUINFO_VA;
#endif

        DPRINTF(PDB_INITLOUD, "avail_start=%lx", p);
        avail_start = PMAP_BOOTSTRAP_VA2PA(p);

        /*
         * Now wire the region and segment tables of the kernel map.
         */
        pmap_kernel()->pm_reg_ptps[0] = (int *) kernel_regtable_store;
        pmap_kernel()->pm_reg_ptps_pa[0] =
                 PMAP_BOOTSTRAP_VA2PA(kernel_regtable_store);

        /* Install L1 table in context 0 */
        setpgt4m(&cpuinfo.ctx_tbl[0],
            (pmap_kernel()->pm_reg_ptps_pa[0] >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);

        for (reg = 0; reg < NKREG; reg++) {
                struct regmap *rp;
                void *kphyssegtbl;

                /*
                 * Entering new region; install & build segtbl
                 */

                rp = &pmap_kernel()->pm_regmap[reg + VA_VREG(KERNBASE)];

                kphyssegtbl = (void *)
                    &kernel_segtable_store[reg * SRMMU_L2SIZE];

                setpgt4m(&pmap_kernel()->pm_reg_ptps[0][reg + VA_VREG(KERNBASE)],
                    (PMAP_BOOTSTRAP_VA2PA(kphyssegtbl) >> SRMMU_PPNPASHIFT) |
                    SRMMU_TEPTD);

                rp->rg_seg_ptps = (int *)kphyssegtbl;

                for (seg = 0; seg < NSEGRG; seg++) {
                        struct segmap *sp;
                        void *kphyspagtbl;

                        rp->rg_nsegmap++;

                        sp = &rp->rg_segmap[seg];
                        kphyspagtbl = (void *)
                            &kernel_pagtable_store
                                [((reg * NSEGRG) + seg) * SRMMU_L3SIZE];

                        setpgt4m(&rp->rg_seg_ptps[seg],
                                 (PMAP_BOOTSTRAP_VA2PA(kphyspagtbl) >> SRMMU_PPNPASHIFT) |
                                 SRMMU_TEPTD);
                        sp->sg_pte = (int *) kphyspagtbl;
                }
        }

        /*
         * Preserve the monitor ROM's reserved VM region, so that
         * we can use L1-A or the monitor's debugger.
         */
        mmu_reservemon4m(&kernel_pmap_store);

        /*
         * Reserve virtual address space for two mappable MD pages
         * for pmap_zero_page and pmap_copy_page, one MI page
         * for /dev/mem, and some more for dumpsys().
         */
        q = p;
        cpuinfo.vpage[0] = (void *)p, p += NBPG;
        cpuinfo.vpage[1] = (void *)p, p += NBPG;
        p = (vaddr_t)reserve_dumppages((void *)p);

        /* Find PTE locations of vpage[] to optimize zero_fill() et.al. */
        for (i = 0; i < 2; i++) {
                struct regmap *rp;
                struct segmap *sp;
                rp = &pmap_kernel()->pm_regmap[VA_VREG(cpuinfo.vpage[i])];
                sp = &rp->rg_segmap[VA_VSEG(cpuinfo.vpage[i])];
                cpuinfo.vpage_pte[i] =
                        &sp->sg_pte[VA_SUN4M_VPG(cpuinfo.vpage[i])];
        }

#if !(defined(PROM_AT_F0) || defined(MSIIEP))
        virtual_avail = p;
#elif defined(MSIIEP)
        virtual_avail = (vaddr_t)0xf0800000; /* Krups */
#else
        virtual_avail = (vaddr_t)0xf0080000; /* Mr.Coffee/OFW */
#endif
        virtual_end = VM_MAX_KERNEL_ADDRESS;

        p = q;                  /* retract to first free phys */

        /*
         * Set up the ctxinfo structures (freelist of contexts)
         */
        ci->c_pmap = pmap_kernel();
        ctx_freelist = ci + 1;
        for (i = 1; i < ncontext; i++) {
                ci++;
                ci->c_nextfree = ci + 1;
        }
        ci->c_nextfree = NULL;
        ctx_kick = 0;
        ctx_kickdir = -1;

        /*
         * Now map the kernel into our new set of page tables, then
         * (finally) switch over to our running page tables.
         * We map from KERNBASE to p into context 0's page tables (and
         * the kernel pmap).
         */
#ifdef DEBUG                    /* Sanity checks */
        if (p % NBPG != 0)
                panic("pmap_bootstrap4m: p misaligned?!?");
        if (KERNBASE % NBPRG != 0)
                panic("pmap_bootstrap4m: KERNBASE not region-aligned");
#endif

        for (q = KERNBASE; q < p; q += NBPG) {
                struct regmap *rp;
                struct segmap *sp;
                int pte, *ptep;

                /*
                 * Now install entry for current page.
                 */
                rp = &pmap_kernel()->pm_regmap[VA_VREG(q)];
                sp = &rp->rg_segmap[VA_VSEG(q)];
                ptep = &sp->sg_pte[VA_VPG(q)];

                /*
                 * Unmap the `etext gap'; it'll be made available
                 * to the VM manager.
                 */
                if (q >= etext_gap_start && q < etext_gap_end) {
                        setpgt4m(ptep, 0);
                        continue;
                }

                pte = PMAP_BOOTSTRAP_VA2PA(q) >> SRMMU_PPNPASHIFT;
                pte |= PPROT_N_RX | SRMMU_TEPTE;

                /* Deal with the cacheable bit for pagetable memory */
                if ((CACHEINFO.c_flags & CACHE_PAGETABLES) != 0 ||
                    q < pagetables_start || q >= pagetables_end)
                        pte |= SRMMU_PG_C;

                /* write-protect kernel text */
                if (q < (vaddr_t)trapbase || q >= (vaddr_t)etext)
                        pte |= PPROT_WRITE;

                setpgt4m(ptep, pte);
                pmap_kernel()->pm_stats.resident_count++;
        }

        if ((CACHEINFO.c_flags & CACHE_PAGETABLES) == 0) {
                /*
                 * The page tables have been setup. Since we're still
                 * running on the PROM's memory map, the memory we
                 * allocated for our page tables might still be cached.
                 * Flush it now, and don't touch it again until we
                 * switch to our own tables (will be done immediately below).
                 */
                int size = pagetables_end - pagetables_start;
                if (CACHEINFO.c_vactype != VAC_NONE) {
                        va = (vaddr_t)pagetables_start;
                        while (size > 0) {
                                cache_flush_page(va, 0);
                                va += NBPG;
                                size -= NBPG;
                        }
                } else if (cpuinfo.pcache_flush_page != NULL) {
                        paddr_t pa = pagetables_start_pa;
                        while (size > 0) {
                                pcache_flush_page(pa, 0);
                                pa += NBPG;
                                size -= NBPG;
                        }
                }
        }

        /*
         * Now switch to kernel pagetables (finally!)
         */
        mmu_install_tables(&cpuinfo);

#if defined(MULTIPROCESSOR)
        /*
         * Initialise any cpu-specific data now.
         */
        cpu_init_system();

        /*
         * Setup the cpus[] array and the ci_self links.
         */
        for (i = 0; i < sparc_ncpus; i++) {
                sva = (vaddr_t) (cpuinfo_data + (cpuinfo_len * i));
                cpuinfo_va = sva +
                   (((CPUINFO_VA & (align - 1)) + align - sva) & (align - 1));

                /*
                 * Either remap from CPUINFO_VA to the new correct value
                 * or clear out this cpuinfo.
                 */
                if (i == 0) {
                        for (off = 0, va = cpuinfo_va;
                             sparc_ncpus > 1 && off < sizeof(struct cpu_info);
                             va += NBPG, off += NBPG) {
                                paddr_t pa =
                                    PMAP_BOOTSTRAP_VA2PA(CPUINFO_VA + off);

                                pmap_kremove(va, NBPG);
                                pmap_kenter_pa(va, pa,
                                               VM_PROT_READ | VM_PROT_WRITE, 0);
                        }

                } else
                        memset((void *)cpuinfo_va, 0, sizeof(struct cpu_info));

                cpus[i] = (struct cpu_info *)cpuinfo_va;
                cpus[i]->ci_self = cpus[i];

                /* Unmap and prepare to return unused pages */
                if (cpuinfo_va != sva) {
                        cpus[i]->ci_free_sva1 = sva;
                        cpus[i]->ci_free_eva1 = cpuinfo_va;
                        for (va = cpus[i]->ci_free_sva1;
                             va < cpus[i]->ci_free_eva1;
                             va += NBPG)
                                setpte4m(va, 0);
                }
                if (cpuinfo_va + sz != sva + cpuinfo_len) {
                        cpus[i]->ci_free_sva2 = cpuinfo_va + sz;
                        cpus[i]->ci_free_eva2 = sva + cpuinfo_len;
                        for (va = cpus[i]->ci_free_sva2;
                             va < cpus[i]->ci_free_eva2;
                             va += NBPG)
                                setpte4m(va, 0);
                }
        }
#else
        cpus[0] = (struct cpu_info *)CPUINFO_VA;
#endif

        pmap_update(pmap_kernel());

#ifdef DIAGNOSTIC
        if (curcpu()->ci_self != cpus[0]) {
                prom_printf("curcpu()->ci_self %p != cpus[0] %p\n", curcpu()->ci_self, cpus[0]);
                panic("cpuinfo inconsistent");
        }
#endif
}

static u_long prom_ctxreg;

void
mmu_install_tables(struct cpu_info *sc)
{

#ifdef DEBUG
        prom_printf("pmap_bootstrap: installing kernel page tables...");
#endif
        setcontext4m(0);        /* paranoia? %%%: Make 0x3 a define! below */

        /* Enable MMU tablewalk caching, flush TLB */
        if (sc->mmu_enable != 0)
                sc->mmu_enable();

        tlb_flush_all_real();
        prom_ctxreg = lda(SRMMU_CXTPTR, ASI_SRMMU);

        sta(SRMMU_CXTPTR, ASI_SRMMU,
                (sc->ctx_tbl_pa >> SRMMU_PPNPASHIFT) & ~0x3);

        tlb_flush_all_real();

#ifdef DEBUG
        prom_printf("done.\n");
#endif
}

void srmmu_restore_prom_ctx(void);

void
srmmu_restore_prom_ctx(void)
{

        tlb_flush_all();
        sta(SRMMU_CXTPTR, ASI_SRMMU, prom_ctxreg);
        tlb_flush_all();
}
#endif /* SUN4M || SUN4D */

#if defined(MULTIPROCESSOR)
/*
 * Allocate per-CPU page tables. One region, segment and page table
 * is needed to map CPUINFO_VA to different physical addresses on
 * each CPU. Since the kernel region and segment tables are all
 * pre-wired (in bootstrap() above) and we also assume that the
 * first segment (256K) of kernel space is fully populated with
 * pages from the start, these per-CPU tables will never need
 * to be updated when mapping kernel virtual memory.
 *
 * Note: this routine is called in the context of the boot CPU
 * during autoconfig.
 */
void
pmap_alloc_cpu(struct cpu_info *sc)
{
#if defined(SUN4M) || defined(SUN4D)    /* Only implemented for SUN4M/D */
        vaddr_t va;
        paddr_t pa;
        paddr_t alignment;
        u_int *ctxtable, *regtable, *segtable, *pagtable;
        u_int *ctxtable_pa, *regtable_pa, *segtable_pa, *pagtable_pa;
        psize_t ctxsize, size;
        int vr, vs, vpg;
        struct regmap *rp;
        struct segmap *sp;
        struct pglist mlist;
        int cachebit;
        int pagesz = NBPG;
        int i;

        cachebit = (CACHEINFO.c_flags & CACHE_PAGETABLES) != 0;

        /*
         * Allocate properly aligned and contiguous physically memory
         * for the PTE tables.
         */
        ctxsize = (sc->mmu_ncontext * sizeof(int) + pagesz - 1) & -pagesz;
        alignment = ctxsize;

        /* The region, segment and page table we need fit in one page */
        size = ctxsize + pagesz;

        if (uvm_pglistalloc(size, vm_first_phys, vm_first_phys+vm_num_phys,
                            alignment, 0, &mlist, 1, 0) != 0)
                panic("pmap_alloc_cpu: no memory");

        pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&mlist));

        /* Allocate virtual memory */
        va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY);
        if (va == 0)
                panic("pmap_alloc_cpu: no memory");

        /*
         * Layout the page tables in our chunk of memory
         */
        ctxtable = (u_int *)va;
        regtable = (u_int *)(va + ctxsize);
        segtable = regtable + SRMMU_L1SIZE;
        pagtable = segtable + SRMMU_L2SIZE;

        ctxtable_pa = (u_int *)pa;
        regtable_pa = (u_int *)(pa + ctxsize);
        segtable_pa = regtable_pa + SRMMU_L1SIZE;
        pagtable_pa = segtable_pa + SRMMU_L2SIZE;

        /* Map the pages */
        while (size != 0) {
                pmap_kenter_pa(va, pa | (cachebit ? 0 : PMAP_NC),
                    VM_PROT_READ | VM_PROT_WRITE, 0);
                va += pagesz;
                pa += pagesz;
                size -= pagesz;
        }
        pmap_update(pmap_kernel());

        /*
         * Store the region table pointer (and its corresponding physical
         * address) in the CPU's slot in the kernel pmap region table
         * pointer table.
         */
        pmap_kernel()->pm_reg_ptps[sc->ci_cpuid] = regtable;
        pmap_kernel()->pm_reg_ptps_pa[sc->ci_cpuid] = (paddr_t)regtable_pa;

        vr = VA_VREG(CPUINFO_VA);
        vs = VA_VSEG(CPUINFO_VA);
        vpg = VA_VPG(CPUINFO_VA);
        rp = &pmap_kernel()->pm_regmap[vr];
        sp = &rp->rg_segmap[vs];

        /*
         * Copy page tables from CPU #0, then modify entry for CPUINFO_VA
         * so that it points at the per-CPU pages.
         */
        qcopy(pmap_kernel()->pm_reg_ptps[0], regtable,
                SRMMU_L1SIZE * sizeof(int));
        qcopy(rp->rg_seg_ptps, segtable, SRMMU_L2SIZE * sizeof(int));
        qcopy(sp->sg_pte, pagtable, SRMMU_L3SIZE * sizeof(int));

        setpgt4m(&ctxtable[0],
                 ((u_long)regtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
        setpgt4m(&regtable[vr],
                 ((u_long)segtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
        setpgt4m(&segtable[vs],
                 ((u_long)pagtable_pa >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
        setpgt4m(&pagtable[vpg],
                (VA2PA((void *)sc) >> SRMMU_PPNPASHIFT) |
                (SRMMU_TEPTE | PPROT_N_RWX | SRMMU_PG_C));

        /* Install this CPU's context table */
        sc->ctx_tbl = ctxtable;
        sc->ctx_tbl_pa = (paddr_t)ctxtable_pa;

        /* Pre-compute this CPU's vpage[] PTEs */
        for (i = 0; i < 2; i++) {
                rp = &pmap_kernel()->pm_regmap[VA_VREG(sc->vpage[i])];
                sp = &rp->rg_segmap[VA_VSEG(sc->vpage[i])];
                sc->vpage_pte[i] = &sp->sg_pte[VA_SUN4M_VPG(sc->vpage[i])];
        }
#endif /* SUN4M || SUN4D */
}
#endif /* MULTIPROCESSOR */


void
pmap_init(void)
{
        u_int sz;

        if (PAGE_SIZE != NBPG)
                panic("pmap_init: PAGE_SIZE!=NBPG");

        vm_num_phys = vm_last_phys - vm_first_phys;

        /* Setup a pool for additional pvlist structures */
        pool_init(&pv_pool, sizeof(struct pvlist), 0, 0, 0, "pvtable", NULL,
            IPL_NONE);

        /*
         * Setup a pool for pmap structures.
         * The pool size includes space for an array of per-CPU
         * region table pointers & physical addresses
         */
        sz = ALIGN(sizeof(struct pmap)) +
             ALIGN(NUREG * sizeof(struct regmap)) +
             sparc_ncpus * sizeof(int *) +      /* pm_reg_ptps */
             sparc_ncpus * sizeof(int);         /* pm_reg_ptps_pa */
        pool_cache_bootstrap(&pmap_cache, sz, 0, 0, 0, "pmappl", NULL,
            IPL_NONE, pmap_pmap_pool_ctor, pmap_pmap_pool_dtor, NULL);

        sz = NSEGRG * sizeof (struct segmap);
        pool_init(&segmap_pool, sz, 0, 0, 0, "segmap", NULL, IPL_NONE);

#if defined(SUN4M) || defined(SUN4D)
        if (CPU_HAS_SRMMU) {
                /*
                 * The SRMMU only ever needs chunks in one of two sizes:
                 * 1024 (for region level tables) and 256 (for segment
                 * and page level tables).
                 */
                sz = SRMMU_L1SIZE * sizeof(int);
                pool_init(&L1_pool, sz, sz, 0, 0, "L1 pagetable",
                          &pgt_page_allocator, IPL_NONE);

                sz = SRMMU_L2SIZE * sizeof(int);
                pool_init(&L23_pool, sz, sz, 0, 0, "L2/L3 pagetable",
                          &pgt_page_allocator, IPL_NONE);
        }
#endif /* SUN4M || SUN4D */
#if defined(SUN4) || defined(SUN4C)
        if (CPU_HAS_SUNMMU) {
                sz = NPTESG * sizeof(int);
                pool_init(&pte_pool, sz, 0, 0, 0, "ptemap", NULL,
                    IPL_NONE);
        }
#endif /* SUN4 || SUN4C */
}


/*
 * Map physical addresses into kernel VM.
 */
vaddr_t
pmap_map(vaddr_t va, paddr_t pa, paddr_t endpa, int prot)
{
        int pgsize = PAGE_SIZE;

        while (pa < endpa) {
                pmap_kenter_pa(va, pa, prot, 0);
                va += pgsize;
                pa += pgsize;
        }
        pmap_update(pmap_kernel());
        return (va);
}

/*
 * Check a pmap for spuriously lingering mappings
 */
static inline void
pmap_quiet_check(struct pmap *pm)
{
#ifdef DEBUG
        int vs, vr;

        if (CPU_HAS_SUNMMU) {
#if defined(SUN4_MMU3L)
                if (TAILQ_FIRST(&pm->pm_reglist))
                        panic("pmap_destroy: region list not empty");
#endif
                if (TAILQ_FIRST(&pm->pm_seglist))
                        panic("pmap_destroy: segment list not empty");
        }

        for (vr = 0; vr < NUREG; vr++) {
                struct regmap *rp = &pm->pm_regmap[vr];

                if (HASSUN4_MMU3L) {
                        if (rp->rg_smeg != reginval)
                                printf("pmap_chk: spurious smeg in "
                                       "user region %d\n", vr);
                }
                if (CPU_HAS_SRMMU) {
                        int n;
#if defined(MULTIPROCESSOR)
                        for (n = 0; n < sparc_ncpus; n++)
#else
                        n = 0;
#endif
                        {
                                /* Did this cpu attach? */
                                if (pmap_kernel()->pm_reg_ptps[n] == 0)
                                        continue;

                                if (pm->pm_reg_ptps[n][vr] != SRMMU_TEINVALID)
                                        printf("pmap_chk: spurious PTP in user "
                                                "region %d on CPU %d\n", vr, n);
                        }
                }
                if (rp->rg_nsegmap != 0)
                        printf("pmap_chk: %d segments remain in "
                                "region %d\n", rp->rg_nsegmap, vr);
                if (rp->rg_segmap != NULL) {
                        printf("pmap_chk: segments still "
                                "allocated in region %d\n", vr);
                        for (vs = 0; vs < NSEGRG; vs++) {
                                struct segmap *sp = &rp->rg_segmap[vs];
                                if (sp->sg_npte != 0)
                                        printf("pmap_chk: %d ptes "
                                             "remain in segment %d\n",
                                                sp->sg_npte, vs);
                                if (sp->sg_pte != NULL) {
                                        printf("pmap_chk: ptes still "
                                             "allocated in segment %d\n", vs);
                                }
                                if (CPU_HAS_SUNMMU) {
                                        if (sp->sg_pmeg != seginval)
                                                printf("pmap_chk: pm %p(%d,%d) "
                                                  "spurious soft pmeg %d\n",
                                                  pm, vr, vs, sp->sg_pmeg);
                                }
                        }
                }

                /* Check for spurious pmeg entries in the MMU */
                if (pm->pm_ctx == NULL)
                        continue;
                if (CPU_HAS_SUNMMU) {
                        int ctx;
                        if (mmu_has_hole && (vr >= 32 && vr < (256 - 32)))
                                continue;
                        ctx = getcontext4();
                        setcontext4(pm->pm_ctxnum);
                        for (vs = 0; vs < NSEGRG; vs++) {
                                vaddr_t va = VSTOVA(vr,vs);
                                int pmeg = getsegmap(va);
                                if (pmeg != seginval)
                                        printf("pmap_chk: pm %p(%d,%d:%x): "
                                                "spurious pmeg %d\n",
                                                pm, vr, vs, (u_int)va, pmeg);
                        }
                        setcontext4(ctx);
                }
        }
        if (pm->pm_stats.resident_count) {
                printf("pmap_chk: res count %ld\n",
                       pm->pm_stats.resident_count);
        }
        if (pm->pm_stats.wired_count) {
                printf("pmap_chk: wired count %ld\n",
                       pm->pm_stats.wired_count);
        }
#endif /* DEBUG */
}

int
pmap_pmap_pool_ctor(void *arg, void *object, int flags)
{
        struct pmap *pm = object;
        u_long addr;

        memset(pm, 0, sizeof *pm);

        /*
         * `pmap_pool' entries include space for the per-CPU
         * region table pointer arrays.
         */
        addr = (u_long)pm + ALIGN(sizeof(struct pmap));
        pm->pm_regmap = (void *)addr;
        addr += ALIGN(NUREG * sizeof(struct regmap));
        pm->pm_reg_ptps = (int **)addr;
        addr += sparc_ncpus * sizeof(int *);
        pm->pm_reg_ptps_pa = (int *)addr;

        qzero((void *)pm->pm_regmap, NUREG * sizeof(struct regmap));

        /* pm->pm_ctx = NULL; // already done */

        if (CPU_HAS_SUNMMU) {
                TAILQ_INIT(&pm->pm_seglist);
#if defined(SUN4_MMU3L)
                TAILQ_INIT(&pm->pm_reglist);
                if (HASSUN4_MMU3L) {
                        int i;
                        for (i = NUREG; --i >= 0;)
                                pm->pm_regmap[i].rg_smeg = reginval;
                }
#endif
        }
#if defined(SUN4M) || defined(SUN4D)
        else {
                int i, n;

                /*
                 * We must allocate and initialize hardware-readable (MMU)
                 * pagetables. We must also map the kernel regions into this
                 * pmap's pagetables, so that we can access the kernel from
                 * this user context.
                 */
#if defined(MULTIPROCESSOR)
                for (n = 0; n < sparc_ncpus; n++)
#else
                n = 0;
#endif
                {
                        int *upt, *kpt;

#if defined(MULTIPROCESSOR)
                        /* Did this cpu attach? */
                        if (pmap_kernel()->pm_reg_ptps[n] == 0)
                                continue;
#endif

                        upt = pool_get(&L1_pool, flags);
                        pm->pm_reg_ptps[n] = upt;
                        pm->pm_reg_ptps_pa[n] = VA2PA((char *)upt);

                        /* Invalidate user space regions */
                        for (i = 0; i < NUREG; i++)
                                setpgt4m(upt++, SRMMU_TEINVALID);

                        /* Copy kernel regions */
                        kpt = &pmap_kernel()->pm_reg_ptps[n][VA_VREG(KERNBASE)];
                        for (i = 0; i < NKREG; i++)
                                setpgt4m(upt++, kpt[i]);
                }
        }
#endif /* SUN4M || SUN4D */

        /* XXX - a peculiar place to do this, but we can't do it in pmap_init
         * and here at least it's off the beaten code track.
         */
{static int x; if (x == 0) pool_setlowat(&pv_pool, 512), x = 1; }

        return (0);
}

void
pmap_pmap_pool_dtor(void *arg, void *object)
{
        struct pmap *pm = object;
        union ctxinfo *c;
        int s = splvm();        /* paranoia */

        DPRINTF(PDB_DESTROY, "pmap_pmap_pool_dtor(%p)", pm);

        if ((c = pm->pm_ctx) != NULL) {
                mutex_spin_enter(&ctx_lock);
                ctx_free(pm);
                mutex_spin_exit(&ctx_lock);
        }

#if defined(SUN4M) || defined(SUN4D)
        if (CPU_HAS_SRMMU) {
                int n;

#if defined(MULTIPROCESSOR)
                for (n = 0; n < sparc_ncpus; n++)
#else
                n = 0;
#endif
                {
                        int *pt;

#if defined(MULTIPROCESSOR)
                        /* Did this cpu attach? */
                        if (pmap_kernel()->pm_reg_ptps[n] == 0)
                                continue;
#endif

                        pt = pm->pm_reg_ptps[n];
                        pm->pm_reg_ptps[n] = NULL;
                        pm->pm_reg_ptps_pa[n] = 0;
                        pool_put(&L1_pool, pt);
                }
        }
#endif /* SUN4M || SUN4D */
        splx(s);
}

/*
 * Create and return a physical map.
 */
struct pmap *
pmap_create(void)
{
        struct pmap *pm;

        pm = pool_cache_get(&pmap_cache, PR_WAITOK);

        /*
         * Reset fields that are not preserved in the pmap cache pool.
         */
        pm->pm_refcount = 1;
#if defined(MULTIPROCESSOR)
        /* reset active CPU set */
        pm->pm_cpuset = 0;
#endif
        if (CPU_HAS_SUNMMU) {
                /* reset the region gap */
                pm->pm_gap_start = 0;
                pm->pm_gap_end = VA_VREG(VM_MAXUSER_ADDRESS);
        }

        DPRINTF(PDB_CREATE, "pmap_create[%d]: created %p", cpu_number(), pm);
        pmap_quiet_check(pm);

        return (pm);
}

/*
 * Retire the given pmap from service.
 * Should only be called if the map contains no valid mappings.
 */
void
pmap_destroy(struct pmap *pm)
{

        DPRINTF(PDB_DESTROY, "pmap_destroy[%d](%p)", cpu_number(), pm);
        membar_release();
        if (atomic_dec_uint_nv(&pm->pm_refcount) == 0) {
                membar_acquire();
                pmap_quiet_check(pm);
                pool_cache_put(&pmap_cache, pm);
        }
}

/*
 * Add a reference to the given pmap.
 */
void
pmap_reference(struct pmap *pm)
{

        atomic_inc_uint(&pm->pm_refcount);
}

#if defined(SUN4) || defined(SUN4C)
/*
 * helper to deallocate level 2 & 3 page tables.
 */
static void
pgt_lvl23_remove4_4c(struct pmap *pm, struct regmap *rp, struct segmap *sp,
                     int vr, int vs)
{
        vaddr_t va, tva;
        int i, pmeg;

        va = VSTOVA(vr,vs);
        if ((pmeg = sp->sg_pmeg) != seginval) {
                if (CTX_USABLE(pm,rp)) {
                        setcontext4(pm->pm_ctxnum);
                        setsegmap(va, seginval);
                } else {
                        /* no context, use context 0 */
                        setcontext4(0);
                        if (HASSUN4_MMU3L && rp->rg_smeg != reginval) {
                                setregmap(0, rp->rg_smeg);
                                tva = vs << SGSHIFT;
                                setsegmap(tva, seginval);
                        }
                }
                if (!HASSUN4_MMU3L) {
                        if (pm == pmap_kernel()) {
                                /* Unmap segment from all contexts */
                                for (i = ncontext; --i >= 0;) {
                                        setcontext4(i);
                                        setsegmap(va, seginval);
                                }
                        }
                }
                me_free(pm, pmeg);
                sp->sg_pmeg = seginval;
        }
        /* Free software tables for non-kernel maps */
        if (pm != pmap_kernel()) {
                pool_put(&pte_pool, sp->sg_pte);
                sp->sg_pte = NULL;
        }

        if (rp->rg_nsegmap <= 0)
                panic("pgt_rm: pm %p: nsegmap = %d\n", pm, rp->rg_nsegmap);

        if (--rp->rg_nsegmap == 0) {
#if defined(SUN4_MMU3L)
                if (HASSUN4_MMU3L) {
                        if (rp->rg_smeg != reginval) {
                                if (pm == pmap_kernel()) {
                                        /* Unmap from all contexts */
                                        for (i = ncontext; --i >= 0;) {
                                                setcontext4(i);
                                                setregmap(va, reginval);
                                        }
                                } else if (pm->pm_ctx) {
                                        setcontext4(pm->pm_ctxnum);
                                        setregmap(va, reginval);
                                }

                                /* Release MMU resource */
                                region_free(pm, rp->rg_smeg);
                                rp->rg_smeg = reginval;
                        }
                }
#endif /* SUN4_MMU3L */
                /* Free software tables for non-kernel maps */
                if (pm != pmap_kernel()) {
                        GAP_WIDEN(pm,vr);
                        pool_put(&segmap_pool, rp->rg_segmap);
                        rp->rg_segmap = NULL;
                }
        }
}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)
/*
 * SRMMU helper to deallocate level 2 & 3 page tables.
 */
static void
pgt_lvl23_remove4m(struct pmap *pm, struct regmap *rp, struct segmap *sp,
    int vr, int vs)
{

        /* Invalidate level 2 PTP entry */
        if (pm->pm_ctx)
                tlb_flush_segment(VSTOVA(vr,vs), pm->pm_ctxnum,
                                  PMAP_CPUSET(pm));
        setpgt4m(&rp->rg_seg_ptps[vs], SRMMU_TEINVALID);
        pool_put(&L23_pool, sp->sg_pte);
        sp->sg_pte = NULL;

        /* If region is now empty, remove level 2 pagetable as well */
        if (--rp->rg_nsegmap == 0) {
                int n = 0;
                if (pm->pm_ctx)
                        tlb_flush_region(VRTOVA(vr), pm->pm_ctxnum,
                                         PMAP_CPUSET(pm));
#if defined(MULTIPROCESSOR)
                /* Invalidate level 1 PTP entries on all CPUs */
                for (; n < sparc_ncpus; n++) {
                        if ((cpus[n]->flags & CPUFLG_HATCHED) == 0)
                                continue;
#endif
                        setpgt4m(&pm->pm_reg_ptps[n][vr], SRMMU_TEINVALID);
#if defined(MULTIPROCESSOR)
                }
#endif

                pool_put(&segmap_pool, rp->rg_segmap);
                rp->rg_segmap = NULL;
                pool_put(&L23_pool, rp->rg_seg_ptps);
        }
}
#endif /* SUN4M || SUN4D */

bool
pmap_remove_all(struct pmap *pm)
{
        if (pm->pm_ctx == NULL)
                return false;

#if defined(SUN4) || defined(SUN4C)
        if (CPU_HAS_SUNMMU) {
                int ctx = getcontext4();
                setcontext4(pm->pm_ctxnum);
                cache_flush_context(pm->pm_ctxnum);
                setcontext4(ctx);
        }
#endif

#if defined(SUN4M) || defined(SUN4D)
        if (CPU_HAS_SRMMU) {
                cache_flush_context(pm->pm_ctxnum);
        }
#endif

        pm->pm_flags |= PMAP_USERCACHECLEAN;
        return false;
}

/*
 * Remove the given range of mapping entries.
 * The starting and ending addresses are already rounded to pages.
 * Sheer lunacy: pmap_remove is often asked to remove nonexistent
 * mappings.
 */
void
pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva)
{
        vaddr_t nva;
        int vr, vs, s, ctx;
        void (*rm)(struct pmap *, vaddr_t, vaddr_t, int, int);

        DPRINTF(PDB_REMOVE, "pmap_remove[%d](%p, 0x%lx, 0x%lx)",
            cpu_number(), pm, va, endva);

        if (!CPU_HAS_SRMMU)
                write_user_windows();

        if (pm == pmap_kernel()) {
                /*
                 * Removing from kernel address space.
                 */
                rm = pmap_rmk;
        } else {
                /*
                 * Removing from user address space.
                 */
                rm = pmap_rmu;
        }

        ctx = getcontext();
        s = splvm();
        PMAP_LOCK();
        for (; va < endva; va = nva) {
                /* do one virtual segment at a time */
                vr = VA_VREG(va);
                vs = VA_VSEG(va);
                nva = VSTOVA(vr, vs + 1);
                if (nva == 0 || nva > endva)
                        nva = endva;
                if (pm->pm_regmap[vr].rg_nsegmap != 0)
                        (*rm)(pm, va, nva, vr, vs);
        }
        PMAP_UNLOCK();
        splx(s);
        setcontext(ctx);
}

/*
 * It is the same amount of work to cache_flush_page 16 pages
 * as to cache_flush_segment 1 segment, assuming a 64K cache size
 * and a 4K page size or a 128K cache size and 8K page size.
 */
#define PMAP_SFL_THRESHOLD      16      /* if > magic, use cache_flush_segment */

/*
 * Remove a range contained within a single segment.
 * These are egregiously complicated routines.
 */

#if defined(SUN4) || defined(SUN4C)

/* remove from kernel */
/*static*/ void
pmap_rmk4_4c(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
{
        int pte, mmupte, *ptep, perpage, npg;
        struct vm_page *pg;
        int nleft, pmeg, inmmu;
        struct regmap *rp;
        struct segmap *sp;

        rp = &pm->pm_regmap[vr];
        sp = &rp->rg_segmap[vs];

        if (rp->rg_nsegmap == 0)
                return;
        if ((nleft = sp->sg_npte) == 0)
                return;
        pmeg = sp->sg_pmeg;
        inmmu = pmeg != seginval;
        ptep = &sp->sg_pte[VA_VPG(va)];

        /* decide how to flush cache */
        npg = (endva - va) >> PGSHIFT;
        if (!inmmu) {
                perpage = 0;
        } else if (npg > PMAP_SFL_THRESHOLD) {
                /* flush the whole segment */
                perpage = 0;
                cache_flush_segment(vr, vs, 0);
        } else {
                /* flush each page individually; some never need flushing */
                perpage = (CACHEINFO.c_vactype != VAC_NONE);
        }

        for (; va < endva; va += NBPG, ptep++) {
                pte = *ptep;
                mmupte = inmmu ? getpte4(va) : 0;
                if ((pte & PG_V) == 0) {
#ifdef DIAGNOSTIC
                        if (inmmu && (mmupte & PG_V) != 0)
                                printf("rmk: inconsistent ptes va=%lx\n", va);
#endif
                        continue;
                }
                if ((pte & PG_TYPE) == PG_OBMEM) {
                        /* if cacheable, flush page as needed */
                        if (perpage && (mmupte & PG_NC) == 0)
                                cache_flush_page(va, 0);
                        if ((pg = pvhead4_4c(pte)) != NULL) {
                                if (inmmu)
                                        VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(mmupte);
                                pv_unlink4_4c(pg, pm, va);
                        }
                }
                nleft--;
#ifdef DIAGNOSTIC
                if (nleft < 0)
                        panic("pmap_rmk: too many PTEs in segment; "
                              "va 0x%lx; endva 0x%lx", va, endva);
#endif
                if (pte & PG_WIRED) {
                        sp->sg_nwired--;
                        pm->pm_stats.wired_count--;
                }

                if (inmmu)
                        setpte4(va, 0);
                *ptep = 0;
                pm->pm_stats.resident_count--;
        }

#ifdef DIAGNOSTIC
        if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
                panic("pmap_rmk: pm %p, va %lx: nleft=%d, nwired=%d",
                        pm, va, nleft, sp->sg_nwired);
#endif
        if ((sp->sg_npte = nleft) == 0)
                pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
        else if (sp->sg_nwired == 0) {
                if (sp->sg_pmeg != seginval)
                        mmu_pmeg_unlock(sp->sg_pmeg);
        }
}

#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)    /* SRMMU version of pmap_rmk */

/* remove from kernel (4m)*/
/* pm is already locked */
/*static*/ void
pmap_rmk4m(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
{
        int tpte, perpage, npg;
        struct vm_page *pg;
        struct regmap *rp;
        struct segmap *sp;

        rp = &pm->pm_regmap[vr];
        sp = &rp->rg_segmap[vs];
        if (rp->rg_nsegmap == 0)
                return;

        /* decide how to flush cache */
        npg = (endva - va) >> PGSHIFT;
        if (npg > PMAP_SFL_THRESHOLD) {
                /* flush the whole segment */
                perpage = 0;
                if (CACHEINFO.c_vactype != VAC_NONE)
                        cache_flush_segment(vr, vs, 0);
        } else {
                /* flush each page individually; some never need flushing */
                perpage = (CACHEINFO.c_vactype != VAC_NONE);
        }
        while (va < endva) {
                tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
                if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) {
#ifdef DEBUG
                        if ((pmapdebug & PDB_SANITYCHK) &&
                            (getpte4m(va) & SRMMU_TETYPE) == SRMMU_TEPTE)
                                panic("pmap_rmk: Spurious kTLB entry for 0x%lx",
                                      va);
#endif
                        va += NBPG;
                        continue;
                }
                if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
                        /* if cacheable, flush page as needed */
                        if (perpage && (tpte & SRMMU_PG_C))
                                cache_flush_page(va, 0);
                        if ((pg = pvhead4m(tpte)) != NULL) {
                                VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(tpte);
                                pv_unlink4m(pg, pm, va);
                        }
                }
                setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
                    SRMMU_TEINVALID, 1, 0, CPUSET_ALL);
                pm->pm_stats.resident_count--;
                va += NBPG;
        }
}
#endif /* SUN4M || SUN4D */

#if defined(SUN4) || defined(SUN4C)

/* remove from user */
/*static*/ void
pmap_rmu4_4c(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
{
        int *ptep, pteva, pte, perpage, npg;
        struct vm_page *pg;
        int nleft, pmeg, inmmu;
        struct regmap *rp;
        struct segmap *sp;

        rp = &pm->pm_regmap[vr];
        if (rp->rg_nsegmap == 0)
                return;
        sp = &rp->rg_segmap[vs];
        if ((nleft = sp->sg_npte) == 0)
                return;
        pmeg = sp->sg_pmeg;
        inmmu = pmeg != seginval;

        /*
         * PTEs are in MMU.  Invalidate in hardware, update ref &
         * mod bits, and flush cache if required.
         */
        if (!inmmu) {
                perpage = 0;
                pteva = 0;
        } else if (CTX_USABLE(pm,rp)) {
                /* process has a context, must flush cache */
                npg = (endva - va) >> PGSHIFT;
                setcontext4(pm->pm_ctxnum);
                if ((pm->pm_flags & PMAP_USERCACHECLEAN) != 0)
                        perpage = 0;
                else if (npg > PMAP_SFL_THRESHOLD) {
                        perpage = 0; /* flush the whole segment */
                        cache_flush_segment(vr, vs, pm->pm_ctxnum);
                } else
                        perpage = (CACHEINFO.c_vactype != VAC_NONE);
                pteva = va;
        } else {
                /* no context, use context 0; cache flush unnecessary */
                setcontext4(0);
                if (HASSUN4_MMU3L)
                        setregmap(0, tregion);
                /* XXX use per-CPU pteva? */
                setsegmap(0, pmeg);
                pteva = VA_VPG(va) << PGSHIFT;
                perpage = 0;
        }

        ptep = sp->sg_pte + VA_VPG(va);
        for (; va < endva; ptep++, pteva += NBPG, va += NBPG) {
                int mmupte;
                pte = *ptep;
                mmupte = inmmu ? getpte4(pteva) : 0;

                if ((pte & PG_V) == 0) {
#ifdef DIAGNOSTIC
                        if (inmmu && (mmupte & PG_V) != 0)
                                printf("pmap_rmu: pte=%x, mmupte=%x\n",
                                        pte, getpte4(pteva));
#endif
                        continue;
                }
                if ((pte & PG_TYPE) == PG_OBMEM) {
                        /* if cacheable, flush page as needed */
                        if (perpage && (mmupte & PG_NC) == 0)
                                cache_flush_page(va, pm->pm_ctxnum);
                        if ((pg = pvhead4_4c(pte)) != NULL) {
                                if (inmmu)
                                        VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4_4C(mmupte);
                                pv_unlink4_4c(pg, pm, va);
                        }
                }
                nleft--;
#ifdef DIAGNOSTIC
                if (nleft < 0)
                        panic("pmap_rmu: too many PTEs in segment; "
                             "va 0x%lx; endva 0x%lx", va, endva);
#endif
                if (inmmu)
                        setpte4(pteva, 0);

                if (pte & PG_WIRED) {
                        sp->sg_nwired--;
                        pm->pm_stats.wired_count--;
                }
                *ptep = 0;
                pm->pm_stats.resident_count--;
        }

#ifdef DIAGNOSTIC
        if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
                panic("pmap_rmu: pm %p, va %lx: nleft=%d, nwired=%d",
                        pm, va, nleft, sp->sg_nwired);
#endif
        if ((sp->sg_npte = nleft) == 0)
                pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
        else if (sp->sg_nwired == 0) {
                if (sp->sg_pmeg != seginval)
                        mmu_pmeg_unlock(sp->sg_pmeg);
        }
}

#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)    /* SRMMU version of pmap_rmu */
/* remove from user */
/* Note: pm is already locked */
/*static*/ void
pmap_rmu4m(struct pmap *pm, vaddr_t va, vaddr_t endva, int vr, int vs)
{
        int *pte0, perpage, npg;
        struct vm_page *pg;
        int nleft;
        struct regmap *rp;
        struct segmap *sp;

        rp = &pm->pm_regmap[vr];
        if (rp->rg_nsegmap == 0)
                return;
        sp = &rp->rg_segmap[vs];
        if ((nleft = sp->sg_npte) == 0)
                return;
        pte0 = sp->sg_pte;

        /*
         * Invalidate PTE in MMU pagetables. Flush cache if necessary.
         */
        if (pm->pm_ctx && (pm->pm_flags & PMAP_USERCACHECLEAN) == 0) {
                /* process has a context, must flush cache */
                if (CACHEINFO.c_vactype != VAC_NONE) {
                        npg = (endva - va) >> PGSHIFT;
                        if (npg > PMAP_SFL_THRESHOLD) {
                                perpage = 0; /* flush the whole segment */
                                cache_flush_segment(vr, vs, pm->pm_ctxnum);
                        } else
                                perpage = 1;
                } else
                        perpage = 0;
        } else {
                /* no context; cache flush unnecessary */
                perpage = 0;
        }
        for (; va < endva; va += NBPG) {
                int tpte;

                tpte = pte0[VA_SUN4M_VPG(va)];

                if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE) {
#ifdef DEBUG
                        if ((pmapdebug & PDB_SANITYCHK) &&
                            pm->pm_ctx &&
                            (getpte4m(va) & SRMMU_TEPTE) == SRMMU_TEPTE)
                                panic("pmap_rmu: Spurious uTLB entry for 0x%lx",
                                      va);
#endif
                        continue;
                }

                if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
                        /* if cacheable, flush page as needed */
                        if (perpage && (tpte & SRMMU_PG_C))
                                cache_flush_page(va, pm->pm_ctxnum);
                        if ((pg = pvhead4m(tpte)) != NULL) {
                                VM_MDPAGE_PVHEAD(pg)->pv_flags |= MR4M(tpte);
                                pv_unlink4m(pg, pm, va);
                        }
                }
                nleft--;
#ifdef DIAGNOSTIC
                if (nleft < 0)
                        panic("pmap_rmu: too many PTEs in segment; "
                              "va 0x%lx; endva 0x%lx", va, endva);
#endif
                setpgt4m_va(va, &pte0[VA_SUN4M_VPG(va)], SRMMU_TEINVALID,
                    pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm));
                pm->pm_stats.resident_count--;
                if (sp->sg_wiremap & (1 << VA_SUN4M_VPG(va))) {
                        sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
                        pm->pm_stats.wired_count--;
                }
        }

        /*
         * If the segment is all gone, and the context is loaded, give
         * the segment back.
         */
        if ((sp->sg_npte = nleft) == 0)
                pgt_lvl23_remove4m(pm, rp, sp, vr, vs);
}
#endif /* SUN4M || SUN4D */

/*
 * Lower (make more strict) the protection on the specified
 * physical page.
 *
 * There are only two cases: either the protection is going to 0
 * (in which case we do the dirty work here), or it is going from
 * to read-only (in which case pv_changepte does the trick).
 */

#if defined(SUN4) || defined(SUN4C)
void
pmap_page_protect4_4c(struct vm_page *pg, vm_prot_t prot)
{
        struct pvlist *pv, *npv;
        struct pmap *pm;
        vaddr_t va;
        int vr, vs, pteva, pte, *ptep;
        int flags, nleft, s, ctx;
        struct regmap *rp;
        struct segmap *sp;

#ifdef DEBUG
        if ((pmapdebug & PDB_CHANGEPROT) ||
            (pmapdebug & PDB_REMOVE && prot == VM_PROT_NONE))
                printf("pmap_page_protect(0x%lx, 0x%x)\n",
                        VM_PAGE_TO_PHYS(pg), prot);
#endif

        /*
         * Skip unmanaged pages, or operations that do not take
         * away write permission.
         */
        if (prot & VM_PROT_WRITE)
                return;

        write_user_windows();   /* paranoia */
        if (prot & VM_PROT_READ) {
                pv_changepte4_4c(pg, 0, PG_W);
                return;
        }

        /*
         * Remove all access to all people talking to this page.
         * Walk down PV list, removing all mappings.
         * The logic is much like that for pmap_remove,
         * but we know we are removing exactly one page.
         */
        s = splvm();
        pv = VM_MDPAGE_PVHEAD(pg);
        if (pv->pv_pmap == NULL) {
                splx(s);
                return;
        }
        ctx = getcontext4();

        /* This pv head will become empty, so clear caching state flags */
        flags = pv->pv_flags & ~(PV_NC|PV_ANC);

        while (pv != NULL) {
                pm = pv->pv_pmap;
                va = pv->pv_va;
                vr = VA_VREG(va);
                vs = VA_VSEG(va);
                rp = &pm->pm_regmap[vr];
                sp = &rp->rg_segmap[vs];
                if ((nleft = sp->sg_npte) <= 0)
                        panic("pmap_page_protect: empty vseg");
                sp->sg_npte = --nleft;
                ptep = &sp->sg_pte[VA_VPG(va)];

                if (*ptep & PG_WIRED) {
                        sp->sg_nwired--;
                        pm->pm_stats.wired_count--;
                }

                if (sp->sg_pmeg != seginval) {
                        /* Update PV flags */
                        if (CTX_USABLE(pm,rp)) {
                                setcontext4(pm->pm_ctxnum);
                                pteva = va;
                                cache_flush_page(va, pm->pm_ctxnum);
                        } else {
                                setcontext4(0);
                                /* XXX use per-CPU pteva? */
                                if (HASSUN4_MMU3L)
                                        setregmap(0, tregion);
                                setsegmap(0, sp->sg_pmeg);
                                pteva = VA_VPG(va) << PGSHIFT;
                        }

                        pte = getpte4(pteva);
#ifdef DIAGNOSTIC
                        if ((pte & PG_V) == 0)
                                panic("pmap_page_protect !PG_V: pg %p "
                                      "ctx %d, va 0x%lx, pte 0x%x",
                                      pg, pm->pm_ctxnum, va, pte);
#endif
                        flags |= MR4_4C(pte);

                        setpte4(pteva, 0);
#ifdef DIAGNOSTIC
                        if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
                                panic("pmap_page_protect: pm %p, va %lx: nleft=%d, nwired=%d",
                                        pm, va, nleft, sp->sg_nwired);
#endif
                        if (sp->sg_nwired == 0)
                                mmu_pmeg_unlock(sp->sg_pmeg);
                }

                *ptep = 0;
                pm->pm_stats.resident_count--;
                if (nleft == 0)
                        pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
                npv = pv->pv_next;
                if (pv != VM_MDPAGE_PVHEAD(pg))
                        pool_put(&pv_pool, pv);
                pv = npv;
        }

        /* Finally, update pv head */
        VM_MDPAGE_PVHEAD(pg)->pv_pmap = NULL;
        VM_MDPAGE_PVHEAD(pg)->pv_next = NULL;
        VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;
        setcontext4(ctx);
        splx(s);
}

/*
 * Lower (make more strict) the protection on the specified
 * range of this pmap.
 *
 * There are only two cases: either the protection is going to 0
 * (in which case we call pmap_remove to do the dirty work), or
 * it is going from read/write to read-only.  The latter is
 * fairly easy.
 */
void
pmap_protect4_4c(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
{
        int va, nva, vr, vs;
        int s, ctx;
        struct regmap *rp;
        struct segmap *sp;

        if ((prot & VM_PROT_READ) == 0) {
                pmap_remove(pm, sva, eva);
                return;
        }

        write_user_windows();
        ctx = getcontext4();
        s = splvm();
        PMAP_LOCK();
        for (va = sva; va < eva;) {
                vr = VA_VREG(va);
                vs = VA_VSEG(va);
                rp = &pm->pm_regmap[vr];
                nva = VSTOVA(vr,vs + 1);
                if (nva > eva)
                        nva = eva;
                if (rp->rg_nsegmap == 0) {
                        va = nva;
                        continue;
                }
#ifdef DEBUG
                if (rp->rg_segmap == NULL)
                        panic("pmap_protect: no segments");
#endif
                sp = &rp->rg_segmap[vs];
                if (sp->sg_npte == 0) {
                        va = nva;
                        continue;
                }
#ifdef DEBUG
                if (sp->sg_pte == NULL)
                        panic("pmap_protect: no pages");
#endif
                if (sp->sg_pmeg == seginval) {
                        int *ptep = &sp->sg_pte[VA_VPG(va)];

                        /* not in MMU; just clear PG_W from core copies */
                        for (; va < nva; va += NBPG)
                                *ptep++ &= ~PG_W;
                } else {
                        /* in MMU: take away write bits from MMU PTEs */
                        if (CTX_USABLE(pm,rp)) {
                                int pte;

                                /*
                                 * Flush cache so that any existing cache
                                 * tags are updated.  This is really only
                                 * needed for PTEs that lose PG_W.
                                 */
                                pmap_stats.ps_npg_prot_all +=
                                        (nva - va) >> PGSHIFT;
                                setcontext4(pm->pm_ctxnum);
                                for (; va < nva; va += NBPG) {
                                        pte = getpte4(va);
                                        if ((pte & (PG_W|PG_TYPE)) ==
                                            (PG_W|PG_OBMEM)) {
                                                pmap_stats.ps_npg_prot_actual++;
                                                cache_flush_page(va, pm->pm_ctxnum);
                                                setpte4(va, pte & ~PG_W);
                                        }
                                }
                        } else {
                                int pteva;

                                /*
                                 * No context, hence not cached;
                                 * just update PTEs.
                                 */
                                setcontext4(0);
                                /* XXX use per-CPU pteva? */
                                if (HASSUN4_MMU3L)
                                        setregmap(0, tregion);
                                setsegmap(0, sp->sg_pmeg);
                                pteva = VA_VPG(va) << PGSHIFT;
                                for (; va < nva; pteva += NBPG, va += NBPG)
                                        setpte4(pteva, getpte4(pteva) & ~PG_W);
                        }
                }
        }
        PMAP_UNLOCK();
        splx(s);
        setcontext4(ctx);
}

/*
 * Change the protection and/or wired status of the given (MI) virtual page.
 * XXX: should have separate function (or flag) telling whether only wiring
 * is changing.
 */
void
pmap_changeprot4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags)
{
        int vr, vs, newprot, ctx, pte, *ptep;
        int pmeg;
        struct regmap *rp;
        struct segmap *sp;

        DPRINTF(PDB_CHANGEPROT, "pmap_changeprot(%p, 0x%lx, 0x%x, 0x%x)",
            pm, va, prot, flags);

        if (pm == pmap_kernel())
                newprot = prot & VM_PROT_WRITE ? PG_S|PG_W : PG_S;
        else
                newprot = prot & VM_PROT_WRITE ? PG_W : 0;
        vr = VA_VREG(va);
        vs = VA_VSEG(va);
        rp = &pm->pm_regmap[vr];
        sp = &rp->rg_segmap[vs];
        ptep = &sp->sg_pte[VA_VPG(va)];

        pmap_stats.ps_changeprots++;

        pte = *ptep;
        if (pte & PG_WIRED && (flags & PMAP_WIRED) == 0) {
                pte &= ~PG_WIRED;
                sp->sg_nwired--;
                pm->pm_stats.wired_count--;
        } else if ((pte & PG_WIRED) == 0 && flags & PMAP_WIRED) {
                pte |= PG_WIRED;
                sp->sg_nwired++;
                pm->pm_stats.wired_count++;
        }
        pte = (pte & ~PG_PROT) | newprot;
        /* Update S/W pte entry */
        *ptep = pte;

        /* update PTEs in software or hardware */
        if ((pmeg = sp->sg_pmeg) != seginval) {
                /* update in hardware */
                ctx = getcontext4();
                if (CTX_USABLE(pm,rp)) {
                        /*
                         * Use current context.
                         * Flush cache if page has been referenced to
                         * avoid stale protection bits in the cache tags.
                         */
                        setcontext4(pm->pm_ctxnum);
                        pte = getpte4(va);
                        if ((pte & (PG_U|PG_NC|PG_TYPE)) == (PG_U|PG_OBMEM))
                                cache_flush_page(va, pm->pm_ctxnum);
                } else {
                        setcontext4(0);
                        /* XXX use per-CPU va? */
                        if (HASSUN4_MMU3L)
                                setregmap(0, tregion);
                        setsegmap(0, pmeg);
                        va = VA_VPG(va) << PGSHIFT;
                        pte = getpte4(va);
                }
                pte = (pte & ~PG_PROT) | newprot;
                setpte4(va, pte);
                setcontext4(ctx);
#ifdef DIAGNOSTIC
                if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
                        panic("pmap_protect: pm %p, va %lx: nleft=%d, nwired=%d",
                                pm, va, sp->sg_npte, sp->sg_nwired);
#endif
                if (sp->sg_nwired == 0)
                        mmu_pmeg_unlock(pmeg);
                else
                        mmu_pmeg_lock(pmeg);
        }
}

#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)
/*
 * Lower (make more strict) the protection on the specified
 * physical page.
 *
 * There are only two cases: either the protection is going to 0
 * (in which case we do the dirty work here), or it is going
 * to read-only (in which case pv_changepte does the trick).
 */
void
pmap_page_protect4m(struct vm_page *pg, vm_prot_t prot)
{
        struct pvlist *pv, *npv;
        struct pmap *pm;
        vaddr_t va;
        int vr, vs, tpte;
        int flags, nleft, s;
        struct regmap *rp;
        struct segmap *sp;

#ifdef DEBUG
        if ((pmapdebug & PDB_CHANGEPROT) ||
            (pmapdebug & PDB_REMOVE && prot == VM_PROT_NONE))
                printf("pmap_page_protect[%d](0x%lx, 0x%x)\n",
                        cpu_number(), VM_PAGE_TO_PHYS(pg), prot);
#endif
        s = splvm();
        PMAP_LOCK();

        if (prot & VM_PROT_READ) {
                pv_changepte4m(pg, 0, PPROT_WRITE);
                goto out;
        }

        /*
         * Remove all access to all people talking to this page.
         * Walk down PV list, removing all mappings. The logic is much
         * like that for pmap_remove, but we know we are removing exactly
         * one page.
         */
        pv = VM_MDPAGE_PVHEAD(pg);
        if (pv->pv_pmap == NULL)
                goto out;

        /* This pv head will become empty, so clear caching state flags */
        flags = pv->pv_flags & ~(PV_NC|PV_ANC);
        while (pv != NULL) {
                pm = pv->pv_pmap;
                va = pv->pv_va;
                vr = VA_VREG(va);
                vs = VA_VSEG(va);
                rp = &pm->pm_regmap[vr];
                if (rp->rg_nsegmap == 0)
                        panic("pmap_remove_all: empty vreg");
                sp = &rp->rg_segmap[vs];
                nleft = sp->sg_npte;
                if (pm != pmap_kernel()) {
                        if (nleft <= 0)
                                panic("pmap_page_protect: empty vseg");
                        sp->sg_npte = --nleft;
                }

                /*
                 * Invalidate PTE in MMU pagetables.
                 * Flush cache if necessary.
                 */
                if (pm->pm_ctx) {
                        cache_flush_page(va, pm->pm_ctxnum);
                }

                tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
                setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)], SRMMU_TEINVALID,
                    pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm));

                pm->pm_stats.resident_count--;
                if (sp->sg_wiremap & (1 << VA_SUN4M_VPG(va))) {
                        sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
                        pm->pm_stats.wired_count--;
                }

                if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE)
                        panic("pmap_page_protect !PG_V: pg %p va %lx", pg, va);

                flags |= MR4M(tpte);

                if (pm != pmap_kernel() && nleft == 0)
                        /*
                         * Entire user mode segment is gone
                         */
                        pgt_lvl23_remove4m(pm, rp, sp, vr, vs);

                npv = pv->pv_next;
                if (pv != VM_MDPAGE_PVHEAD(pg))
                        pool_put(&pv_pool, pv);
                pv = npv;
        }

        /* Finally, update pv head */
        VM_MDPAGE_PVHEAD(pg)->pv_pmap = NULL;
        VM_MDPAGE_PVHEAD(pg)->pv_next = NULL;
        VM_MDPAGE_PVHEAD(pg)->pv_flags = flags;

out:
        PMAP_UNLOCK();
        splx(s);
}

/*
 * Lower (make more strict) the protection on the specified
 * range of this pmap.
 */
void
pmap_protect4m(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
{
        vaddr_t va, nva;
        int s, vr, vs;
        struct regmap *rp;
        struct segmap *sp;
        int newprot;

        if ((prot & VM_PROT_READ) == 0) {
                pmap_remove(pm, sva, eva);
                return;
        }

        DPRINTF(PDB_CHANGEPROT,
            "pmap_protect[%d][curpid %d, ctx %d,%d](%lx, %lx, %x)",
            cpu_number(), curproc->p_pid, getcontext4m(),
            pm->pm_ctx ? pm->pm_ctxnum : -1, sva, eva, prot);

        newprot = pte_prot4m(pm, prot);

        write_user_windows();
        s = splvm();
        PMAP_LOCK();

        for (va = sva; va < eva;) {
                vr = VA_VREG(va);
                vs = VA_VSEG(va);
                rp = &pm->pm_regmap[vr];
                nva = VSTOVA(vr,vs + 1);
                if (nva > eva)
                        nva = eva;
                if (rp->rg_nsegmap == 0) {
                        va = nva;
                        continue;
                }
                sp = &rp->rg_segmap[vs];
                if (pm != pmap_kernel() && sp->sg_npte == 0) {
                        va = nva;
                        continue;
                }

                /*
                 * pages loaded: take away write bits from MMU PTEs
                 */
                pmap_stats.ps_npg_prot_all += (nva - va) >> PGSHIFT;
                for (; va < nva; va += NBPG) {
                        int tpte, npte;

                        tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
                        if ((tpte & SRMMU_PGTYPE) != PG_SUN4M_OBMEM)
                                continue;
                        if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE)
                                continue;
                        npte = (tpte & ~SRMMU_PROT_MASK) | newprot;
                        if (npte == tpte)
                                continue;

                        /*
                         * Flush cache so that any existing cache
                         * tags are updated.
                         */

                        pmap_stats.ps_npg_prot_actual++;
                        if (pm->pm_ctx) {
                                cache_flush_page(va, pm->pm_ctxnum);
                        }
                        updatepte4m(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
                            SRMMU_PROT_MASK, newprot, pm->pm_ctxnum,
                            PMAP_CPUSET(pm));
                }
        }
        PMAP_UNLOCK();
        splx(s);
}

/*
 * Change the protection and/or wired status of the given (MI) virtual page.
 * XXX: should have separate function (or flag) telling whether only wiring
 * is changing.
 */
void
pmap_changeprot4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags)
{
        int pte, newprot;
        struct regmap *rp;
        struct segmap *sp;
        bool owired;

        DPRINTF(PDB_CHANGEPROT, "pmap_changeprot[%d](%p, 0x%lx, 0x%x, 0x%x)",
            cpu_number(), pm, va, prot, flags);

        newprot = pte_prot4m(pm, prot);

        pmap_stats.ps_changeprots++;

        rp = &pm->pm_regmap[VA_VREG(va)];
        sp = &rp->rg_segmap[VA_VSEG(va)];

        pte = sp->sg_pte[VA_SUN4M_VPG(va)];
        owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va));

        if (owired) {
                pm->pm_stats.wired_count--;
                sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
        }
        if (flags & PMAP_WIRED) {
                pm->pm_stats.wired_count++;
                sp->sg_wiremap |= (1 << VA_SUN4M_VPG(va));
        }

        if (pm->pm_ctx) {
                /*
                 * Use current context.
                 * Flush cache if page has been referenced to
                 * avoid stale protection bits in the cache tags.
                 */

                if ((pte & (SRMMU_PG_C|SRMMU_PGTYPE)) ==
                    (SRMMU_PG_C|PG_SUN4M_OBMEM))
                        cache_flush_page(va, pm->pm_ctxnum);
        }

        setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
                 (pte & ~SRMMU_PROT_MASK) | newprot,
                 pm->pm_ctx != NULL, pm->pm_ctxnum, PMAP_CPUSET(pm));

}
#endif /* SUN4M || SUN4D */

/*
 * Insert (MI) physical page pa at virtual address va in the given pmap.
 * NB: the pa parameter includes type bits PMAP_OBIO, PMAP_NC as necessary.
 *
 * If pa is not in the `managed' range it will not be `bank mapped'.
 * This works during bootstrap only because the first 4MB happens to
 * map one-to-one.
 *
 * There may already be something else there, or we might just be
 * changing protections and/or wiring on an existing mapping.
 *      XXX     should have different entry points for changing!
 */

#if defined(SUN4) || defined(SUN4C)

int
pmap_enter4_4c(struct pmap *pm, vaddr_t va, paddr_t pa,
               vm_prot_t prot, u_int flags)
{
        struct vm_page *pg;
        int pteproto, ctx;
        int error;

        if (VA_INHOLE(va)) {
#ifdef DEBUG
                printf("pmap_enter: pm %p, va 0x%lx, pa 0x%lx: in MMU hole\n",
                        pm, va, pa);
#endif
                return 0;
        }

        DPRINTF(PDB_ENTER, "pmap_enter(%p, 0x%lx, 0x%lx, 0x%x, 0x%x)",
            pm, va, pa, prot, flags);

        pg = PHYS_TO_VM_PAGE(pa);
        pteproto = PG_V | PMAP_T2PTE_4(pa);
        pa &= ~PMAP_TNC_4;

        /*
         * Set up prototype for new PTE.  Cannot set PG_NC from PV_NC yet
         * since the pvlist no-cache bit might change as a result of the
         * new mapping.
         */
        pteproto |= atop(pa) & PG_PFNUM;
        if (prot & VM_PROT_WRITE)
                pteproto |= PG_W;
        if ((flags & PMAP_WIRED) != 0)
                pteproto |= PG_WIRED;
        if (flags & VM_PROT_ALL) {
                pteproto |= PG_U;
                if (flags & VM_PROT_WRITE) {
                        pteproto |= PG_M;
                }
        }

        write_user_windows();
        ctx = getcontext4();
        if (pm == pmap_kernel())
                error = pmap_enk4_4c(pm, va, prot, flags, pg, pteproto | PG_S);
        else
                error = pmap_enu4_4c(pm, va, prot, flags, pg, pteproto);
        setcontext4(ctx);
        return (error);
}

/* enter new (or change existing) kernel mapping */
int
pmap_enk4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
             struct vm_page *pg, int pteproto)
{
        int vr, vs, pte, s, inmmu;
        int *ptep;
        struct regmap *rp;
        struct segmap *sp;
        int error = 0;

        vr = VA_VREG(va);
        vs = VA_VSEG(va);
        rp = &pm->pm_regmap[vr];
        sp = &rp->rg_segmap[vs];
        ptep = &sp->sg_pte[VA_VPG(va)];
        s = splvm();            /* XXX way too conservative */

#if defined(SUN4_MMU3L)
        if (HASSUN4_MMU3L && rp->rg_smeg == reginval)
                mmu_pagein_reg(pm, rp, va, vr, &region_locked);
#endif

        inmmu = sp->sg_pmeg != seginval;
        if ((pte = *ptep) & PG_V) {

                /* old mapping exists, and is of the same pa type */
                if ((pte & (PG_PFNUM|PG_TYPE)) ==
                    (pteproto & (PG_PFNUM|PG_TYPE))) {
                        /* just changing protection and/or wiring */
                        pmap_changeprot4_4c(pm, va, prot, flags);
                        splx(s);
                        return (0);
                }

                if ((pte & PG_TYPE) == PG_OBMEM) {
                        struct vm_page *opg;

                        /*
                         * Switcheroo: changing pa for this va.
                         * If old pa was managed, remove from pvlist.
                         * If old page was cached, flush cache.
                         */
                        if ((opg = pvhead4_4c(pte)) != NULL)
                                pv_unlink4_4c(opg, pm, va);
                        if (inmmu && (pte & PG_NC) == 0) {
                                setcontext4(0); /* ??? */
                                cache_flush_page(va, 0);
                        }
                }
                *ptep = 0;
                if (inmmu)
                        setpte4(va, 0);
                if (pte & PG_WIRED) {
                        sp->sg_nwired--;
                        pm->pm_stats.wired_count--;
                }
                pm->pm_stats.resident_count--;
        } else {
                /* adding new entry */
                if (sp->sg_npte++ == 0) {
#ifdef DIAGNOSTIC
                        int i; for (i = 0; i < NPTESG; i++) {
                                if (sp->sg_pte[i] == 0)
                                        continue;
                                panic("pmap_enk: pm %p, va %lx: pte[%d] not empty\n",
                                        pm, va, i);
                        }
#endif
                        rp->rg_nsegmap++;
                }
        }

        /*
         * If the new mapping is for a managed PA, enter into pvlist.
         */
        if (pg != NULL && (error = pv_link4_4c(pg, pm, va, &pteproto)) != 0) {
                if (--sp->sg_npte == 0)
                        pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
                if ((flags & PMAP_CANFAIL) != 0)
                        goto out;
                panic("pmap_enter: cannot allocate PV entry");
        }

        /* Update S/W page table */
        *ptep = pteproto;
        if (pteproto & PG_WIRED) {
                sp->sg_nwired++;
                pm->pm_stats.wired_count++;
        }
        pm->pm_stats.resident_count++;

#ifdef DIAGNOSTIC
        if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
                panic("pmap_enk: pm %p, va %lx: nleft=%d, nwired=%d",
                        pm, va, sp->sg_npte, sp->sg_nwired);
#endif
        if (sp->sg_pmeg == seginval)
                mmu_pagein_seg(pm, sp, va, vr, vs,
                        (pteproto & PG_WIRED) != 0 ? &segm_locked : &segm_lru);
        else if ((pteproto & PG_WIRED) != 0)
                mmu_pmeg_lock(sp->sg_pmeg);

        /* Update H/W page table */
        setpte4(va, pteproto & ~PG_MBZ);
out:
        splx(s);
        return (error);
}

/* enter new (or change existing) user mapping */
int
pmap_enu4_4c(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
             struct vm_page *pg, int pteproto)
{
        int vr, vs, *ptep, pte, pmeg, s;
        int error = 0;
        struct regmap *rp;
        struct segmap *sp;

        pm->pm_flags &= ~PMAP_USERCACHECLEAN;

        vr = VA_VREG(va);
        vs = VA_VSEG(va);
        rp = &pm->pm_regmap[vr];
        s = splvm();                    /* XXX conservative */

        /*
         * If there is no space in which the PTEs can be written
         * while they are not in the hardware, this must be a new
         * virtual segment.  Get PTE space and count the segment.
         *
         * TO SPEED UP CTX ALLOC, PUT SEGMENT BOUNDS STUFF HERE
         * AND IN pmap_rmu()
         */

        GAP_SHRINK(pm,vr);

#ifdef DEBUG
        if (pm->pm_gap_end < pm->pm_gap_start) {
                printf("pmap_enu: gap_start 0x%x, gap_end 0x%x",
                        pm->pm_gap_start, pm->pm_gap_end);
                panic("pmap_enu: gap botch");
        }
#endif

        if (rp->rg_segmap == NULL) {
                /* definitely a new mapping */
                int i;
                int mflag = PR_NOWAIT;

        rretry:
                sp = (struct segmap *)pool_get(&segmap_pool, mflag);
                if (sp == NULL) {
                        if ((flags & PMAP_CANFAIL) != 0) {
                                error = ENOMEM;
                                goto out;
                        }
                        mflag = PR_WAITOK;
                        goto rretry;
                }
#ifdef DEBUG
                if (rp->rg_segmap != NULL)
                        panic("pmap_enter: segment filled during sleep");
#endif
                qzero((void *)sp, NSEGRG * sizeof (struct segmap));
                rp->rg_segmap = sp;
                rp->rg_nsegmap = 0;
                for (i = NSEGRG; --i >= 0;)
                        sp++->sg_pmeg = seginval;
        }

        sp = &rp->rg_segmap[vs];

        if ((ptep = sp->sg_pte) == NULL) {
                /* definitely a new mapping */
                int size = NPTESG * sizeof *ptep;
                int mflag = PR_NOWAIT;

        sretry:
                ptep = (int *)pool_get(&pte_pool, mflag);
                if (ptep == NULL) {
                        if ((flags & PMAP_CANFAIL) != 0) {
                                error = ENOMEM;
                                goto out;
                        }
                        mflag = PR_WAITOK;
                        goto sretry;
                }
#ifdef DEBUG
                if (sp->sg_pte != NULL)
                        panic("pmap_enter: pte filled during sleep");
                if (sp->sg_pmeg != seginval)
                        panic("pmap_enter: new ptes, but not seginval");
#endif
                qzero((void *)ptep, size);
                sp->sg_pte = ptep;
                sp->sg_npte = 1;
                rp->rg_nsegmap++;
        } else {
                /* might be a change: fetch old pte */
                pte = ptep[VA_VPG(va)];
                if (pte & PG_V) {
                        /* old mapping exists, and is of the same pa type */
                        if ((pte & (PG_PFNUM|PG_TYPE)) ==
                            (pteproto & (PG_PFNUM|PG_TYPE))) {
                                /* just changing prot and/or wiring */
                                pmap_changeprot4_4c(pm, va, prot, flags);
                                splx(s);
                                return (0);
                        }
                        /*
                         * Switcheroo: changing pa for this va.
                         * If old pa was managed, remove from pvlist.
                         * If old page was cached, flush cache.
                         */
#if 0
                        printf("%s[%d]: pmap_enu: changing existing "
                                "va(0x%lx)=>pa entry\n",
                                curproc->p_comm, curproc->p_pid, va);
#endif
                        if ((pte & PG_TYPE) == PG_OBMEM) {
                                struct vm_page *opg;
                                if ((opg = pvhead4_4c(pte)) != NULL)
                                        pv_unlink4_4c(opg, pm, va);
                                if (CACHEINFO.c_vactype != VAC_NONE &&
                                    (pmeg = sp->sg_pmeg) != seginval) {
                                        /* hardware pte */
                                        if (CTX_USABLE(pm,rp)) {
                                                setcontext4(pm->pm_ctxnum);
                                        } else {
                                                setcontext4(0);
                                                /* XXX use per-CPU pteva? */
                                                if (HASSUN4_MMU3L)
                                                        setregmap(0, tregion);
                                                setsegmap(0, pmeg);
                                        }
                                        cache_flush_page(va, pm->pm_ctxnum);
                                }
                        }
                        if (pte & PG_WIRED) {
                                sp->sg_nwired--;
                                pm->pm_stats.wired_count--;
                        }
                        pm->pm_stats.resident_count--;
                        ptep[VA_VPG(va)] = 0;
                        if (sp->sg_pmeg != seginval)
                                setpte4(va, 0);
                } else {
                        /* adding new entry */
                        sp->sg_npte++;
                }
        }

        if (pg != NULL && (error = pv_link4_4c(pg, pm, va, &pteproto)) != 0) {
                if (--sp->sg_npte == 0)
                        /* Sigh, undo pgt allocations */
                        pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);

                if ((flags & PMAP_CANFAIL) != 0)
                        goto out;
                panic("pmap_enter: cannot allocate PV entry");
        }

        /* Update S/W page table */
        ptep += VA_VPG(va);
        *ptep = pteproto;
        if (pteproto & PG_WIRED) {
                sp->sg_nwired++;
                pm->pm_stats.wired_count++;
        }
        pm->pm_stats.resident_count++;

#ifdef DIAGNOSTIC
        if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
                panic("pmap_enu: pm %p, va %lx: nleft=%d, nwired=%d",
                        pm, va, sp->sg_npte, sp->sg_nwired);
#endif

        if ((pmeg = sp->sg_pmeg) != seginval) {
                /* Update H/W page table */
                if (CTX_USABLE(pm,rp))
                        setcontext4(pm->pm_ctxnum);
                else {
                        setcontext4(0);
                        if (HASSUN4_MMU3L)
                                setregmap(0, tregion);
                        setsegmap(0, pmeg);
                        va = VA_VPG(va) << PGSHIFT;
                }
                setpte4(va, pteproto & ~PG_MBZ);
        }

out:
        splx(s);
        return (error);
}

void
pmap_kenter_pa4_4c(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
{
        struct pmap *pm = pmap_kernel();
        struct regmap *rp;
        struct segmap *sp;
        int vr, vs, s;
        int *ptep, pteproto;
        int lockit = 1;

        pteproto = PG_S | PG_V | PMAP_T2PTE_4(pa);
        pa &= ~PMAP_TNC_4;
        pteproto |= atop(pa) & PG_PFNUM;
        if (prot & VM_PROT_WRITE)
                pteproto |= PG_W;

        vr = VA_VREG(va);
        vs = VA_VSEG(va);
        rp = &pm->pm_regmap[vr];
        sp = &rp->rg_segmap[vs];
        ptep = &sp->sg_pte[VA_VPG(va)];

        if (lockit) {
                pteproto |= PG_WIRED;
                sp->sg_nwired++;
        }

        KASSERT((*ptep & PG_V) == 0);

        s = splvm();
#if defined(SUN4_MMU3L)
        if (HASSUN4_MMU3L && rp->rg_smeg == reginval)
                mmu_pagein_reg(pm, rp, va, vr, &region_locked);
#endif

        if (sp->sg_npte++ == 0) {
#ifdef DIAGNOSTIC
                int i; for (i = 0; i < NPTESG; i++) {
                        if (sp->sg_pte[i] == 0)
                                continue;
                        panic("pmap_enk: pm %p, va %lx: pte[%d] not empty\n",
                                pm, va, i);
                }
#endif
                rp->rg_nsegmap++;
        }

        /* Update S/W page table */
        *ptep = pteproto;

#ifdef DIAGNOSTIC
        if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
                panic("pmap_kenter: pm %p, va %lx: nleft=%d, nwired=%d",
                        pm, va, sp->sg_npte, sp->sg_nwired);
#endif

        if (sp->sg_pmeg == seginval) {
                mmu_pagein_seg(pm, sp, va, vr, vs,
                                lockit ? &segm_locked : &segm_lru);
        } else if (lockit)
                mmu_pmeg_lock(sp->sg_pmeg);

        /* Update H/W page table */
        setpte4(va, pteproto & ~PG_MBZ);
        splx(s);
}

#if notyet /* XXXMRG delete */
void pmap_lockmmu(vaddr_t sva, size_t sz);

void
pmap_lockmmu(vaddr_t sva, size_t sz)
{
        struct pmap *pm = pmap_kernel();
        vaddr_t va, eva;
        struct regmap *rp;
        struct segmap *sp;
        int vr, vs;

        if (CPU_HAS_SRMMU)
                return;

        eva = sva + sz;
        va = VA_ROUNDDOWNTOSEG(sva);

        for (; va < eva; va += NBPSG) {
                vr = VA_VREG(va);
                vs = VA_VSEG(va);
                rp = &pm->pm_regmap[vr];
                sp = &rp->rg_segmap[vs];

                KASSERT(sp->sg_npte != 0);

                if (sp->sg_pmeg == seginval)
                        mmu_pagein_seg(pm, sp, va, vr, vs, &segm_locked);
                else
                        mmu_pmeg_lock(sp->sg_pmeg);
        }
}
#endif

void
pmap_kremove4_4c(vaddr_t va, vsize_t len)
{
        struct pmap *pm = pmap_kernel();
        struct regmap *rp;
        struct segmap *sp;
        vaddr_t nva, endva;
        int pte, mmupte, *ptep, perpage, npg, inmmu;
        int nleft, pmeg;
        int vr, vs, s, ctx;

        endva = va + len;
        DPRINTF(PDB_REMOVE, "pmap_kremove(0x%lx, 0x%lx)", va, endva);

        write_user_windows();

        s = splvm();
        ctx = getcontext();
        PMAP_LOCK();
        setcontext4(0);
        for (; va < endva; va = nva) {
                /* do one virtual segment at a time */
                vr = VA_VREG(va);
                vs = VA_VSEG(va);
                nva = VSTOVA(vr, vs + 1);
                if (nva == 0 || nva > endva)
                        nva = endva;

                rp = &pm->pm_regmap[vr];
                sp = &rp->rg_segmap[vs];

                if (rp->rg_nsegmap == 0)
                        continue;
                nleft = sp->sg_npte;
                if (nleft == 0)
                        continue;
                pmeg = sp->sg_pmeg;
                inmmu = (pmeg != seginval);
                ptep = &sp->sg_pte[VA_VPG(va)];

                /* decide how to flush cache */
                npg = (nva - va) >> PGSHIFT;
                if (!inmmu) {
                        perpage = 0;
                } else if (npg > PMAP_SFL_THRESHOLD) {
                        /* flush the whole segment */
                        perpage = 0;
                        cache_flush_segment(vr, vs, 0);
                } else {
                        /*
                         * flush each page individually;
                         * some never need flushing
                         */
                        perpage = (CACHEINFO.c_vactype != VAC_NONE);
                }

                for (; va < nva; va += NBPG, ptep++) {
                        pte = *ptep;
                        mmupte = inmmu ? getpte4(va) : 0;
                        if ((pte & PG_V) == 0) {
#ifdef DIAGNOSTIC
                                if (inmmu && (mmupte & PG_V) != 0)
                                        printf("rmk: inconsistent ptes va=%lx\n", va);
#endif
                                continue;
                        }
                        if ((pte & PG_TYPE) == PG_OBMEM) {
                                /* if cacheable, flush page as needed */
                                if (perpage && (mmupte & PG_NC) == 0)
                                        cache_flush_page(va, 0);
                        }
                        nleft--;
#ifdef DIAGNOSTIC
                        if (nleft < 0)
                                panic("pmap_kremove: too many PTEs in segment; "
                                      "va 0x%lx; endva 0x%lx", va, endva);
#endif
                        if (pte & PG_WIRED)
                                sp->sg_nwired--;

                        if (inmmu)
                                setpte4(va, 0);
                        *ptep = 0;
                }

#ifdef DIAGNOSTIC
                if (sp->sg_nwired > nleft || sp->sg_nwired < 0)
                        panic("pmap_kremove: pm %p, va %lx: nleft=%d, nwired=%d",
                                pm, va, nleft, sp->sg_nwired);
#endif

                if ((sp->sg_npte = nleft) == 0)
                        pgt_lvl23_remove4_4c(pm, rp, sp, vr, vs);
                else if (sp->sg_nwired == 0) {
                        if (sp->sg_pmeg != seginval)
                                mmu_pmeg_unlock(sp->sg_pmeg);
                }
        }
        PMAP_UNLOCK();
        setcontext4(ctx);
        splx(s);
}

/*
 * Change protection on a range of kernel addresses.
 */
void
pmap_kprotect4_4c(vaddr_t va, vsize_t size, vm_prot_t prot)
{
        int pte, newprot, ctx;

        size = roundup(size,NBPG);
        newprot = prot & VM_PROT_WRITE ? PG_S|PG_W : PG_S;

        ctx = getcontext4();
        setcontext4(0);
        while (size > 0) {
                pte = getpte4(va);

                /*
                 * Flush cache if page has been referenced to
                 * avoid stale protection bits in the cache tags.
                 */
                if ((pte & (PG_NC|PG_TYPE)) == PG_OBMEM)
                        cache_flush_page(va, 0);

                pte = (pte & ~PG_PROT) | newprot;
                setpte4(va, pte);

                va += NBPG;
                size -= NBPG;
        }
        setcontext4(ctx);
}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)    /* SRMMU versions of enter routines */
/*
 * Insert (MI) physical page pa at virtual address va in the given pmap.
 * NB: the pa parameter includes type bits PMAP_OBIO, PMAP_NC as necessary.
 *
 * If pa is not in the `managed' range it will not be `bank mapped'.
 * This works during bootstrap only because the first 4MB happens to
 * map one-to-one.
 *
 * There may already be something else there, or we might just be
 * changing protections and/or wiring on an existing mapping.
 *      XXX     should have different entry points for changing!
 */

int
pmap_enter4m(struct pmap *pm, vaddr_t va, paddr_t pa,
             vm_prot_t prot, u_int flags)
{
        struct vm_page *pg;
        int pteproto;
        int error;

        DPRINTF(PDB_ENTER, "pmap_enter[curcpu %d, curpid %d, ctx %d,%d]"
            "(%p, 0x%lx, 0x%lx, 0x%x, 0x%x)",
            cpu_number(), curproc == NULL ? -1 : curproc->p_pid,
            getcontext4m(), pm->pm_ctx == NULL ? -1 : pm->pm_ctxnum,
            pm, va, pa, prot, flags);

        pg = PHYS_TO_VM_PAGE(pa);

        /* Initialise pteproto with cache bit */
        pteproto = (pa & PMAP_NC) == 0 ? SRMMU_PG_C : 0;

#ifdef DEBUG
        if (pa & PMAP_TYPE_SRMMU) {     /* this page goes in an iospace */
                if (cpuinfo.cpu_type == CPUTYP_MS1)
                        panic("pmap_enter4m: attempt to use 36-bit iospace on"
                              " MicroSPARC");
        }
#endif
        pteproto |= SRMMU_TEPTE;
        pteproto |= PMAP_T2PTE_SRMMU(pa);
        pa &= ~PMAP_TNC_SRMMU;

        /*
         * Set up prototype for new PTE.  Cannot set PG_NC from PV_NC yet
         * since the pvlist no-cache bit might change as a result of the
         * new mapping.
         */
        pteproto |= (atop(pa) << SRMMU_PPNSHIFT);

        /* Make sure we get a pte with appropriate perms! */
        pteproto |= pte_prot4m(pm, prot);
        if (flags & VM_PROT_ALL) {
                pteproto |= SRMMU_PG_R;
                if (flags & VM_PROT_WRITE) {
                        pteproto |= SRMMU_PG_M;
                }
        }

        if (pm == pmap_kernel())
                error = pmap_enk4m(pm, va, prot, flags, pg, pteproto | PPROT_S);
        else
                error = pmap_enu4m(pm, va, prot, flags, pg, pteproto);

        return (error);
}

/* enter new (or change existing) kernel mapping */
int
pmap_enk4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
           struct vm_page *pg, int pteproto)
{
        int vr, vs, tpte, s;
        struct regmap *rp;
        struct segmap *sp;
        int error = 0;

#ifdef DEBUG
        if (va < KERNBASE)
                panic("pmap_enk4m: can't enter va 0x%lx below KERNBASE", va);
#endif
        vr = VA_VREG(va);
        vs = VA_VSEG(va);
        rp = &pm->pm_regmap[vr];
        sp = &rp->rg_segmap[vs];

        kpreempt_disable();
        s = splvm();
        PMAP_LOCK();

        if (rp->rg_seg_ptps == NULL) /* enter new region */
                panic("pmap_enk4m: missing kernel region table for va 0x%lx",va);

        tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
        if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE) {

                /* old mapping exists, and is of the same pa type */

                if ((tpte & SRMMU_PPNMASK) == (pteproto & SRMMU_PPNMASK)) {
                        /* just changing protection and/or wiring */
                        pmap_changeprot4m(pm, va, prot, flags);
                        error = 0;
                        goto out;
                }

                if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
                        struct vm_page *opg;
#ifdef DEBUG
printf("pmap_enk4m: changing existing va=>pa entry: va 0x%lx, pteproto 0x%x, "
       "oldpte 0x%x\n", va, pteproto, tpte);
#endif
                        /*
                         * Switcheroo: changing pa for this va.
                         * If old pa was managed, remove from pvlist.
                         * If old page was cached, flush cache.
                         */
                        if ((opg = pvhead4m(tpte)) != NULL)
                                pv_unlink4m(opg, pm, va);
                        if (tpte & SRMMU_PG_C) {
                                cache_flush_page(va, 0);
                        }
                }

                /*
                 * Invalidate the mapping now, so we can avoid the
                 * de-map and update protocol when setting the new
                 * PTE below.
                 */
                setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
                        SRMMU_TEINVALID, pm->pm_ctx != NULL,
                        pm->pm_ctxnum, PMAP_CPUSET(pm));
                pm->pm_stats.resident_count--;
        }

        /*
         * If the new mapping is for a managed PA, enter into pvlist.
         */
        if (pg != NULL && (error = pv_link4m(pg, pm, va, &pteproto)) != 0) {
                if ((flags & PMAP_CANFAIL) != 0)
                        goto out;
                panic("pmap_enter: cannot allocate PV entry");
        }

        setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto);
        pm->pm_stats.resident_count++;
out:
        PMAP_UNLOCK();
        splx(s);
        kpreempt_enable();
        return (error);
}

/* enter new (or change existing) user mapping */
int
pmap_enu4m(struct pmap *pm, vaddr_t va, vm_prot_t prot, int flags,
           struct vm_page *pg, int pteproto)
{
        int vr, vs, *pte, tpte, s;
        int error = 0;
        struct regmap *rp;
        struct segmap *sp;
        bool owired;

#ifdef DEBUG
        if (KERNBASE < va)
                panic("pmap_enu4m: can't enter va 0x%lx above KERNBASE", va);
#endif

        pm->pm_flags &= ~PMAP_USERCACHECLEAN;

        vr = VA_VREG(va);
        vs = VA_VSEG(va);
        rp = &pm->pm_regmap[vr];
        s = splvm();
        PMAP_LOCK();

        if (rp->rg_segmap == NULL) {
                /* definitely a new mapping */
                int mflag = PR_NOWAIT;

        rretry:
                sp = (struct segmap *)pool_get(&segmap_pool, mflag);
                if (sp == NULL) {
                        if ((flags & PMAP_CANFAIL) != 0) {
                                error = ENOMEM;
                                goto out;
                        }
                        mflag = PR_WAITOK;
                        goto rretry;
                }
#ifdef DEBUG
                if (rp->rg_segmap != NULL)
                        panic("pmap_enu4m: segment filled during sleep");
#endif
                qzero((void *)sp, NSEGRG * sizeof (struct segmap));
                rp->rg_segmap = sp;
                rp->rg_nsegmap = 0;
                rp->rg_seg_ptps = NULL;
        }
        if (rp->rg_seg_ptps == NULL) {
                /* Need a segment table */
                int i, *ptd;
                int mflag = PR_NOWAIT;

        sretry:
                ptd = pool_get(&L23_pool, mflag);
                if (ptd == NULL) {
                        if ((flags & PMAP_CANFAIL) != 0) {
                                error = ENOMEM;
                                goto out;
                        }
                        mflag = PR_WAITOK;
                        goto sretry;
                }

                rp->rg_seg_ptps = ptd;
                for (i = 0; i < SRMMU_L2SIZE; i++)
                        setpgt4m(&ptd[i], SRMMU_TEINVALID);

                /* Replicate segment allocation in each CPU's region table */
#if defined(MULTIPROCESSOR)
                for (i = 0; i < sparc_ncpus; i++)
#else
                i = 0;
#endif
                {
#if defined(MULTIPROCESSOR)
                        if ((cpus[i]->flags & CPUFLG_HATCHED) == 0)
                                continue;
#endif
                        setpgt4m(&pm->pm_reg_ptps[i][vr],
                                 (VA2PA((void *)ptd) >> SRMMU_PPNPASHIFT) |
                                        SRMMU_TEPTD);
                }
        }

        sp = &rp->rg_segmap[vs];

        owired = false;
        if ((pte = sp->sg_pte) == NULL) {
                /* definitely a new mapping */
                int i;
                int mflag = PR_NOWAIT;

                pte = pool_get(&L23_pool, mflag);
                if (pte == NULL) {
                        if ((flags & PMAP_CANFAIL) != 0) {
                                error = ENOMEM;
                                goto out;
                        }
                        panic("pmap_enter: cannot allocate PTE table");
                }

                sp->sg_pte = pte;
                sp->sg_npte = 1;
                rp->rg_nsegmap++;
                for (i = 0; i < SRMMU_L3SIZE; i++)
                        setpgt4m(&pte[i], SRMMU_TEINVALID);
                setpgt4m(&rp->rg_seg_ptps[vs],
                        (VA2PA((void *)pte) >> SRMMU_PPNPASHIFT) | SRMMU_TEPTD);
        } else {
#ifdef DIAGNOSTIC
                if (sp->sg_npte <= 0)
                        panic("pm %p: npte %d", pm, sp->sg_npte);
#endif
                /*
                 * Might be a change: fetch old pte
                 */
                tpte = pte[VA_SUN4M_VPG(va)];

                if ((tpte & SRMMU_TETYPE) == SRMMU_TEPTE) {

                        /* old mapping exists, and is of the same pa type */
                        if ((tpte & SRMMU_PPNMASK) ==
                            (pteproto & SRMMU_PPNMASK)) {
                                /* just changing prot and/or wiring */
                                /* caller should call this directly: */
                                pmap_changeprot4m(pm, va, prot, flags);
                                error = 0;
                                goto out;
                        }
                        /*
                         * Switcheroo: changing pa for this va.
                         * If old pa was managed, remove from pvlist.
                         * If old page was cached, flush cache.
                         */
                        DPRINTF(PDB_SWITCHMAP,
                            "%s[%d]: pmap_enu: changing existing "
                            "va 0x%x: pte 0x%x=>0x%x",
                            curproc->p_comm, curproc->p_pid,
                            (int)va, tpte, pteproto);

                        if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
                                struct vm_page *opg;
                                if ((opg = pvhead4m(tpte)) != NULL) {
                                        VM_MDPAGE_PVHEAD(opg)->pv_flags |=
                                                        MR4M(tpte);
                                        pv_unlink4m(opg, pm, va);
                                }
                                if (pm->pm_ctx && (tpte & SRMMU_PG_C))
                                        cache_flush_page(va, pm->pm_ctxnum);
                        }
                        /*
                         * We end up in this `change map' branch relatively
                         * infrequently.
                         * Invalidate the mapping now, so we can avoid the
                         * de-map and update protocol when setting the new
                         * PTE below.
                         */
                        setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
                                SRMMU_TEINVALID, pm->pm_ctx != NULL,
                                pm->pm_ctxnum, PMAP_CPUSET(pm));
                        pm->pm_stats.resident_count--;
                        owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va));
                } else {
                        /* adding new entry */
                        sp->sg_npte++;
                }
        }

        if (pg != NULL && (error = pv_link4m(pg, pm, va, &pteproto)) != 0) {
                if (--sp->sg_npte == 0)
                        /* Sigh, undo pgt allocations */
                        pgt_lvl23_remove4m(pm, rp, sp, vr, vs);

                if ((flags & PMAP_CANFAIL) != 0)
                        goto out;
                panic("pmap_enter: cannot allocate PV entry");
        }

        /*
         * Update PTEs, flush TLB as necessary.
         */
        setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto);
        pm->pm_stats.resident_count++;
        if (owired) {
                pm->pm_stats.wired_count--;
                sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
        }
        if (flags & PMAP_WIRED) {
                pm->pm_stats.wired_count++;
                sp->sg_wiremap |= (1 << VA_SUN4M_VPG(va));
        }

out:
        PMAP_UNLOCK();
        splx(s);
        return (error);
}

void
pmap_kenter_pa4m(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
{
        struct pmap *pm = pmap_kernel();
        struct regmap *rp;
        struct segmap *sp;
        int pteproto, vr, vs;

        /* Initialise pteproto with cache bit */
        pteproto = (pa & PMAP_NC) == 0 ? SRMMU_PG_C : 0;
        pteproto |= SRMMU_TEPTE | PPROT_S;
        pteproto |= PMAP_T2PTE_SRMMU(pa);
        pteproto |= (atop(pa & ~PMAP_TNC_SRMMU) << SRMMU_PPNSHIFT);
        pteproto |= pte_kprot4m(prot);

        vr = VA_VREG(va);
        vs = VA_VSEG(va);
        rp = &pm->pm_regmap[vr];
        sp = &rp->rg_segmap[vs];

        KASSERT((sp->sg_pte[VA_SUN4M_VPG(va)] & SRMMU_TETYPE) != SRMMU_TEPTE);

        setpgt4m(&sp->sg_pte[VA_SUN4M_VPG(va)], pteproto);
}

void
pmap_kremove4m(vaddr_t va, vsize_t len)
{
        struct pmap *pm = pmap_kernel();
        struct regmap *rp;
        struct segmap *sp;
        vaddr_t endva, nva;
        int vr, vs;
        int tpte, perpage, npg, s;

        /*
         * The kernel pmap doesn't need to be locked, but the demap lock
         * in updatepte() requires interrupt protection.
         */
        kpreempt_disable();
        s = splvm();

        endva = va + len;
        for (; va < endva; va = nva) {
                /* do one virtual segment at a time */
                vr = VA_VREG(va);
                vs = VA_VSEG(va);
                nva = VSTOVA(vr, vs + 1);
                if (nva == 0 || nva > endva) {
                        nva = endva;
                }

                rp = &pm->pm_regmap[vr];
                sp = &rp->rg_segmap[vs];

                /* decide how to flush the cache */
                npg = (nva - va) >> PGSHIFT;
                if (npg > PMAP_SFL_THRESHOLD) {
                        /* flush the whole segment */
                        perpage = 0;
                        if (CACHEINFO.c_vactype != VAC_NONE) {
                                cache_flush_segment(vr, vs, 0);
                        }
                } else {
                        /*
                         * flush each page individually;
                         * some never need flushing
                         */
                        perpage = (CACHEINFO.c_vactype != VAC_NONE);
                }
                for (; va < nva; va += NBPG) {
                        tpte = sp->sg_pte[VA_SUN4M_VPG(va)];
                        if ((tpte & SRMMU_TETYPE) != SRMMU_TEPTE)
                                continue;

                        if ((tpte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
                                /* if cacheable, flush page as needed */
                                if (perpage && (tpte & SRMMU_PG_C))
                                        cache_flush_page(va, 0);
                        }
                        setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
                                 SRMMU_TEINVALID, 1, 0, CPUSET_ALL);
                }
        }
        splx(s);
        kpreempt_enable();
}

/*
 * Change protection on a range of kernel addresses.
 */
void
pmap_kprotect4m(vaddr_t va, vsize_t size, vm_prot_t prot)
{
        struct pmap *pm = pmap_kernel();
        int pte, newprot, s;
        struct regmap *rp;
        struct segmap *sp;

        size = roundup(size,NBPG);
        newprot = pte_kprot4m(prot);

        /*
         * The kernel pmap doesn't need to be locked, but the demap lock
         * in updatepte() requires interrupt protection.
         */
        kpreempt_disable();
        s = splvm();

        while (size > 0) {
                rp = &pm->pm_regmap[VA_VREG(va)];
                sp = &rp->rg_segmap[VA_VSEG(va)];
                pte = sp->sg_pte[VA_SUN4M_VPG(va)];

                /*
                 * Flush cache if page has been referenced to
                 * avoid stale protection bits in the cache tags.
                 */
                if ((pte & (SRMMU_PG_C|SRMMU_PGTYPE)) ==
                    (SRMMU_PG_C|PG_SUN4M_OBMEM))
                        cache_flush_page(va, 0);

                setpgt4m_va(va, &sp->sg_pte[VA_SUN4M_VPG(va)],
                         (pte & ~SRMMU_PROT_MASK) | newprot,
                         1, pm->pm_ctxnum, PMAP_CPUSET(pm));

                va += NBPG;
                size -= NBPG;
        }
        splx(s);
        kpreempt_enable();
}
#endif /* SUN4M || SUN4D */

/*
 * Clear the wiring attribute for a map/virtual-address pair.
 */
/* ARGSUSED */
void
pmap_unwire(struct pmap *pm, vaddr_t va)
{
        int vr, vs, *ptep;
        struct regmap *rp;
        struct segmap *sp;
        bool owired;

        kpreempt_disable();
        vr = VA_VREG(va);
        vs = VA_VSEG(va);
        rp = &pm->pm_regmap[vr];
        sp = &rp->rg_segmap[vs];

        owired = false;
        if (CPU_HAS_SUNMMU) {
                ptep = &sp->sg_pte[VA_VPG(va)];
                owired = *ptep & PG_WIRED;
                *ptep &= ~PG_WIRED;
        }
        if (CPU_HAS_SRMMU) {
                owired = sp->sg_wiremap & (1 << VA_SUN4M_VPG(va));
                sp->sg_wiremap &= ~(1 << VA_SUN4M_VPG(va));
        }
        if (!owired) {
                pmap_stats.ps_useless_changewire++;
                kpreempt_enable();
                return;
        }

        pm->pm_stats.wired_count--;
#if defined(SUN4) || defined(SUN4C)
        if (CPU_HAS_SUNMMU && --sp->sg_nwired <= 0) {
#ifdef DIAGNOSTIC
                if (sp->sg_nwired > sp->sg_npte || sp->sg_nwired < 0)
                        panic("pmap_unwire: pm %p, va %lx: nleft=%d, nwired=%d",
                                pm, va, sp->sg_npte, sp->sg_nwired);
#endif
                if (sp->sg_pmeg != seginval)
                        mmu_pmeg_unlock(sp->sg_pmeg);
        }
#endif /* SUN4 || SUN4C */
        kpreempt_enable();
}

/*
 * Extract the physical page address associated
 * with the given map/virtual_address pair.
 * GRR, the vm code knows; we should not have to do this!
 */

#if defined(SUN4) || defined(SUN4C)
bool
pmap_extract4_4c(struct pmap *pm, vaddr_t va, paddr_t *pap)
{
        int vr, vs;
        struct regmap *rp;
        struct segmap *sp;
        int pte, *ptep;

        vr = VA_VREG(va);
        vs = VA_VSEG(va);
        rp = &pm->pm_regmap[vr];
        if (rp->rg_segmap == NULL) {
                DPRINTF(PDB_FOLLOW, "pmap_extract: invalid segment (%d)", vr);
                return (false);
        }
        sp = &rp->rg_segmap[vs];
        ptep = sp->sg_pte;
        if (ptep == NULL) {
                DPRINTF(PDB_FOLLOW, "pmap_extract: invalid segment");
                return (false);
        }
        pte = ptep[VA_VPG(va)];

        if ((pte & PG_V) == 0) {
                DPRINTF(PDB_FOLLOW, "pmap_extract: invalid pte");
                return (false);
        }
        pte &= PG_PFNUM;
        if (pap != NULL)
                *pap = (pte << PGSHIFT) | (va & PGOFSET);
        return (true);
}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)    /* SRMMU version of pmap_extract */
/*
 * Extract the physical page address associated
 * with the given map/virtual_address pair.
 * GRR, the vm code knows; we should not have to do this!
 */
bool
pmap_extract4m(struct pmap *pm, vaddr_t va, paddr_t *pap)
{
        struct regmap *rp;
        struct segmap *sp;
        int pte;
        int vr, vs, s, v = false;
        bool can_lock = lock_available;

        vr = VA_VREG(va);
        vs = VA_VSEG(va);

        /*
         * The kernel pmap doesn't need to be locked, but the demap lock
         * requires interrupt protection.
         */
        s = splvm();
        if (pm != pmap_kernel()) {
                PMAP_LOCK();
        }

        rp = &pm->pm_regmap[vr];
        if (rp->rg_segmap == NULL) {
                DPRINTF(PDB_FOLLOW, "pmap_extract: no segmap");
                goto out;
        }

        sp = &rp->rg_segmap[vs];
        if (sp->sg_pte == NULL) {
                DPRINTF(PDB_FOLLOW, "pmap_extract: no ptes");
                goto out;
        }

        pte = sp->sg_pte[VA_SUN4M_VPG(va)];
        if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE) {
                DPRINTF(PDB_FOLLOW, "pmap_extract: invalid pte of type %d",
                    pte & SRMMU_TETYPE);
                /*
                 * We can read a spurious invalid pte if the system is in
                 * the middle of the PTE update protocol. So, acquire the
                 * demap lock and retry.
                 */
                if (__predict_true(can_lock))
                        mutex_spin_enter(&demap_lock);
                pte = sp->sg_pte[VA_SUN4M_VPG(va)];
                if (__predict_true(can_lock))
                        mutex_spin_exit(&demap_lock);
                if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE)
                        goto out;
        }
#ifdef DIAGNOSTIC
        if (pm != pmap_kernel() && sp->sg_npte <= 0)
                panic("pmap_extract: pm %p: npte = %d\n", pm, sp->sg_npte);
#endif

        if (pap != NULL)
                *pap = ptoa((pte & SRMMU_PPNMASK) >> SRMMU_PPNSHIFT) |
                    VA_OFF(va);

        v = true;
out:
        if (pm != pmap_kernel()) {
                PMAP_UNLOCK();
        }
        splx(s);
        return (v);
}
#endif /* sun4m */

int pmap_copy_disabled=0;

/*
 * Copy the range specified by src_addr/len
 * from the source map to the range dst_addr/len
 * in the destination map.
 *
 * This routine is only advisory and need not do anything.
 */
/* ARGSUSED */
void
pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap,
          vaddr_t dst_addr, vsize_t len, vaddr_t src_addr)
{
#if notyet
        struct regmap *rp;
        struct segmap *sp;

        if (pmap_copy_disabled)
                return;
#ifdef DIAGNOSTIC
        if (VA_OFF(src_addr) != 0)
                printf("pmap_copy: addr not page aligned: 0x%lx\n", src_addr);
        if ((len & (NBPG-1)) != 0)
                printf("pmap_copy: length not page aligned: 0x%lx\n", len);
#endif

        if (src_pmap == NULL)
                return;

        if (CPU_HAS_SRMMU) {
                int i, npg, pte;
                paddr_t pa;

                npg = len >> PGSHIFT;
                for (i = 0; i < npg; i++) {
                        if ((rp = src_pmap->pm_regmap) == NULL)
                                continue;
                        rp += VA_VREG(src_addr);

                        if ((sp = rp->rg_segmap) == NULL)
                                continue;
                        sp += VA_VSEG(src_addr);
                        if (sp->sg_npte == 0)
                                continue;

                        pte = sp->sg_pte[VA_SUN4M_VPG(src_addr)];
                        if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE)
                                continue;

                        pa = ptoa((pte & SRMMU_PPNMASK) >> SRMMU_PPNSHIFT);
                        pmap_enter(dst_pmap, dst_addr,
                                   pa,
                                   /* XXX - need to copy VM_PROT_EXEC too */
                                   (pte & PPROT_WRITE)
                                        ? (VM_PROT_WRITE | VM_PROT_READ)
                                        : VM_PROT_READ,
                                   0);
                        src_addr += NBPG;
                        dst_addr += NBPG;
                }
                pmap_update(dst_pmap);
        }
#endif
}

#if defined(SUN4) || defined(SUN4C)
/*
 * Clear the modify bit for the given physical page.
 */
bool
pmap_clear_modify4_4c(struct vm_page *pg)
{
        bool rv;

        (void) pv_syncflags4_4c(pg);
        rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD;
        VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_MOD;
        return (rv);
}

/*
 * Tell whether the given physical page has been modified.
 */
bool
pmap_is_modified4_4c(struct vm_page *pg)
{

        return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD ||
                pv_syncflags4_4c(pg) & PV_MOD);
}

/*
 * Clear the reference bit for the given physical page.
 */
bool
pmap_clear_reference4_4c(struct vm_page *pg)
{
        bool rv;

        (void) pv_syncflags4_4c(pg);
        rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF;
        VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_REF;
        return (rv);
}

/*
 * Tell whether the given physical page has been referenced.
 */
bool
pmap_is_referenced4_4c(struct vm_page *pg)
{

        return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF ||
                pv_syncflags4_4c(pg) & PV_REF);
}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)

/*
 * SRMMU versions of bit test/set routines
 *
 * Note that the 4m-specific routines should eventually service these
 * requests from their page tables, and the whole pvlist bit mess should
 * be dropped for the 4m (unless this causes a performance hit from
 * tracing down pagetables/regmap/segmaps).
 */

/*
 * Clear the modify bit for the given physical page.
 */
bool
pmap_clear_modify4m(struct vm_page *pg)
{
        bool rv;

        (void) pv_syncflags4m(pg);
        rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD4M;
        VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_MOD4M;
        return (rv);
}

/*
 * Tell whether the given physical page has been modified.
 */
bool
pmap_is_modified4m(struct vm_page *pg)
{

        return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_MOD4M ||
                pv_syncflags4m(pg) & PV_MOD4M);
}

/*
 * Clear the reference bit for the given physical page.
 */
bool
pmap_clear_reference4m(struct vm_page *pg)
{
        bool rv;

        (void) pv_syncflags4m(pg);
        rv = VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF4M;
        VM_MDPAGE_PVHEAD(pg)->pv_flags &= ~PV_REF4M;
        return (rv);
}

/*
 * Tell whether the given physical page has been referenced.
 */
bool
pmap_is_referenced4m(struct vm_page *pg)
{

        return (VM_MDPAGE_PVHEAD(pg)->pv_flags & PV_REF4M ||
                pv_syncflags4m(pg) & PV_REF4M);
}
#endif /* SUN4M || SUN4D */

/*
 * Fill the given MI physical page with zero bytes.
 *
 * We avoid stomping on the cache.
 * XXX  might be faster to use destination's context and allow cache to fill?
 */

#if defined(SUN4) || defined(SUN4C)

void
pmap_zero_page4_4c(paddr_t pa)
{
        struct vm_page *pg;
        void *va;
        int pte;

        if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
                /*
                 * The following might not be necessary since the page
                 * is being cleared because it is about to be allocated,
                 * i.e., is in use by no one.
                 */
                pv_flushcache4_4c(pg);
        }
        pte = PG_V | PG_S | PG_W | PG_NC | (atop(pa) & PG_PFNUM);

        va = cpuinfo.vpage[0];
        setpte4(va, pte);
        qzero(va, NBPG);
        setpte4(va, 0);
}

/*
 * Copy the given MI physical source page to its destination.
 *
 * We avoid stomping on the cache as above (with same `XXX' note).
 * We must first flush any write-back cache for the source page.
 * We go ahead and stomp on the kernel's virtual cache for the
 * source page, since the cache can read memory MUCH faster than
 * the processor.
 */
void
pmap_copy_page4_4c(paddr_t src, paddr_t dst)
{
        struct vm_page *pg;
        char *sva, *dva;
        int spte, dpte;

        if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) {
                if (CACHEINFO.c_vactype == VAC_WRITEBACK)
                        pv_flushcache4_4c(pg);
        }
        spte = PG_V | PG_S | (atop(src) & PG_PFNUM);

        if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) {
                /* similar `might not be necessary' comment applies */
                if (CACHEINFO.c_vactype != VAC_NONE)
                        pv_flushcache4_4c(pg);
        }
        dpte = PG_V | PG_S | PG_W | PG_NC | (atop(dst) & PG_PFNUM);

        sva = cpuinfo.vpage[0];
        dva = cpuinfo.vpage[1];
        setpte4(sva, spte);
        setpte4(dva, dpte);
        qcopy(sva, dva, NBPG);  /* loads cache, so we must ... */
        cache_flush_page((vaddr_t)sva, getcontext4());
        setpte4(sva, 0);
        setpte4(dva, 0);
}
#endif /* SUN4 || SUN4C */

#if defined(SUN4M) || defined(SUN4D)    /* SRMMU version of copy/zero routines */
/*
 * Fill the given MI physical page with zero bytes.
 *
 * We avoid stomping on the cache.
 * XXX  might be faster to use destination's context and allow cache to fill?
 */
void
pmap_zero_page4m(paddr_t pa)
{
        struct vm_page *pg;
        void *va;
        int pte;

        kpreempt_disable();
        if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
                /*
                 * The following VAC flush might not be necessary since the
                 * page is being cleared because it is about to be allocated,
                 * i.e., is in use by no one.
                 * In the case of a physical cache, a flush (or just an
                 * invalidate, if possible) is usually necessary when using
                 * uncached access to clear it.
                 */
                if (CACHEINFO.c_vactype != VAC_NONE)
                        pv_flushcache4m(pg);
                else
                        pcache_flush_page(pa, 1);
        }
        pte = SRMMU_TEPTE | PPROT_N_RWX | (pa >> SRMMU_PPNPASHIFT);
        if (CACHEINFO.c_flags & CACHE_MANDATORY)
                pte |= SRMMU_PG_C;

        va = cpuinfo.vpage[0];
        setpgt4m(cpuinfo.vpage_pte[0], pte);
        qzero(va, NBPG);
        /*
         * Remove temporary mapping (which is kernel-only, so the
         * context used for TLB flushing does not matter)
         */
        sp_tlb_flush((int)va, 0, ASI_SRMMUFP_L3);
        setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
        kpreempt_enable();
}

/*
 * Viking/MXCC specific version of pmap_zero_page
 */
void
pmap_zero_page_viking_mxcc(paddr_t pa)
{
        u_int offset;
        u_int stream_data_addr = MXCC_STREAM_DATA;
        uint64_t v = (uint64_t)pa;

        kpreempt_disable();
        /* Load MXCC stream data register with 0 (bottom 32 bytes only) */
        stda(stream_data_addr+0, ASI_CONTROL, 0);
        stda(stream_data_addr+8, ASI_CONTROL, 0);
        stda(stream_data_addr+16, ASI_CONTROL, 0);
        stda(stream_data_addr+24, ASI_CONTROL, 0);

        /* Then write the stream data register to each block in the page */
        v |= MXCC_STREAM_C;
        for (offset = 0; offset < NBPG; offset += MXCC_STREAM_BLKSZ) {
                stda(MXCC_STREAM_DST, ASI_CONTROL, v | offset);
        }
        kpreempt_enable();
}

/*
 * HyperSPARC/RT625 specific version of pmap_zero_page
 */
void
pmap_zero_page_hypersparc(paddr_t pa)
{
        struct vm_page *pg;
        void *va;
        int pte;
        int offset;

        kpreempt_disable();
        /*
         * We still have to map the page, since ASI_BLOCKFILL
         * takes virtual addresses. This also means we have to
         * consider cache aliasing; therefore we still need
         * to flush the cache here. All we gain is the speed-up
         * in zero-fill loop itself..
         */
        if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
                /*
                 * The following might not be necessary since the page
                 * is being cleared because it is about to be allocated,
                 * i.e., is in use by no one.
                 */
                if (CACHEINFO.c_vactype != VAC_NONE)
                        pv_flushcache4m(pg);
        }
        pte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RWX | (pa >> SRMMU_PPNPASHIFT);

        va = cpuinfo.vpage[0];
        setpgt4m(cpuinfo.vpage_pte[0], pte);
        for (offset = 0; offset < NBPG; offset += 32) {
                sta((char *)va + offset, ASI_BLOCKFILL, 0);
        }
        /* Remove temporary mapping */
        sp_tlb_flush((int)va, 0, ASI_SRMMUFP_L3);
        setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
        kpreempt_enable();
}

/*
 * Copy the given MI physical source page to its destination.
 *
 * We avoid stomping on the cache as above (with same `XXX' note).
 * We must first flush any write-back cache for the source page.
 * We go ahead and stomp on the kernel's virtual cache for the
 * source page, since the cache can read memory MUCH faster than
 * the processor.
 */
void
pmap_copy_page4m(paddr_t src, paddr_t dst)
{
        struct vm_page *pg;
        void *sva, *dva;
        int spte, dpte;

        kpreempt_disable();
        if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) {
                if (CACHEINFO.c_vactype == VAC_WRITEBACK)
                        pv_flushcache4m(pg);
        }

        spte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RX |
                (src >> SRMMU_PPNPASHIFT);

        if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) {
                /* similar `might not be necessary' comment applies */
                if (CACHEINFO.c_vactype != VAC_NONE)
                        pv_flushcache4m(pg);
                else
                        pcache_flush_page(dst, 1);
        }

        dpte = SRMMU_TEPTE | PPROT_N_RWX | (dst >> SRMMU_PPNPASHIFT);
        if (CACHEINFO.c_flags & CACHE_MANDATORY)
                dpte |= SRMMU_PG_C;

        sva = cpuinfo.vpage[0];
        dva = cpuinfo.vpage[1];
        setpgt4m(cpuinfo.vpage_pte[0], spte);
        setpgt4m(cpuinfo.vpage_pte[1], dpte);
        qcopy(sva, dva, NBPG);  /* loads cache, so we must ... */
        cpuinfo.sp_vcache_flush_page((vaddr_t)sva, getcontext4m());
        sp_tlb_flush((int)sva, 0, ASI_SRMMUFP_L3);
        setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
        sp_tlb_flush((int)dva, 0, ASI_SRMMUFP_L3);
        setpgt4m(cpuinfo.vpage_pte[1], SRMMU_TEINVALID);
        kpreempt_enable();
}

/*
 * Viking/MXCC specific version of pmap_copy_page
 */
void
pmap_copy_page_viking_mxcc(paddr_t src, paddr_t dst)
{
        u_int offset;
        uint64_t v1 = (uint64_t)src;
        uint64_t v2 = (uint64_t)dst;

        kpreempt_disable();
        /* Enable cache-coherency */
        v1 |= MXCC_STREAM_C;
        v2 |= MXCC_STREAM_C;

        /* Copy through stream data register */
        for (offset = 0; offset < NBPG; offset += MXCC_STREAM_BLKSZ) {
                stda(MXCC_STREAM_SRC, ASI_CONTROL, v1 | offset);
                stda(MXCC_STREAM_DST, ASI_CONTROL, v2 | offset);
        }
        kpreempt_enable();
}

/*
 * HyperSPARC/RT625 specific version of pmap_copy_page
 */
void
pmap_copy_page_hypersparc(paddr_t src, paddr_t dst)
{
        struct vm_page *pg;
        void *sva, *dva;
        int spte, dpte;
        int offset;

        kpreempt_disable();
        /*
         * We still have to map the pages, since ASI_BLOCKCOPY
         * takes virtual addresses. This also means we have to
         * consider cache aliasing; therefore we still need
         * to flush the cache here. All we gain is the speed-up
         * in copy loop itself..
         */

        if ((pg = PHYS_TO_VM_PAGE(src)) != NULL) {
                if (CACHEINFO.c_vactype == VAC_WRITEBACK)
                        pv_flushcache4m(pg);
        }

        spte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RX |
                (src >> SRMMU_PPNPASHIFT);

        if ((pg = PHYS_TO_VM_PAGE(dst)) != NULL) {
                /* similar `might not be necessary' comment applies */
                if (CACHEINFO.c_vactype != VAC_NONE)
                        pv_flushcache4m(pg);
        }

        dpte = SRMMU_TEPTE | SRMMU_PG_C | PPROT_N_RWX |
                (dst >> SRMMU_PPNPASHIFT);

        sva = cpuinfo.vpage[0];
        dva = cpuinfo.vpage[1];
        setpgt4m(cpuinfo.vpage_pte[0], spte);
        setpgt4m(cpuinfo.vpage_pte[1], dpte);

        for (offset = 0; offset < NBPG; offset += 32) {
                sta((char *)dva + offset, ASI_BLOCKCOPY, (char *)sva + offset);
        }

        sp_tlb_flush((int)sva, 0, ASI_SRMMUFP_L3);
        setpgt4m(cpuinfo.vpage_pte[0], SRMMU_TEINVALID);
        sp_tlb_flush((int)dva, 0, ASI_SRMMUFP_L3);
        setpgt4m(cpuinfo.vpage_pte[1], SRMMU_TEINVALID);
        kpreempt_enable();
}
#endif /* SUN4M || SUN4D */

/*
 * Turn off cache for a given (va, number of pages).
 *
 * We just assert PG_NC for each PTE; the addresses must reside
 * in locked kernel space.  A cache flush is also done.
 */
void
kvm_uncache(char *va, int npages)
{
        struct vm_page *pg;
        int pte;

        if (CPU_HAS_SRMMU) {
#if defined(SUN4M) || defined(SUN4D)
                for (; --npages >= 0; va = (char *)va + NBPG) {
                        pte = getpte4m((vaddr_t) va);
                        if ((pte & SRMMU_TETYPE) != SRMMU_TEPTE)
                                panic("kvm_uncache: table entry not pte");

                        if ((pte & SRMMU_PGTYPE) == PG_SUN4M_OBMEM) {
                                if ((pg = pvhead4m(pte)) != NULL) {
                                        pv_uncache(pg);
                                        return;
                                }
                                cache_flush_page((vaddr_t)va, 0);
                        }

                        pte &= ~SRMMU_PG_C;
                        setpte4m((vaddr_t)va, pte);
                }
#endif
        } else {
#if defined(SUN4) || defined(SUN4C)
                for (; --npages >= 0; va += NBPG) {
                        pte = getpte4(va);
                        if ((pte & PG_V) == 0)
                                panic("kvm_uncache !pg_v");

                        if ((pte & PG_TYPE) == PG_OBMEM) {
                                if ((pg = pvhead4_4c(pte)) != NULL) {
                                        pv_uncache(pg);
                                        return;
                                }
                                cache_flush_page((vaddr_t)va, 0);
                        }
                        pte |= PG_NC;
                        setpte4(va, pte);
                }
#endif
        }
}

#if 0 /* not used */
/*
 * Turn on IO cache for a given (va, number of pages).
 *
 * We just assert PG_NC for each PTE; the addresses must reside
 * in locked kernel space.  A cache flush is also done.
 */
void
kvm_iocache(char *va, int npages)
{

#if defined(SUN4M)
        if (CPU_ISSUN4M) /* %%%: Implement! */
                panic("kvm_iocache: 4m iocache not implemented");
#endif
#if defined(SUN4D)
        if (CPU_ISSUN4D) /* %%%: Implement! */
                panic("kvm_iocache: 4d iocache not implemented");
#endif
#if defined(SUN4) || defined(SUN4C)
        for (; --npages >= 0; va += NBPG) {
                int pte = getpte4(va);
                if ((pte & PG_V) == 0)
                        panic("kvm_iocache !pg_v");
                pte |= PG_IOC;
                setpte4(va, pte);
        }
#endif
}
#endif

/*
 * Find first virtual address >= *va that is
 * least likely to cause cache aliases.
 * (This will just seg-align mappings.)
 */
void
pmap_prefer(vaddr_t foff, vaddr_t *vap, size_t size, int td)
{
        vaddr_t va = *vap;
        long m;

        m = CACHE_ALIAS_DIST;
        if (m == 0)             /* m=0 => no cache aliasing */
                return;

        if (VA_INHOLE(va)) {
                if (td)
                        va = MMU_HOLE_START - size;
                else
                        va = MMU_HOLE_END;
        }

        va = (va & ~(m - 1)) | (foff & (m - 1));

        if (td) {
                if (va > *vap)
                        va -= m;
        } else {
                if (va < *vap)
                        va += m;
        }
        *vap = va;
}

void
pmap_redzone(void)
{

        pmap_remove(pmap_kernel(), KERNBASE, KERNBASE+NBPG);
}

/*
 * Activate the address space for the specified process.  If the
 * process is the current process, load the new MMU context.
 */
void
pmap_activate(struct lwp *l)
{
        pmap_t pm = l->l_proc->p_vmspace->vm_map.pmap;

        if (pm == pmap_kernel() || l != curlwp) {
                return;
        }

        mutex_spin_enter(&ctx_lock);
        if (pm->pm_ctx == NULL) {
                ctx_alloc(pm);  /* performs setcontext() */
        } else {
                setcontext(pm->pm_ctxnum);
        }
        PMAP_SET_CPUSET(pm, &cpuinfo);
        mutex_spin_exit(&ctx_lock);
}

/*
 * Deactivate the address space of the specified process.
 */
void
pmap_deactivate(struct lwp *l)
{
        struct proc *p = l->l_proc;
        pmap_t pm = p->p_vmspace->vm_map.pmap;

        if (pm == pmap_kernel() || l != curlwp) {
                return;
        }

        write_user_windows();
        mutex_spin_enter(&ctx_lock);
        if (pm->pm_ctx) {
                (*cpuinfo.pure_vcache_flush)();

#if defined(SUN4M) || defined(SUN4D)
                if (CPU_HAS_SRMMU)
                        sp_tlb_flush(0, pm->pm_ctxnum, ASI_SRMMUFP_L0);
#endif
        }

        /* we no longer need broadcast tlb flushes for this pmap. */
        PMAP_CLR_CPUSET(pm, &cpuinfo);
        mutex_spin_exit(&ctx_lock);
}

#ifdef DEBUG
/*
 * Check consistency of a pmap (time consuming!).
 */
void
pm_check(char *s, struct pmap *pm)
{

        if (pm == pmap_kernel())
                pm_check_k(s, pm);
        else
                pm_check_u(s, pm);
}

void
pm_check_u(char *s, struct pmap *pm)
{
        struct regmap *rp;
        struct segmap *sp;
        int cpu, n, vs, vr, j, m, *pte;

        cpu = cpuinfo.ci_cpuid;

        if (pm->pm_regmap == NULL)
                panic("%s: CPU %d: CHK(pmap %p): no region mapping",
                        s, cpu, pm);

#if defined(SUN4M) || defined(SUN4D)
        if (CPU_HAS_SRMMU &&
            (pm->pm_reg_ptps[cpu] == NULL ||
             pm->pm_reg_ptps_pa[cpu] != VA2PA((void *)pm->pm_reg_ptps[cpu])))
                panic("%s: CPU %d: CHK(pmap %p): no SRMMU region table or bad pa: "
                      "tblva=%p, tblpa=0x%x",
                        s, cpu, pm, pm->pm_reg_ptps[cpu], pm->pm_reg_ptps_pa[cpu]);

        if (CPU_HAS_SRMMU && pm->pm_ctx != NULL &&
            (cpuinfo.ctx_tbl[pm->pm_ctxnum] != ((VA2PA((void *)pm->pm_reg_ptps[cpu])
                                              >> SRMMU_PPNPASHIFT) |
                                             SRMMU_TEPTD)))
            panic("%s: CPU %d: CHK(pmap %p): SRMMU region table at 0x%x not installed "
                  "for context %d", s, cpu, pm, pm->pm_reg_ptps_pa[cpu], pm->pm_ctxnum);
#endif

        for (vr = 0; vr < NUREG; vr++) {
                rp = &pm->pm_regmap[vr];
                if (rp->rg_nsegmap == 0)
                        continue;
                if (rp->rg_segmap == NULL)
                        panic("%s: CPU %d: CHK(vr %d): nsegmap = %d; sp==NULL",
                                s, cpu, vr, rp->rg_nsegmap);
#if defined(SUN4M) || defined(SUN4D)
                if (CPU_HAS_SRMMU && rp->rg_seg_ptps == NULL)
                    panic("%s: CPU %d: CHK(vr %d): nsegmap=%d; no SRMMU segment table",
                          s, cpu, vr, rp->rg_nsegmap);
                if (CPU_HAS_SRMMU &&
                    pm->pm_reg_ptps[cpu][vr] != ((VA2PA((void *)rp->rg_seg_ptps) >>
                                            SRMMU_PPNPASHIFT) | SRMMU_TEPTD))
                    panic("%s: CPU %d: CHK(vr %d): SRMMU segtbl not installed",
                                s, cpu, vr);
#endif
                if ((unsigned int)rp < KERNBASE)
                        panic("%s: CPU %d: rp=%p", s, cpu, rp);
                n = 0;
                for (vs = 0; vs < NSEGRG; vs++) {
                        sp = &rp->rg_segmap[vs];
                        if ((unsigned int)sp < KERNBASE)
                                panic("%s: CPU %d: sp=%p", s, cpu, sp);
                        if (sp->sg_npte != 0) {
                                n++;
                                if (sp->sg_pte == NULL)
                                        panic("%s: CPU %d: CHK(vr %d, vs %d): npte=%d, "
                                           "pte=NULL", s, cpu, vr, vs, sp->sg_npte);
#if defined(SUN4M) || defined(SUN4D)
                                if (CPU_HAS_SRMMU &&
                                    rp->rg_seg_ptps[vs] !=
                                     ((VA2PA((void *)sp->sg_pte)
                                        >> SRMMU_PPNPASHIFT) |
                                       SRMMU_TEPTD))
                                    panic("%s: CPU %d: CHK(vr %d, vs %d): SRMMU page "
                                          "table not installed correctly",
                                                s, cpu, vr, vs);
#endif
                                pte=sp->sg_pte;
                                m = 0;
                                for (j=0; j<NPTESG; j++,pte++)
                                    if ((CPU_HAS_SRMMU
                                         ?((*pte & SRMMU_TETYPE) == SRMMU_TEPTE)
                                         :(*pte & PG_V)))
                                        m++;
                                if (m != sp->sg_npte)
                                        printf("%s: CPU %d: user CHK(vr %d, vs %d): "
                                            "npte(%d) != # valid(%d)\n",
                                                s, cpu, vr, vs, sp->sg_npte, m);
                        }
                }
                if (n != rp->rg_nsegmap)
                        panic("%s: CPU %d: CHK(vr %d): inconsistent "
                                "# of pte's: %d, should be %d",
                                s, cpu, vr, rp->rg_nsegmap, n);
        }
        return;
}

/* Note: not as extensive as pm_check_u. */
void
pm_check_k(char *s, struct pmap *pm)
{
        struct regmap *rp;
        int cpu, vr, vs, n;

        cpu = cpu_number();

        if (pm->pm_regmap == NULL)
                panic("%s: CHK(pmap %p): no region mapping", s, pm);

#if defined(SUN4M) || defined(SUN4D)
        if (CPU_HAS_SRMMU &&
            (pm->pm_reg_ptps[cpu] == NULL ||
             pm->pm_reg_ptps_pa[cpu] != VA2PA((void *)pm->pm_reg_ptps[cpu])))
            panic("%s: CPU %d: CHK(pmap %p): no SRMMU region table or bad pa: tblva=%p, tblpa=0x%x",
                  s, cpu, pm, pm->pm_reg_ptps[cpu], pm->pm_reg_ptps_pa[cpu]);

        if (CPU_HAS_SRMMU &&
            (cpuinfo.ctx_tbl[0] != ((VA2PA((void *)pm->pm_reg_ptps[cpu]) >>
                                             SRMMU_PPNPASHIFT) | SRMMU_TEPTD)))
            panic("%s: CPU %d: CHK(pmap %p): SRMMU region table at 0x%x not installed "
                  "for context %d", s, cpu, pm, pm->pm_reg_ptps_pa[cpu], 0);
#endif
        for (vr = NUREG; vr < NUREG+NKREG; vr++) {
                rp = &pm->pm_regmap[vr];
                if (rp->rg_segmap == NULL)
                        panic("%s: CPU %d: CHK(vr %d): nsegmap = %d; sp==NULL",
                                s, cpu, vr, rp->rg_nsegmap);
                if (rp->rg_nsegmap == 0)
                        continue;
#if defined(SUN4M) || defined(SUN4D)
                if (CPU_HAS_SRMMU && rp->rg_seg_ptps == NULL)
                    panic("%s: CPU %d: CHK(vr %d): nsegmap=%d; no SRMMU segment table",
                          s, cpu, vr, rp->rg_nsegmap);

                if (CPU_HAS_SRMMU && vr != NUREG /* 1st kseg is per CPU */ &&
                    pm->pm_reg_ptps[cpu][vr] != ((VA2PA((void *)rp->rg_seg_ptps) >>
                                            SRMMU_PPNPASHIFT) | SRMMU_TEPTD))
                    panic("%s: CPU %d: CHK(vr %d): SRMMU segtbl not installed",
                                s, cpu, vr);
#endif
                if (CPU_HAS_SRMMU) {
                        n = NSEGRG;
                } else {
                        for (n = 0, vs = 0; vs < NSEGRG; vs++) {
                                if (rp->rg_segmap[vs].sg_npte)
                                        n++;
                        }
                }
                if (n != rp->rg_nsegmap)
                        printf("%s: CPU %d: kernel CHK(vr %d): inconsistent "
                                "# of pte's: %d, should be %d\n",
                                s, cpu, vr, rp->rg_nsegmap, n);
        }
        return;
}
#endif

/*
 * Return the number of disk blocks that pmap_dumpmmu() will dump.
 */
int
pmap_dumpsize(void)
{
        int     sz;

        sz = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t));
        sz += npmemarr * sizeof(phys_ram_seg_t);
        sz += sizeof(kernel_segmap_store);

        if (CPU_HAS_SUNMMU)
                /* For each pmeg in the MMU, we'll write NPTESG PTEs. */
                sz += (seginval + 1) * NPTESG * sizeof(int);

        return btodb(sz + DEV_BSIZE - 1);
}

/*
 * Write the core dump headers and MD data to the dump device.
 * We dump the following items:
 *
 *      kcore_seg_t              MI header defined in <sys/kcore.h>)
 *      cpu_kcore_hdr_t          MD header defined in <machine/kcore.h>)
 *      phys_ram_seg_t[npmemarr] physical memory segments
 *      segmap_t[NKREG*NSEGRG]   the kernel's segment map
 *      the MMU pmegs on sun4/sun4c
 */
int
pmap_dumpmmu(int (*dump)(dev_t, daddr_t, void *, size_t),
             daddr_t blkno)
{
        kcore_seg_t     *ksegp;
        cpu_kcore_hdr_t *kcpup;
        phys_ram_seg_t  memseg;
        int             error = 0;
        int             i, memsegoffset, segmapoffset, pmegoffset;
        int             buffer[dbtob(1) / sizeof(int)];
        int             *bp, *ep;
#if defined(SUN4C) || defined(SUN4)
        int     pmeg;
#endif

#define EXPEDITE(p,n) do {                                              \
        int *sp = (int *)(p);                                           \
        int sz = (n);                                                   \
        while (sz > 0) {                                                \
                *bp++ = *sp++;                                          \
                if (bp >= ep) {                                         \
                        error = (*dump)(dumpdev, blkno,                 \
                                        (void *)buffer, dbtob(1));      \
                        if (error != 0)                                 \
                                return (error);                         \
                        ++blkno;                                        \
                        bp = buffer;                                    \
                }                                                       \
                sz -= 4;                                                \
        }                                                               \
} while (0)

        setcontext(0);

        /* Setup bookkeeping pointers */
        bp = buffer;
        ep = &buffer[sizeof(buffer) / sizeof(buffer[0])];

        /* Fill in MI segment header */
        ksegp = (kcore_seg_t *)bp;
        CORE_SETMAGIC(*ksegp, KCORE_MAGIC, MID_MACHINE, CORE_CPU);
        ksegp->c_size = dbtob(pmap_dumpsize()) - ALIGN(sizeof(kcore_seg_t));

        /* Fill in MD segment header (interpreted by MD part of libkvm) */
        kcpup = (cpu_kcore_hdr_t *)((int)bp + ALIGN(sizeof(kcore_seg_t)));
        kcpup->cputype = cputyp;
        kcpup->kernbase = KERNBASE;
        kcpup->nmemseg = npmemarr;
        kcpup->memsegoffset = memsegoffset = ALIGN(sizeof(cpu_kcore_hdr_t));
        kcpup->nsegmap = NKREG*NSEGRG;
        kcpup->segmapoffset = segmapoffset =
                memsegoffset + npmemarr * sizeof(phys_ram_seg_t);

        kcpup->npmeg = (CPU_HAS_SUNMMU) ? seginval + 1 : 0;
        kcpup->pmegoffset = pmegoffset =
                segmapoffset + kcpup->nsegmap * sizeof(struct segmap);

        /* Note: we have assumed everything fits in buffer[] so far... */
        bp = (int *)((int)kcpup + ALIGN(sizeof(cpu_kcore_hdr_t)));

#if 0
        /* Align storage for upcoming quad-aligned segment array */
        while (bp != (int *)ALIGN(bp)) {
                int dummy = 0;
                EXPEDITE(&dummy, 4);
        }
#endif

        for (i = 0; i < npmemarr; i++) {
                memseg.start = pmemarr[i].addr;
                memseg.size = pmemarr[i].len;
                EXPEDITE((void *)&memseg, sizeof(phys_ram_seg_t));
        }

        EXPEDITE(&kernel_segmap_store, sizeof(kernel_segmap_store));

        if (CPU_HAS_SRMMU)
                goto out;

#if defined(SUN4C) || defined(SUN4)
        /*
         * dump page table entries
         *
         * We dump each pmeg in order (by segment number).  Since the MMU
         * automatically maps the given virtual segment to a pmeg we must
         * iterate over the segments by incrementing an unused segment slot
         * in the MMU.  This fixed segment number is used in the virtual
         * address argument to getpte().
         */

        /*
         * Go through the pmegs and dump each one.
         */
        for (pmeg = 0; pmeg <= seginval; ++pmeg) {
                int va = 0;

                setsegmap(va, pmeg);
                i = NPTESG;
                do {
                        int pte = getpte4(va);
                        EXPEDITE(&pte, sizeof(pte));
                        va += NBPG;
                } while (--i > 0);
        }
        setsegmap(0, seginval);
#endif

out:
        if (bp != buffer)
                error = (*dump)(dumpdev, blkno++, (void *)buffer, dbtob(1));

        return (error);
}

/*
 * Helper function for debuggers.
 */
void
pmap_writetext(unsigned char *dst, int ch)
{
        int s, pte0, pte, ctx;
        vaddr_t va;

        s = splvm();
        va = (unsigned long)dst & (~PGOFSET);
        cache_flush(dst, 1);

        ctx = getcontext();
        setcontext(0);

#if defined(SUN4M) || defined(SUN4D)
        if (CPU_HAS_SRMMU) {
                pte0 = getpte4m(va);
                if ((pte0 & SRMMU_TETYPE) != SRMMU_TEPTE) {
                        goto out;
                }
                pte = pte0 | PPROT_WRITE;
                setpte4m(va, pte);
                *dst = (unsigned char)ch;
                setpte4m(va, pte0);

        }
#endif
#if defined(SUN4) || defined(SUN4C)
        if (CPU_ISSUN4C || CPU_ISSUN4) {
                pte0 = getpte4(va);
                if ((pte0 & PG_V) == 0) {
                        goto out;
                }
                pte = pte0 | PG_W;
                setpte4(va, pte);
                *dst = (unsigned char)ch;
                setpte4(va, pte0);
        }
#endif
        cache_flush(dst, 1);

out:
        setcontext(ctx);
        splx(s);
}

#ifdef EXTREME_DEBUG

void debug_pagetables(void);
void print_fe_map(void);

static void test_region(int, int, int);


void
debug_pagetables(void)
{
        struct promvec *promvec = romp;
        int *regtbl;
        int te;
        int i;

        printf("\nncontext=%d. ", ncontext);
        printf("Context table is at va %p. Level 0 PTP: 0x%x\n",
               cpuinfo.ctx_tbl, cpuinfo.ctx_tbl[0]);
        printf("Context 0 region table is at va %p, pa 0x%x. Contents:\n",
               pmap_kernel()->pm_reg_ptps[0], pmap_kernel()->pm_reg_ptps_pa[0]);

        regtbl = pmap_kernel()->pm_reg_ptps[0];

        printf("PROM vector is at %p\n", promvec);
        printf("PROM reboot routine is at %p\n", promvec->pv_reboot);
        printf("PROM abort routine is at %p\n", promvec->pv_abort);
        printf("PROM halt routine is at %p\n", promvec->pv_halt);

        printf("Testing region 0xfe: ");
        test_region(0xfe,0,16*1024*1024);
        printf("Testing region 0xff: ");
        test_region(0xff,0,16*1024*1024);
        printf("Testing kernel region 0x%x: ", VA_VREG(KERNBASE));
        test_region(VA_VREG(KERNBASE), 4096, avail_start);
        cngetc();

        for (i = 0; i < SRMMU_L1SIZE; i++) {
                te = regtbl[i];
                if ((te & SRMMU_TETYPE) == SRMMU_TEINVALID)
                    continue;
                printf("Region 0x%x: PTE=0x%x <%s> L2PA=0x%x kernL2VA=%p\n",
                       i, te, ((te & SRMMU_TETYPE) == SRMMU_TEPTE ? "pte" :
                               ((te & SRMMU_TETYPE) == SRMMU_TEPTD ? "ptd" :
                                ((te & SRMMU_TETYPE) == SRMMU_TEINVALID ?
                                 "invalid" : "reserved"))),
                       (te & ~0x3) << SRMMU_PPNPASHIFT,
                       pmap_kernel()->pm_regmap[i].rg_seg_ptps);
        }
        printf("Press q to halt...\n");
        if (cngetc()=='q')
            callrom();
}

static u_int
VA2PAsw(int ctx, void *addr, int *pte)
{
        int *curtbl;
        int curpte;

#ifdef EXTREME_EXTREME_DEBUG
        printf("Looking up addr 0x%x in context 0x%x\n",addr,ctx);
#endif
        /* L0 */
        *pte = curpte = cpuinfo.ctx_tbl[ctx];
#ifdef EXTREME_EXTREME_DEBUG
        printf("Got L0 pte 0x%x\n",pte);
#endif
        if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE) {
                return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
                        ((u_int)addr & 0xffffffff));
        }
        if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) {
                printf("Bad context table entry 0x%x for context 0x%x\n",
                       curpte, ctx);
                return 0;
        }
        /* L1 */
        curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */
        *pte = curpte = curtbl[VA_VREG(addr)];
#ifdef EXTREME_EXTREME_DEBUG
        printf("L1 table at 0x%x.\nGot L1 pte 0x%x\n",curtbl,curpte);
#endif
        if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE)
            return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
                    ((u_int)addr & 0xffffff));
        if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) {
                printf("Bad region table entry 0x%x for region 0x%x\n",
                       curpte, VA_VREG(addr));
                return 0;
        }
        /* L2 */
        curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */
        *pte = curpte = curtbl[VA_VSEG(addr)];
#ifdef EXTREME_EXTREME_DEBUG
        printf("L2 table at 0x%x.\nGot L2 pte 0x%x\n",curtbl,curpte);
#endif
        if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE)
            return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
                    ((u_int)addr & 0x3ffff));
        if ((curpte & SRMMU_TETYPE) != SRMMU_TEPTD) {
                printf("Bad segment table entry 0x%x for reg 0x%x, seg 0x%x\n",
                       curpte, VA_VREG(addr), VA_VSEG(addr));
                return 0;
        }
        /* L3 */
        curtbl = (int *)(((curpte & ~0x3) << 4) | KERNBASE); /* correct for krn */
        *pte = curpte = curtbl[VA_VPG(addr)];
#ifdef EXTREME_EXTREME_DEBUG
        printf("L3 table at %p.\nGot L3 pte 0x%x\n", curtbl, curpte);
#endif
        if ((curpte & SRMMU_TETYPE) == SRMMU_TEPTE)
            return (((curpte & SRMMU_PPNMASK) << SRMMU_PPNPASHIFT) |
                    ((u_int)addr & 0xfff));
        else {
                printf("Bad L3 pte 0x%x for reg 0x%x, seg 0x%x, pg 0x%x\n",
                       curpte, VA_VREG(addr), VA_VSEG(addr), VA_VPG(addr));
                return 0;
        }
        printf("Bizarreness with address %p!\n", addr);
}

static void
test_region(int reg, int start, int stop)
{
        int i;
        int addr;
        int pte;
        int ptesw;
/*      int cnt=0;
*/

        for (i = start; i < stop; i += NBPG) {
                addr = (reg << RGSHIFT) | i;
                pte = lda(((u_int)(addr)) | ASI_SRMMUFP_LN, ASI_SRMMUFP);
                if (pte) {
/*                      printf("Valid address 0x%x\n",addr);
                        if (++cnt == 20) {
                                cngetc();
                                cnt = 0;
                        }
*/
                        if (VA2PA((void *)addr) != VA2PAsw(0, (void *)addr, &ptesw)) {
                                printf("Mismatch at address 0x%x.\n", addr);
                                if (cngetc() == 'q')
                                        break;
                        }
                        if (reg == VA_VREG(KERNBASE))
                                /* kernel permissions are different */
                                continue;
                        if ((pte & SRMMU_PROT_MASK) != (ptesw & SRMMU_PROT_MASK)) {
                                printf("Mismatched protections at address "
                                       "0x%x; pte=0x%x, ptesw=0x%x\n",
                                       addr, pte, ptesw);
                                if (cngetc() == 'q')
                                        break;
                        }
                }
        }
        printf("done.\n");
}


void
print_fe_map(void)
{
        u_int i, pte;

        printf("map of region 0xfe:\n");
        for (i = 0xfe000000; i < 0xff000000; i += 4096) {
                if (((pte = getpte4m(i)) & SRMMU_TETYPE) != SRMMU_TEPTE)
                    continue;
                printf("0x%x -> 0x%x%x (pte 0x%x)\n", i, pte >> 28,
                       (pte & ~0xff) << 4, pte);
        }
        printf("done\n");
}
#endif /* EXTREME_DEBUG */

#ifdef DDB
int pmap_dump(struct pmap *pm);

int
pmap_dump(struct pmap *pm)
{
        int startvr, endvr, vr, vs, i, n;
        struct regmap *rp;
        struct segmap *sp;

        if (pm == NULL)
                pm = pmap_kernel();

        if (pm == pmap_kernel()) {
                startvr = NUREG;
                endvr = 256;
        } else {
                startvr = 0;
                endvr = NUREG;
        }

        for (vr = startvr; vr < endvr; vr++) {
                rp = &pm->pm_regmap[vr];
                if (rp->rg_nsegmap == 0)
                        continue;
                printf("vr %d: %d segments", vr, rp->rg_nsegmap);
                if (rp->rg_segmap == NULL) {
                        printf("[no segments]\n");
                        continue;
                }
                for (vs = 0; vs < NSEGRG; vs++) {
                        sp = &rp->rg_segmap[vs];
                        if (sp->sg_npte == 0)
                                continue;
                        if ((vs & 3) == 0)
                                printf("\n   ");
                        printf(" %d: n %d w %d p %d,", vs,
                                sp->sg_npte, sp->sg_nwired, sp->sg_pmeg);
                        if (sp->sg_pte == NULL) {
                                printf("[no ptes]");
                                continue;
                        }
                        for (n = 0, i = 0; i < NPTESG; i++) {
                                if (CPU_HAS_SUNMMU && sp->sg_pte[i] & PG_WIRED)
                                        n++;
                                if (CPU_HAS_SRMMU && sp->sg_wiremap & (1 << i))
                                        n++;
                        }
                        if (n != sp->sg_nwired)
                                printf("[wired count %d]", n);
                }
                printf("\n");
        }

        return (0);
}
#endif /* DDB */