/*      $NetBSD: pmap.c,v 1.158 2026/05/21 10:07:02 skrll Exp $ */

/*
 * Copyright (c) 2017 Ryo Shimizu
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.158 2026/05/21 10:07:02 skrll Exp $");

#include "opt_arm_debug.h"
#include "opt_cpuoptions.h"
#include "opt_ddb.h"
#include "opt_efi.h"
#include "opt_modular.h"
#include "opt_multiprocessor.h"
#include "opt_pmap.h"
#include "opt_uvmhist.h"

#include <sys/param.h>
#include <sys/types.h>

#include <sys/asan.h>
#include <sys/atomic.h>
#include <sys/cpu.h>
#include <sys/kmem.h>
#include <sys/vmem.h>

#include <uvm/uvm.h>
#include <uvm/pmap/pmap_pvt.h>

#include <arm/cpufunc.h>

#include <aarch64/pmap.h>
#include <aarch64/pte.h>
#include <aarch64/armreg.h>
#include <aarch64/locore.h>
#include <aarch64/machdep.h>
#ifdef DDB
#include <aarch64/db_machdep.h>
#include <ddb/db_access.h>
#endif

#include <arm/cpufunc.h>

//#define PMAP_PV_DEBUG

#ifdef VERBOSE_INIT_ARM
#define VPRINTF(...)    printf(__VA_ARGS__)
#else
#define VPRINTF(...)    __nothing
#endif

#ifdef UVMHIST

#ifndef UVMHIST_PMAPHIST_SIZE
#define UVMHIST_PMAPHIST_SIZE   (1024 * 4)
#endif

struct kern_history_ent pmaphistbuf[UVMHIST_PMAPHIST_SIZE];
UVMHIST_DEFINE(pmaphist) = UVMHIST_INITIALIZER(pmaphist, pmaphistbuf);

static void
pmap_hist_init(void)
{
        static bool inited = false;
        if (inited == false) {
                UVMHIST_LINK_STATIC(pmaphist);
                inited = true;
        }
}
#define PMAP_HIST_INIT()        pmap_hist_init()

#else /* UVMHIST */

#define PMAP_HIST_INIT()        ((void)0)

#endif /* UVMHIST */


#ifdef PMAPCOUNTERS
#define PMAP_COUNT(name)                (pmap_evcnt_##name.ev_count++ + 0)
#define PMAP_COUNTER(name, desc)                                        \
        struct evcnt pmap_evcnt_##name =                                \
            EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", desc);     \
        EVCNT_ATTACH_STATIC(pmap_evcnt_##name)

PMAP_COUNTER(pdp_alloc_boot, "page table page allocate (uvm_pageboot_alloc)");
PMAP_COUNTER(pdp_alloc, "page table page allocate (uvm_pagealloc)");
PMAP_COUNTER(pdp_free, "page table page free (uvm_pagefree)");

PMAP_COUNTER(pv_enter, "pv_entry fill");
PMAP_COUNTER(pv_remove_dyn, "pv_entry free and unlink dynamic");
PMAP_COUNTER(pv_remove_emb, "pv_entry clear embedded");
PMAP_COUNTER(pv_remove_nopv, "no pv_entry found when removing pv");

PMAP_COUNTER(activate, "pmap_activate call");
PMAP_COUNTER(deactivate, "pmap_deactivate call");
PMAP_COUNTER(create, "pmap_create call");
PMAP_COUNTER(destroy, "pmap_destroy call");

PMAP_COUNTER(page_protect, "pmap_page_protect call");
PMAP_COUNTER(protect, "pmap_protect call");
PMAP_COUNTER(protect_remove_fallback, "pmap_protect with no-read");
PMAP_COUNTER(protect_none, "pmap_protect non-exists pages");
PMAP_COUNTER(protect_managed, "pmap_protect managed pages");
PMAP_COUNTER(protect_unmanaged, "pmap_protect unmanaged pages");
PMAP_COUNTER(protect_pvmanaged, "pmap_protect pv-tracked unmanaged pages");

PMAP_COUNTER(clear_modify, "pmap_clear_modify call");
PMAP_COUNTER(clear_modify_pages, "pmap_clear_modify pages");
PMAP_COUNTER(clear_reference, "pmap_clear_reference call");
PMAP_COUNTER(clear_reference_pages, "pmap_clear_reference pages");

PMAP_COUNTER(fixup_referenced, "page reference emulations");
PMAP_COUNTER(fixup_modified, "page modification emulations");

PMAP_COUNTER(kern_mappings_bad, "kernel pages mapped (bad color)");
PMAP_COUNTER(kern_mappings_bad_wired, "kernel pages mapped (wired bad color)");
PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color, not wired)");
PMAP_COUNTER(user_mappings_bad_wired, "user pages mapped (bad color, wired)");
PMAP_COUNTER(kern_mappings, "kernel pages mapped");
PMAP_COUNTER(user_mappings, "user pages mapped");
PMAP_COUNTER(user_mappings_changed, "user mapping changed");
PMAP_COUNTER(kern_mappings_changed, "kernel mapping changed");
PMAP_COUNTER(uncached_mappings, "uncached pages mapped");
PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped");
PMAP_COUNTER(pvmanaged_mappings, "pv-tracked unmanaged pages mapped");
PMAP_COUNTER(managed_mappings, "managed pages mapped");
PMAP_COUNTER(mappings, "pages mapped (including remapped)");
PMAP_COUNTER(remappings, "pages remapped");

PMAP_COUNTER(pv_entry_cannotalloc, "pv_entry allocation failure");

PMAP_COUNTER(unwire, "pmap_unwire call");
PMAP_COUNTER(unwire_failure, "pmap_unwire failure");

#else /* PMAPCOUNTERS */
#define PMAP_COUNT(name)                __nothing
#endif /* PMAPCOUNTERS */

/*
 * invalidate TLB entry for ASID and VA.
 */
#define AARCH64_TLBI_BY_ASID_VA(asid, va)                       \
        do {                                                    \
                if ((asid) == 0)                                \
                        aarch64_tlbi_by_va((va));               \
                else                                            \
                        aarch64_tlbi_by_asid_va((asid), (va));  \
        } while (0/*CONSTCOND*/)

/*
 * require access permission in pte to invalidate instruction cache.
 * change the pte to be accessible temporarily before cpu_icache_sync_range().
 * this macro modifies PTE (*ptep). need to update PTE after this.
 */
#define PTE_ICACHE_SYNC_PAGE(pte, ptep, asid, va)                       \
        do {                                                            \
                atomic_swap_64((ptep), (pte) | LX_BLKPAG_AF);           \
                AARCH64_TLBI_BY_ASID_VA((asid), (va));                  \
                cpu_icache_sync_range((va), PAGE_SIZE);                 \
        } while (0/*CONSTCOND*/)

#define VM_PAGE_TO_PP(pg)       (&(pg)->mdpage.mdpg_pp)

#define L3INDEXMASK     (L3_SIZE * Ln_ENTRIES - 1)
#define PDPSWEEP_TRIGGER        512

static pt_entry_t *_pmap_pte_lookup_l3(struct pmap *, vaddr_t);
static pt_entry_t *_pmap_pte_lookup_bs(struct pmap *, vaddr_t, vsize_t *);
static pt_entry_t _pmap_pte_adjust_prot(pt_entry_t, vm_prot_t, vm_prot_t, bool);
static pt_entry_t _pmap_pte_adjust_cacheflags(pt_entry_t, u_int);
static void _pmap_remove(struct pmap *, vaddr_t, vaddr_t, bool,
    struct pv_entry **);
static int _pmap_enter(struct pmap *, vaddr_t, paddr_t, vm_prot_t, u_int, bool);
static int _pmap_get_pdp(struct pmap *, vaddr_t, bool, int, paddr_t *,
    struct vm_page **);

static struct pmap kernel_pmap __cacheline_aligned;
struct pmap * const kernel_pmap_ptr = &kernel_pmap;

#if defined(EFI_RUNTIME)
static struct pmap efirt_pmap __cacheline_aligned;

pmap_t
pmap_efirt(void)
{
        return &efirt_pmap;
}
#endif

static vaddr_t pmap_maxkvaddr;

vaddr_t virtual_avail, virtual_end;
vaddr_t virtual_devmap_addr;

static struct pool_cache _pmap_cache;
static struct pool_cache _pmap_pv_pool;

/* Set to LX_BLKPAG_GP if supported. */
uint64_t pmap_attr_gp = 0;

static inline void
pmap_pv_lock(struct pmap_page *pp)
{

        mutex_enter(&pp->pp_pvlock);
}

static inline void
pmap_pv_unlock(struct pmap_page *pp)
{

        mutex_exit(&pp->pp_pvlock);
}


static inline void
pm_lock(struct pmap *pm)
{
        mutex_enter(&pm->pm_lock);
}

static inline void
pm_unlock(struct pmap *pm)
{
        mutex_exit(&pm->pm_lock);
}

static bool
pm_reverse_lock(struct pmap *pm, struct pmap_page *pp)
{

        KASSERT(mutex_owned(&pp->pp_pvlock));

        if (__predict_true(mutex_tryenter(&pm->pm_lock)))
                return true;

        if (pm != pmap_kernel())
                pmap_reference(pm);
        mutex_exit(&pp->pp_pvlock);
        mutex_enter(&pm->pm_lock);
        /* nothing, just wait for lock */
        mutex_exit(&pm->pm_lock);
        if (pm != pmap_kernel())
                pmap_destroy(pm);
        mutex_enter(&pp->pp_pvlock);
        return false;
}

static inline struct pmap_page *
phys_to_pp(paddr_t pa)
{
        struct vm_page *pg;

        pg = PHYS_TO_VM_PAGE(pa);
        if (pg != NULL)
                return VM_PAGE_TO_PP(pg);

#ifdef __HAVE_PMAP_PV_TRACK
        return pmap_pv_tracked(pa);
#else
        return NULL;
#endif /* __HAVE_PMAP_PV_TRACK */
}

#define IN_RANGE(va, sta, end)  (((sta) <= (va)) && ((va) < (end)))

#define IN_DIRECTMAP_ADDR(va)   \
        IN_RANGE((va), AARCH64_DIRECTMAP_START, AARCH64_DIRECTMAP_END)

#define PMAP_EFIVA_P(va) \
     IN_RANGE((va), EFI_RUNTIME_VA, EFI_RUNTIME_VA + EFI_RUNTIME_SIZE)

#ifdef MODULAR
#define IN_MODULE_VA(va)        IN_RANGE((va), module_start, module_end)
#else
#define IN_MODULE_VA(va)        false
#endif

#ifdef DIAGNOSTIC

#define KERNEL_ADDR_P(va)                                               \
    (IN_RANGE((va), VM_MIN_KERNEL_ADDRESS,  VM_MAX_KERNEL_ADDRESS) ||   \
     PMAP_EFIVA_P(va))

#define KASSERT_PM_ADDR(pm, va)                                         \
    do {                                                                \
        int space = aarch64_addressspace(va);                           \
        if ((pm) == pmap_kernel()) {                                    \
                KASSERTMSG(space == AARCH64_ADDRSPACE_UPPER,            \
                    "%s: kernel pm %p: va=%016lx"                       \
                    " is out of upper address space",                   \
                    __func__, (pm), (va));                              \
                KASSERTMSG(KERNEL_ADDR_P(va),                           \
                    "%s: kernel pm %p: va=%016lx"                       \
                    " is not kernel address",                           \
                    __func__, (pm), (va));                              \
        } else {                                                        \
                KASSERTMSG(space == AARCH64_ADDRSPACE_LOWER,            \
                    "%s: user pm %p: va=%016lx"                         \
                    " is out of lower address space",                   \
                    __func__, (pm), (va));                              \
                KASSERTMSG(IN_RANGE((va),                               \
                    VM_MIN_ADDRESS, VM_MAX_ADDRESS),                    \
                    "%s: user pm %p: va=%016lx"                         \
                    " is not user address",                             \
                    __func__, (pm), (va));                              \
        }                                                               \
    } while (0 /* CONSTCOND */)
#else /* DIAGNOSTIC */
#define KASSERT_PM_ADDR(pm,va)
#endif /* DIAGNOSTIC */


vsize_t
pmap_kenter_range(vaddr_t va, paddr_t pa, vsize_t size,
    vm_prot_t prot, u_int flags)
{
        pt_entry_t attr;
        vsize_t resid = round_page(size);

        attr = _pmap_pte_adjust_prot(0, prot, VM_PROT_ALL, false);
        attr = _pmap_pte_adjust_cacheflags(attr, flags);
        pmapboot_enter_range(va, pa, resid, attr, printf);

        return resid;
}


void
pmap_bootstrap(vaddr_t vstart, vaddr_t vend)
{
        struct pmap *kpm;
        pd_entry_t *l0;
        paddr_t l0pa;

        PMAP_HIST_INIT();       /* init once */

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "vstart=%#jx vend=%#jx", (uintptr_t)vstart,
            (uintptr_t)vend, 0, 0);

        uvmexp.ncolors = aarch64_cache_vindexsize / PAGE_SIZE;

        /* devmap already uses last of va? */
        if (virtual_devmap_addr != 0 && virtual_devmap_addr < vend)
                vend = virtual_devmap_addr;

        virtual_avail = vstart;
        virtual_end = vend;
        pmap_maxkvaddr = vstart;

        l0pa = reg_ttbr1_el1_read();
        l0 = (void *)AARCH64_PA_TO_KVA(l0pa);

        pmap_tlb_info_init(&pmap_tlb0_info);

        memset(&kernel_pmap, 0, sizeof(kernel_pmap));

        kpm = pmap_kernel();
        struct pmap_asid_info * const pai = PMAP_PAI(kpm, cpu_tlb_info(ci));

        pai->pai_asid = KERNEL_PID;
        kpm->pm_refcnt = 1;
        kpm->pm_idlepdp = 0;
        kpm->pm_l0table = l0;
        kpm->pm_l0table_pa = l0pa;
        kpm->pm_onproc = kcpuset_running;
        kpm->pm_active = kcpuset_running;
        kpm->pm_activated = true;
        LIST_INIT(&kpm->pm_vmlist);
        LIST_INIT(&kpm->pm_pvlist);     /* not used for kernel pmap */
        mutex_init(&kpm->pm_lock, MUTEX_DEFAULT, IPL_NONE);

        CTASSERT(sizeof(kpm->pm_stats.wired_count) == sizeof(long));
        CTASSERT(sizeof(kpm->pm_stats.resident_count) == sizeof(long));

#if defined(EFI_RUNTIME)
        memset(&efirt_pmap, 0, sizeof(efirt_pmap));
        struct pmap * const efipm = &efirt_pmap;
        struct pmap_asid_info * const efipai = PMAP_PAI(efipm, cpu_tlb_info(ci));

        efipai->pai_asid = KERNEL_PID;
        efipm->pm_refcnt = 1;

        vaddr_t efi_l0va = uvm_pageboot_alloc(Ln_TABLE_SIZE);
        KASSERT((efi_l0va & PAGE_MASK) == 0);

        efipm->pm_l0table = (pd_entry_t *)efi_l0va;
        memset(efipm->pm_l0table, 0, Ln_TABLE_SIZE);

        efipm->pm_l0table_pa = AARCH64_KVA_TO_PA(efi_l0va);

        efipm->pm_activated = false;
        LIST_INIT(&efipm->pm_vmlist);
        LIST_INIT(&efipm->pm_pvlist);   /* not used for efi pmap */
        mutex_init(&efipm->pm_lock, MUTEX_DEFAULT, IPL_NONE);
#endif
}

#ifdef MULTIPROCESSOR
void
pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci)
{
        /* nothing */
}
#endif /* MULTIPROCESSOR */

static inline void
_pmap_adj_wired_count(struct pmap *pm, int adj)
{

        if (pm == pmap_kernel()) {
                atomic_add_long(&pm->pm_stats.wired_count, adj);
        } else {
                KASSERT(mutex_owned(&pm->pm_lock));
                pm->pm_stats.wired_count += adj;
        }
}

static inline void
_pmap_adj_resident_count(struct pmap *pm, int adj)
{

        if (pm == pmap_kernel()) {
                atomic_add_long(&pm->pm_stats.resident_count, adj);
        } else {
                KASSERT(mutex_owned(&pm->pm_lock));
                pm->pm_stats.resident_count += adj;
        }
}

inline static int
_pmap_color(vaddr_t addr)       /* or paddr_t */
{
        return (addr >> PGSHIFT) & (uvmexp.ncolors - 1);
}

static int
_pmap_pmap_ctor(void *arg, void *v, int flags)
{
        memset(v, 0, sizeof(struct pmap));
        return 0;
}

static int
_pmap_pv_ctor(void *arg, void *v, int flags)
{
        memset(v, 0, sizeof(struct pv_entry));
        return 0;
}

pd_entry_t *
pmap_l0table(struct pmap *pm)
{

        return pm->pm_l0table;
}

void
pmap_init(void)
{

        pool_cache_bootstrap(&_pmap_cache, sizeof(struct pmap),
            coherency_unit, 0, 0, "pmappl", NULL, IPL_NONE, _pmap_pmap_ctor,
            NULL, NULL);

        pool_cache_bootstrap(&_pmap_pv_pool, sizeof(struct pv_entry),
            32, 0, PR_LARGECACHE, "pvpl", NULL, IPL_NONE, _pmap_pv_ctor,
            NULL, NULL);

        pmap_tlb_info_evcnt_attach(&pmap_tlb0_info);
}

void
pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp)
{
        *vstartp = virtual_avail;
        *vendp = virtual_end;
}

vaddr_t
pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp)
{
        int npage;
        paddr_t pa;
        vaddr_t va;
        psize_t bank_npage;
        uvm_physseg_t bank;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "size=%llu, *vstartp=%llx, *vendp=%llx",
            size, *vstartp, *vendp, 0);

        size = round_page(size);
        npage = atop(size);

        for (bank = uvm_physseg_get_first(); uvm_physseg_valid_p(bank);
            bank = uvm_physseg_get_next(bank)) {

                bank_npage = uvm_physseg_get_avail_end(bank) -
                    uvm_physseg_get_avail_start(bank);
                if (npage <= bank_npage)
                        break;
        }

        if (!uvm_physseg_valid_p(bank)) {
                panic("%s: no memory", __func__);
        }

        /* Steal pages */
        pa = ptoa(uvm_physseg_get_avail_start(bank));
        va = AARCH64_PA_TO_KVA(pa);
        uvm_physseg_unplug(atop(pa), npage);

        for (; npage > 0; npage--, pa += PAGE_SIZE)
                pmap_zero_page(pa);

        return va;
}

void
pmap_reference(struct pmap *pm)
{
        atomic_inc_uint(&pm->pm_refcnt);
}

static paddr_t
pmap_alloc_pdp(struct pmap *pm, struct vm_page **pgp, int flags, bool waitok)
{
        paddr_t pa;
        struct vm_page *pg;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pm=%p, flags=%08x, waitok=%d",
            pm, flags, waitok, 0);

        if (uvm.page_init_done) {
                int aflags = ((flags & PMAP_CANFAIL) ? 0 : UVM_PGA_USERESERVE) |
                    UVM_PGA_ZERO;
 retry:
                pg = uvm_pagealloc(NULL, 0, NULL, aflags);
                if (pg == NULL) {
                        if (waitok) {
                                uvm_wait("pmap_alloc_pdp");
                                goto retry;
                        }
                        return POOL_PADDR_INVALID;
                }

                LIST_INSERT_HEAD(&pm->pm_vmlist, pg, pageq.list);
                pg->flags &= ~PG_BUSY;  /* never busy */
                pg->wire_count = 1;     /* max = 1 + Ln_ENTRIES = 513 */
                pa = VM_PAGE_TO_PHYS(pg);
                PMAP_COUNT(pdp_alloc);
                PMAP_PAGE_INIT(VM_PAGE_TO_PP(pg));
        } else {
                /* uvm_pageboot_alloc() returns a direct mapping address */
                pg = NULL;
                pa = AARCH64_KVA_TO_PA(
                    uvm_pageboot_alloc(Ln_TABLE_SIZE));
                PMAP_COUNT(pdp_alloc_boot);
        }
        if (pgp != NULL)
                *pgp = pg;

        UVMHIST_LOG(pmaphist, "pa=%llx, pg=%llx", pa, pg, 0, 0);

        return pa;
}

static void
pmap_free_pdp(struct pmap *pm, struct vm_page *pg)
{

        KASSERT(pm != pmap_kernel());
        KASSERT(VM_PAGE_TO_PP(pg)->pp_pv.pv_pmap == NULL);
        KASSERT(VM_PAGE_TO_PP(pg)->pp_pv.pv_next == NULL);

        LIST_REMOVE(pg, pageq.list);
        pg->wire_count = 0;
        uvm_pagefree(pg);
        PMAP_COUNT(pdp_free);
}

/* free empty page table pages */
static void
_pmap_sweep_pdp(struct pmap *pm)
{
        struct vm_page *pg, *tmp;
        pd_entry_t *ptep_in_parent, opte __diagused;
        paddr_t pa, pdppa;
        uint16_t wirecount __diagused;

        KASSERT(mutex_owned(&pm->pm_lock) || pm->pm_refcnt == 0);

        LIST_FOREACH_SAFE(pg, &pm->pm_vmlist, pageq.list, tmp) {
                if (pg->wire_count != 1)
                        continue;

                pa = VM_PAGE_TO_PHYS(pg);
                if (pa == pm->pm_l0table_pa)
                        continue;

                ptep_in_parent = VM_PAGE_TO_MD(pg)->mdpg_ptep_parent;
                if (ptep_in_parent == NULL) {
                        /* no parent */
                        pmap_free_pdp(pm, pg);
                        continue;
                }

                /* unlink from parent */
                opte = atomic_swap_64(ptep_in_parent, 0);
                KASSERT(lxpde_valid(opte));
                wirecount = --pg->wire_count; /* 1 -> 0 */
                KASSERT(wirecount == 0);
                pmap_free_pdp(pm, pg);

                /* L3->L2->L1. no need for L0 */
                pdppa = AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep_in_parent));
                if (pdppa == pm->pm_l0table_pa)
                        continue;

                pg = PHYS_TO_VM_PAGE(pdppa);
                KASSERT(pg != NULL);
                KASSERTMSG(pg->wire_count >= 1,
                    "wire_count=%d", pg->wire_count);
                /* decrement wire_count of parent */
                wirecount = --pg->wire_count;
                KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1),
                    "pm=%p, pg=%p, wire_count=%d",
                    pm, pg, pg->wire_count);
        }
        pm->pm_idlepdp = 0;
}

static void
_pmap_free_pdp_all(struct pmap *pm, bool free_l0)
{
        struct vm_page *pg, *pgtmp, *pg_reserve;

        pg_reserve = free_l0 ? NULL : PHYS_TO_VM_PAGE(pm->pm_l0table_pa);
        LIST_FOREACH_SAFE(pg, &pm->pm_vmlist, pageq.list, pgtmp) {
                if (pg == pg_reserve)
                        continue;
                pmap_free_pdp(pm, pg);
        }
}

vaddr_t
pmap_growkernel(vaddr_t maxkvaddr)
{
        struct pmap *pm = pmap_kernel();
        struct vm_page *pg;
        int error;
        vaddr_t va;
        paddr_t pa;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "maxkvaddr=%llx, pmap_maxkvaddr=%llx",
            maxkvaddr, pmap_maxkvaddr, 0, 0);

        mutex_enter(&pm->pm_lock);
        for (va = pmap_maxkvaddr & L2_FRAME; va <= maxkvaddr; va += L2_SIZE) {
                error = _pmap_get_pdp(pm, va, false, 0, &pa, &pg);
                if (error != 0) {
                        panic("%s: cannot allocate L3 table error=%d",
                            __func__, error);
                }
        }
        kasan_shadow_map((void *)pmap_maxkvaddr,
            (size_t)(va - pmap_maxkvaddr));
        pmap_maxkvaddr = va;
        mutex_exit(&pm->pm_lock);

        return va;
}

bool
pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap)
{

        return pmap_extract_coherency(pm, va, pap, NULL);
}

bool
pmap_extract_coherency(struct pmap *pm, vaddr_t va, paddr_t *pap,
    bool *coherencyp)
{
        pt_entry_t *ptep, pte;
        paddr_t pa;
        vsize_t blocksize = 0;
        int space;
        bool coherency, valid;
        extern char __kernel_text[];
        extern char _end[];

        coherency = false;

        space = aarch64_addressspace(va);
        if (pm == pmap_kernel()) {
                if (space != AARCH64_ADDRSPACE_UPPER)
                        return false;

                if (IN_RANGE(va, (vaddr_t)__kernel_text, (vaddr_t)_end)) {
                        /* kernel text/data/bss are definitely linear mapped */
                        pa = KERN_VTOPHYS(va);
                        goto mapped;
                } else if (IN_DIRECTMAP_ADDR(va)) {
                        /*
                         * also direct mapping is linear mapped, but areas that
                         * have no physical memory haven't been mapped.
                         * fast lookup by using the S1E1R/PAR_EL1 registers.
                         */
                        register_t s = daif_disable(DAIF_I | DAIF_F);
                        reg_s1e1r_write(va);
                        isb();
                        uint64_t par = reg_par_el1_read();
                        reg_daif_write(s);

                        if (par & PAR_F)
                                return false;
                        pa = (__SHIFTOUT(par, PAR_PA) << PAR_PA_SHIFT) +
                            (va & __BITS(PAR_PA_SHIFT - 1, 0));
                        goto mapped;
                }
        } else {
                if (space != AARCH64_ADDRSPACE_LOWER)
                        return false;
        }

        /*
         * other areas, it isn't able to examined using the PAR_EL1 register,
         * because the page may be in an access fault state due to
         * reference bit emulation.
         */
        if (pm != pmap_kernel())
                mutex_enter(&pm->pm_lock);
        ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
        valid = (ptep != NULL && lxpde_valid(pte = *ptep));
        if (pm != pmap_kernel())
                mutex_exit(&pm->pm_lock);

        if (!valid) {
                return false;
        }

        pa = lxpde_pa(pte) + (va & (blocksize - 1));

        switch (pte & LX_BLKPAG_ATTR_MASK) {
        case LX_BLKPAG_ATTR_NORMAL_NC:
        case LX_BLKPAG_ATTR_DEVICE_MEM:
        case LX_BLKPAG_ATTR_DEVICE_MEM_NP:
                coherency = true;
                break;
        }

 mapped:
        if (pap != NULL)
                *pap = pa;
        if (coherencyp != NULL)
                *coherencyp = coherency;
        return true;
}

paddr_t
vtophys(vaddr_t va)
{
        struct pmap *pm;
        paddr_t pa;

        /* even if TBI is disabled, AARCH64_ADDRTOP_TAG means KVA */
        if ((uint64_t)va & AARCH64_ADDRTOP_TAG)
                pm = pmap_kernel();
        else
                pm = curlwp->l_proc->p_vmspace->vm_map.pmap;

        if (pmap_extract(pm, va, &pa) == false)
                return VTOPHYS_FAILED;
        return pa;
}

/*
 * return pointer of the pte. regardess of whether the entry is valid or not.
 */
static pt_entry_t *
_pmap_pte_lookup_bs(struct pmap *pm, vaddr_t va, vsize_t *bs)
{
        pt_entry_t *ptep;
        pd_entry_t *l0, *l1, *l2, *l3;
        pd_entry_t pde;
        vsize_t blocksize;
        unsigned int idx;

        KASSERT(pm == pmap_kernel() || mutex_owned(&pm->pm_lock));

        /*
         * traverse L0 -> L1 -> L2 -> L3
         */
        blocksize = L0_SIZE;
        l0 = pm->pm_l0table;
        idx = l0pde_index(va);
        ptep = &l0[idx];
        pde = *ptep;
        if (!l0pde_valid(pde))
                goto done;

        blocksize = L1_SIZE;
        l1 = (pd_entry_t *)AARCH64_PA_TO_KVA(l0pde_pa(pde));
        idx = l1pde_index(va);
        ptep = &l1[idx];
        pde = *ptep;
        if (!l1pde_valid(pde) || l1pde_is_block(pde))
                goto done;

        blocksize = L2_SIZE;
        l2 = (pd_entry_t *)AARCH64_PA_TO_KVA(l1pde_pa(pde));
        idx = l2pde_index(va);
        ptep = &l2[idx];
        pde = *ptep;
        if (!l2pde_valid(pde) || l2pde_is_block(pde))
                goto done;

        blocksize = L3_SIZE;
        l3 = (pd_entry_t *)AARCH64_PA_TO_KVA(l2pde_pa(pde));
        idx = l3pte_index(va);
        ptep = &l3[idx];

 done:
        if (bs != NULL)
                *bs = blocksize;
        return ptep;
}

static pt_entry_t *
_pmap_pte_lookup_l3(struct pmap *pm, vaddr_t va)
{
        pt_entry_t *ptep;
        vsize_t blocksize = 0;

        ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
        if ((ptep != NULL) && (blocksize == L3_SIZE))
                return ptep;

        return NULL;
}

void
pmap_icache_sync_range(pmap_t pm, vaddr_t sva, vaddr_t eva)
{
        pt_entry_t *ptep = NULL, pte;
        vaddr_t va;
        vsize_t blocksize = 0;

        KASSERT_PM_ADDR(pm, sva);

        pm_lock(pm);

        for (va = sva; va < eva; va = (va + blocksize) & ~(blocksize - 1)) {
                /* va is belong to the same L3 table as before? */
                if ((blocksize == L3_SIZE) && ((va & L3INDEXMASK) != 0)) {
                        ptep++;
                } else {
                        ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
                        if (ptep == NULL)
                                break;
                }

                pte = *ptep;
                if (!lxpde_valid(pte))
                        continue;

                vaddr_t eob = (va + blocksize) & ~(blocksize - 1);
                vsize_t len = ulmin(eva, eob) - va;

                if (l3pte_readable(pte)) {
                        cpu_icache_sync_range(va, len);
                } else {
                        /*
                         * change to accessible temporarily
                         * to do cpu_icache_sync_range()
                         */
                        struct pmap_asid_info * const pai = PMAP_PAI(pm,
                            cpu_tlb_info(ci));

                        atomic_swap_64(ptep, pte | LX_BLKPAG_AF);
                        AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va);
                        cpu_icache_sync_range(va, len);
                        atomic_swap_64(ptep, pte);
                        AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va);
                }
        }

        pm_unlock(pm);
}

/*
 * Routine:     pmap_procwr
 *
 * Function:
 *      Synchronize caches corresponding to [addr, addr+len) in p.
 *
 */
void
pmap_procwr(struct proc *p, vaddr_t sva, int len)
{

        if (__predict_true(p == curproc))
                cpu_icache_sync_range(sva, len);
        else {
                struct pmap *pm = p->p_vmspace->vm_map.pmap;
                paddr_t pa;
                vaddr_t va, eva;
                int tlen;

                for (va = sva; len > 0; va = eva, len -= tlen) {
                        eva = uimin(va + len, trunc_page(va + PAGE_SIZE));
                        tlen = eva - va;
                        if (!pmap_extract(pm, va, &pa))
                                continue;
                        va = AARCH64_PA_TO_KVA(pa);
                        cpu_icache_sync_range(va, tlen);
                }
        }
}

static pt_entry_t
_pmap_pte_adjust_prot(pt_entry_t pte, vm_prot_t prot, vm_prot_t refmod,
    bool user)
{
        vm_prot_t masked;
        pt_entry_t xn;

        masked = prot & refmod;
        pte &= ~(LX_BLKPAG_OS_RWMASK | LX_BLKPAG_AF | LX_BLKPAG_DBM | LX_BLKPAG_AP);

        /*
         * keep actual prot in the pte as OS_{READ|WRITE} for ref/mod emulation,
         * and set the DBM bit for HAFDBS if it has write permission.
         */
        pte |= LX_BLKPAG_OS_READ;       /* a valid pte can always be readable */
        if (prot & VM_PROT_WRITE)
                pte |= LX_BLKPAG_OS_WRITE | LX_BLKPAG_DBM;

        switch (masked & (VM_PROT_READ | VM_PROT_WRITE)) {
        case 0:
        default:
                /*
                 * it cannot be accessed because there is no AF bit,
                 * but the AF bit will be added by fixup() or HAFDBS.
                 */
                pte |= LX_BLKPAG_AP_RO;
                break;
        case VM_PROT_READ:
                /*
                 * as it is RO, it cannot be written as is,
                 * but it may be changed to RW by fixup() or HAFDBS.
                 */
                pte |= LX_BLKPAG_AF;
                pte |= LX_BLKPAG_AP_RO;
                break;
        case VM_PROT_WRITE:
        case VM_PROT_READ | VM_PROT_WRITE:
                /* fully readable and writable */
                pte |= LX_BLKPAG_AF;
                pte |= LX_BLKPAG_AP_RW;
                break;
        }

        /* executable for kernel or user? first set never exec both */
        pte |= (LX_BLKPAG_UXN | LX_BLKPAG_PXN);
        /* and either to executable */
        xn = user ? LX_BLKPAG_UXN : LX_BLKPAG_PXN;
        if (prot & VM_PROT_EXECUTE)
                pte &= ~xn;

        return pte;
}

static pt_entry_t
_pmap_pte_adjust_cacheflags(pt_entry_t pte, u_int flags)
{

        pte &= ~LX_BLKPAG_ATTR_MASK;

        switch (flags & (PMAP_CACHE_MASK | PMAP_DEV_MASK)) {
        case PMAP_DEV_NP ... PMAP_DEV_NP | PMAP_CACHE_MASK:
                pte |= LX_BLKPAG_ATTR_DEVICE_MEM_NP;    /* Device-nGnRnE */
                break;
        case PMAP_DEV ... PMAP_DEV | PMAP_CACHE_MASK:
                pte |= LX_BLKPAG_ATTR_DEVICE_MEM;       /* Device-nGnRE */
                break;
        case PMAP_NOCACHE:
        case PMAP_NOCACHE_OVR:
        case PMAP_WRITE_COMBINE:
                pte |= LX_BLKPAG_ATTR_NORMAL_NC;        /* only no-cache */
                break;
        case PMAP_WRITE_BACK:
        case 0:
        default:
                pte |= LX_BLKPAG_ATTR_NORMAL_WB;
                break;
        }

        return pte;
}

#ifdef ARMV81_HAFDBS
static inline void
_pmap_reflect_refmod_in_pp(pt_entry_t pte, struct pmap_page *pp)
{
        if (!lxpde_valid(pte))
                return;

        /*
         * In order to retain referenced/modified information,
         * it should be reflected from pte in the pmap_page.
         */
        if (pte & LX_BLKPAG_AF)
                pp->pp_pv.pv_va |= VM_PROT_READ;
        if ((pte & LX_BLKPAG_AP) == LX_BLKPAG_AP_RW)
                pp->pp_pv.pv_va |= VM_PROT_WRITE;
}
#endif

static struct pv_entry *
_pmap_remove_pv(struct pmap_page *pp, struct pmap *pm, vaddr_t va,
    pt_entry_t pte)
{
        struct pv_entry *pv, *ppv;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pp=%p, pm=%p, va=%llx, pte=%llx",
            pp, pm, va, pte);

        KASSERT(mutex_owned(&pm->pm_lock));     /* for pv_proc */
        KASSERT(mutex_owned(&pp->pp_pvlock));

#ifdef ARMV81_HAFDBS
        if (aarch64_hafdbs_enabled != ID_AA64MMFR1_EL1_HAFDBS_NONE)
                _pmap_reflect_refmod_in_pp(pte, pp);
#endif

        for (ppv = NULL, pv = &pp->pp_pv; pv != NULL; pv = pv->pv_next) {
                if (pv->pv_pmap == pm && trunc_page(pv->pv_va) == va) {
                        break;
                }
                ppv = pv;
        }

        if (pm != pmap_kernel() && pv != NULL)
                LIST_REMOVE(pv, pv_proc);

        if (ppv == NULL) {
                /* embedded in pmap_page */
                pv->pv_pmap = NULL;
                pv = NULL;
                PMAP_COUNT(pv_remove_emb);
        } else if (pv != NULL) {
                /* dynamically allocated */
                ppv->pv_next = pv->pv_next;
                PMAP_COUNT(pv_remove_dyn);
        } else {
                PMAP_COUNT(pv_remove_nopv);
        }

        return pv;
}

#if defined(PMAP_PV_DEBUG) || defined(DDB)

static char *
str_vmflags(uint32_t flags)
{
        static int idx = 0;
        static char buf[4][32]; /* XXX */
        char *p;

        p = buf[idx];
        idx = (idx + 1) & 3;

        p[0] = (flags & VM_PROT_READ) ? 'R' : '-';
        p[1] = (flags & VM_PROT_WRITE) ? 'W' : '-';
        p[2] = (flags & VM_PROT_EXECUTE) ? 'X' : '-';
        if (flags & PMAP_WIRED)
                memcpy(&p[3], ",WIRED\0", 7);
        else
                p[3] = '\0';

        return p;
}

void
pmap_db_mdpg_print(struct vm_page *pg, void (*pr)(const char *, ...) __printflike(1, 2))
{
        struct pmap_page *pp = VM_PAGE_TO_PP(pg);
        struct pv_entry *pv;
        int i, flags;

        i = 0;
        flags = pp->pp_pv.pv_va & (PAGE_SIZE - 1);

        pr("pp=%p\n", pp);
        pr(" pp flags=%08x %s\n", flags, str_vmflags(flags));

        for (pv = &pp->pp_pv; pv != NULL; pv = pv->pv_next) {
                if (pv->pv_pmap == NULL) {
                        KASSERT(pv == &pp->pp_pv);
                        continue;
                }
                struct pmap * const pm = pv->pv_pmap;
                struct pmap_asid_info * const pai = PMAP_PAI(pm,
                    cpu_tlb_info(ci));

                pr("  pv[%d] pv=%p\n", i, pv);
                pr("    pv[%d].pv_pmap = %p (asid=%d)\n", i, pm, pai->pai_asid);
                pr("    pv[%d].pv_va   = %016lx (color=%d)\n", i,
                    trunc_page(pv->pv_va), _pmap_color(pv->pv_va));
                pr("    pv[%d].pv_ptep = %p\n", i, pv->pv_ptep);
                i++;
        }
}
#endif /* PMAP_PV_DEBUG & DDB */

static int
_pmap_enter_pv(struct pmap_page *pp, struct pmap *pm, struct pv_entry **pvp,
    vaddr_t va, pt_entry_t *ptep, paddr_t pa, u_int flags)
{
        struct pv_entry *pv;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pp=%p, pm=%p, va=%llx, pa=%llx", pp, pm,
            va, pa);
        UVMHIST_LOG(pmaphist, "ptep=%p, flags=%08x", ptep, flags, 0, 0);

        KASSERT(mutex_owned(&pp->pp_pvlock));
        KASSERT(trunc_page(va) == va);

        /*
         * mapping cannot be already registered at this VA.
         */
        if (pp->pp_pv.pv_pmap == NULL) {
                /*
                 * claim pv_entry embedded in pmap_page.
                 * take care not to wipe out acc/mod flags.
                 */
                pv = &pp->pp_pv;
                pv->pv_va = (pv->pv_va & (PAGE_SIZE - 1)) | va;
        } else {
                /*
                 * create and link new pv.
                 * pv is already allocated at beginning of _pmap_enter().
                 */
                pv = *pvp;
                if (pv == NULL)
                        return ENOMEM;
                *pvp = NULL;
                pv->pv_next = pp->pp_pv.pv_next;
                pp->pp_pv.pv_next = pv;
                pv->pv_va = va;
        }
        pv->pv_pmap = pm;
        pv->pv_ptep = ptep;
        PMAP_COUNT(pv_enter);

        if (pm != pmap_kernel())
                LIST_INSERT_HEAD(&pm->pm_pvlist, pv, pv_proc);

#ifdef PMAP_PV_DEBUG
        printf("pv %p alias added va=%016lx -> pa=%016lx\n", pv, va, pa);
        pmap_db_mdpg_print(PHYS_TO_VM_PAGE(pa), printf);
#endif

        return 0;
}

void
pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
{

        _pmap_enter(pmap_kernel(), va, pa, prot, flags | PMAP_WIRED, true);
}

void
pmap_kremove(vaddr_t va, vsize_t size)
{
        struct pmap *kpm = pmap_kernel();

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "va=%llx, size=%llx", va, size, 0, 0);

        KDASSERT((va & PGOFSET) == 0);
        KDASSERT((size & PGOFSET) == 0);

        KDASSERT(!IN_DIRECTMAP_ADDR(va));
        KDASSERT(IN_RANGE(va, VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS));

        _pmap_remove(kpm, va, va + size, true, NULL);
}

static void
_pmap_protect_pv(struct pmap_page *pp, struct pv_entry *pv, vm_prot_t prot)
{
        pt_entry_t *ptep, pte;
        vm_prot_t pteprot;
        uint32_t mdattr;
        const bool user = (pv->pv_pmap != pmap_kernel());

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pp=%p, pv=%p, prot=%08x", pp, pv, prot, 0);

        KASSERT(mutex_owned(&pv->pv_pmap->pm_lock));

        ptep = pv->pv_ptep;
        pte = *ptep;

        /* get prot mask from pte */
        pteprot = VM_PROT_READ; /* a valid pte can always be readable */
        if ((pte & (LX_BLKPAG_OS_WRITE | LX_BLKPAG_DBM)) != 0)
                pteprot |= VM_PROT_WRITE;
        if (l3pte_executable(pte, user))
                pteprot |= VM_PROT_EXECUTE;

#ifdef ARMV81_HAFDBS
        if (aarch64_hafdbs_enabled != ID_AA64MMFR1_EL1_HAFDBS_NONE)
                _pmap_reflect_refmod_in_pp(pte, pp);
#endif
        /* get prot mask from referenced/modified */
        mdattr = pp->pp_pv.pv_va & (VM_PROT_READ | VM_PROT_WRITE);

        /* new prot = prot & pteprot & mdattr */
        pte = _pmap_pte_adjust_prot(pte, prot & pteprot, mdattr, user);
        atomic_swap_64(ptep, pte);

        struct pmap * const pm = pv->pv_pmap;
        struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));

        AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, trunc_page(pv->pv_va));
}

void
pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
{
        pt_entry_t *ptep = NULL, pte;
        vaddr_t va;
        vsize_t blocksize = 0;
        const bool user = (pm != pmap_kernel());

        KASSERT((prot & VM_PROT_READ) || !(prot & VM_PROT_WRITE));

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pm=%p, sva=%016lx, eva=%016lx, prot=%08x",
            pm, sva, eva, prot);

        KASSERT_PM_ADDR(pm, sva);
        KASSERT(!IN_DIRECTMAP_ADDR(sva));

        /* PROT_EXEC requires implicit PROT_READ */
        if (prot & VM_PROT_EXECUTE)
                prot |= VM_PROT_READ;

        if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
                PMAP_COUNT(protect_remove_fallback);
                pmap_remove(pm, sva, eva);
                return;
        }
        PMAP_COUNT(protect);

        KDASSERT((sva & PAGE_MASK) == 0);
        KDASSERT((eva & PAGE_MASK) == 0);

        pm_lock(pm);

        for (va = sva; va < eva; va = (va + blocksize) & ~(blocksize - 1)) {
#ifdef UVMHIST
                pt_entry_t opte;
#endif
                struct pmap_page *pp;
                uint32_t mdattr;
                bool executable;

                /* va is belong to the same L3 table as before? */
                if ((blocksize == L3_SIZE) && ((va & L3INDEXMASK) != 0))
                        ptep++;
                else
                        ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);

                pte = *ptep;
                if (!lxpde_valid(pte)) {
                        PMAP_COUNT(protect_none);
                        continue;
                }

                if ((pte & LX_BLKPAG_OS_WIRED) == 0) {
                        const paddr_t pa = lxpde_pa(pte);
                        struct vm_page *const pg = PHYS_TO_VM_PAGE(pa);

                        if (pg != NULL) {
                                pp = VM_PAGE_TO_PP(pg);
                                PMAP_COUNT(protect_managed);
                        } else {
#ifdef __HAVE_PMAP_PV_TRACK
                                pp = pmap_pv_tracked(pa);
#ifdef PMAPCOUNTERS
                                if (pp != NULL)
                                        PMAP_COUNT(protect_pvmanaged);
                                else
                                        PMAP_COUNT(protect_unmanaged);
#endif
#else
                                pp = NULL;
                                PMAP_COUNT(protect_unmanaged);
#endif /* __HAVE_PMAP_PV_TRACK */
                        }
                } else {        /* kenter */
                        pp = NULL;
                        PMAP_COUNT(protect_unmanaged);
                }

                if (pp != NULL) {
#ifdef ARMV81_HAFDBS
                        if (aarch64_hafdbs_enabled != ID_AA64MMFR1_EL1_HAFDBS_NONE)
                                _pmap_reflect_refmod_in_pp(pte, pp);
#endif
                        /* get prot mask from referenced/modified */
                        mdattr = pp->pp_pv.pv_va &
                            (VM_PROT_READ | VM_PROT_WRITE);
                } else {
                        /* unmanaged page */
                        mdattr = VM_PROT_ALL;
                }

#ifdef UVMHIST
                opte = pte;
#endif
                executable = l3pte_executable(pte, user);
                pte = _pmap_pte_adjust_prot(pte, prot, mdattr, user);

                struct pmap_asid_info * const pai = PMAP_PAI(pm,
                    cpu_tlb_info(ci));
                if (!executable && (prot & VM_PROT_EXECUTE)) {
                        /* non-exec -> exec */
                        UVMHIST_LOG(pmaphist, "icache_sync: "
                            "pm=%p, va=%016lx, pte: %016lx -> %016lx",
                            pm, va, opte, pte);

                        if (!l3pte_readable(pte)) {
                                PTE_ICACHE_SYNC_PAGE(pte, ptep, pai->pai_asid,
                                    va);
                                atomic_swap_64(ptep, pte);
                                AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va);
                        } else {
                                atomic_swap_64(ptep, pte);
                                AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va);
                                cpu_icache_sync_range(va, PAGE_SIZE);
                        }
                } else {
                        atomic_swap_64(ptep, pte);
                        AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va);
                }
        }

        pm_unlock(pm);
}

#if defined(EFI_RUNTIME)
void
pmap_activate_efirt(void)
{
        struct cpu_info *ci = curcpu();
        struct pmap *pm = &efirt_pmap;
        struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, " (pm=%#jx)", (uintptr_t)pm, 0, 0, 0);

        KASSERT(kpreempt_disabled());
        KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0);

        ci->ci_pmap_asid_cur = pai->pai_asid;
        UVMHIST_LOG(pmaphist, "setting asid to %#jx", pai->pai_asid, 0, 0, 0);

        pmap_md_asid_activate(pai->pai_asid, pm, NULL);
        pm->pm_activated = true;

        PMAP_COUNT(activate);
}
#endif

void
pmap_activate(struct lwp *l)
{
        struct pmap *pm = l->l_proc->p_vmspace->vm_map.pmap;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "lwp=%p (pid=%d, kernel=%u)", l,
            l->l_proc->p_pid, pm == pmap_kernel() ? 1 : 0, 0);

        KASSERT(kpreempt_disabled());
        KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0);

        if (pm == pmap_kernel())
                return;
        if (l != curlwp)
                return;

        KASSERT(pm->pm_l0table != NULL);

        /* this calls pmap_md_asid_activate */
        pmap_tlb_asid_acquire(pm, l);
        pm->pm_activated = true;

        PMAP_COUNT(activate);
}

#if defined(EFI_RUNTIME)
void
pmap_deactivate_efirt(void)
{
        struct cpu_info * const ci = curcpu();
        struct pmap * const pm = &efirt_pmap;

        UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);

        KASSERT(kpreempt_disabled());

        ci->ci_pmap_asid_cur = KERNEL_PID;
        pmap_md_asid_deactivate(pm);
        pm->pm_activated = false;

        KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0);

        PMAP_COUNT(deactivate);
}
#endif

void
pmap_deactivate(struct lwp *l)
{
        struct pmap *pm = l->l_proc->p_vmspace->vm_map.pmap;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "lwp=%p (pid=%d, (kernel=%u))", l,
            l->l_proc->p_pid, pm == pmap_kernel() ? 1 : 0, 0);

        KASSERT(kpreempt_disabled());

        struct cpu_info * const ci = curcpu();

        UVMHIST_LOG(pmaphist, "lwp=%p, asid=%d", l,
            PMAP_PAI(pm, cpu_tlb_info(ci))->pai_asid, 0, 0);

        ci->ci_pmap_asid_cur = KERNEL_PID;
        pmap_tlb_asid_deactivate(pm);
        pm->pm_activated = false;

        KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0);

        PMAP_COUNT(deactivate);
}

struct pmap *
pmap_create(void)
{
        struct pmap *pm;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLED(pmaphist);

        pm = pool_cache_get(&_pmap_cache, PR_WAITOK);
        memset(pm, 0, sizeof(*pm));
        pm->pm_refcnt = 1;
        pm->pm_idlepdp = 0;
        LIST_INIT(&pm->pm_vmlist);
        LIST_INIT(&pm->pm_pvlist);
        mutex_init(&pm->pm_lock, MUTEX_DEFAULT, IPL_NONE);

        kcpuset_create(&pm->pm_active, true);
        kcpuset_create(&pm->pm_onproc, true);

        pm->pm_l0table_pa = pmap_alloc_pdp(pm, NULL, 0, true);
        KASSERT(pm->pm_l0table_pa != POOL_PADDR_INVALID);
        pm->pm_l0table = (pd_entry_t *)AARCH64_PA_TO_KVA(pm->pm_l0table_pa);
        KASSERT(((vaddr_t)pm->pm_l0table & (PAGE_SIZE - 1)) == 0);

        UVMHIST_LOG(pmaphist, "pm=%p, pm_l0table=%016lx, pm_l0table_pa=%016lx",
            pm, pm->pm_l0table, pm->pm_l0table_pa, 0);

        PMAP_COUNT(create);
        return pm;
}

void
pmap_destroy(struct pmap *pm)
{
        unsigned int refcnt;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pm=%p, pm_l0table=%016lx, refcnt=%jd",
            pm, pm->pm_l0table, pm->pm_refcnt, 0);

        if (pm == NULL)
                return;

        if (pm == pmap_kernel())
                panic("cannot destroy kernel pmap");

        membar_release();
        refcnt = atomic_dec_uint_nv(&pm->pm_refcnt);
        if (refcnt > 0)
                return;
        membar_acquire();

        KASSERT(LIST_EMPTY(&pm->pm_pvlist));
        pmap_tlb_asid_release_all(pm);

        _pmap_free_pdp_all(pm, true);
        mutex_destroy(&pm->pm_lock);

        kcpuset_destroy(pm->pm_active);
        kcpuset_destroy(pm->pm_onproc);

        pool_cache_put(&_pmap_cache, pm);

        PMAP_COUNT(destroy);
}

static inline void
_pmap_pdp_setparent(struct pmap *pm, struct vm_page *pg, pt_entry_t *ptep)
{

        if ((pm != pmap_kernel()) && (pg != NULL)) {
                KASSERT(mutex_owned(&pm->pm_lock));
                VM_PAGE_TO_MD(pg)->mdpg_ptep_parent = ptep;
        }
}

/*
 * increment reference counter of the page descriptor page.
 * the reference counter should be equal to
 *  1 + num of valid entries the page has.
 */
static inline void
_pmap_pdp_addref(struct pmap *pm, paddr_t pdppa, struct vm_page *pdppg_hint)
{
        struct vm_page *pg;

        /* kernel L0-L3 pages will never be freed */
        if (pm == pmap_kernel())
                return;

#if defined(EFI_RUNTIME)
        /* EFI runtme L0-L3 pages will never be freed */
        if (pm == pmap_efirt())
                return;
#endif

        KASSERT(mutex_owned(&pm->pm_lock));

        /* no need for L0 page */
        if (pm->pm_l0table_pa == pdppa)
                return;

        pg = pdppg_hint;
        if (pg == NULL)
                pg = PHYS_TO_VM_PAGE(pdppa);
        KASSERT(pg != NULL);

        pg->wire_count++;

        KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1),
            "pg=%p, wire_count=%d", pg, pg->wire_count);
}

/*
 * decrement reference counter of the page descriptor page.
 * if reference counter is 1(=empty), pages will be freed, and return true.
 * otherwise return false.
 * kernel page, or L0 page descriptor page will be never freed.
 */
static bool
_pmap_pdp_delref(struct pmap *pm, paddr_t pdppa, bool do_free_pdp)
{
        struct vm_page *pg;
        bool removed;
        uint16_t wirecount;

        /* kernel L0-L3 pages will never be freed */
        if (pm == pmap_kernel())
                return false;

#if defined(EFI_RUNTIME)
        /* EFI runtme L0-L3 pages will never be freed */
        if (pm == pmap_efirt())
                return false;
#endif

        KASSERT(mutex_owned(&pm->pm_lock));

        /* no need for L0 page */
        if (pm->pm_l0table_pa == pdppa)
                return false;

        pg = PHYS_TO_VM_PAGE(pdppa);
        KASSERT(pg != NULL);

        wirecount = --pg->wire_count;

        if (!do_free_pdp) {
                /*
                 * pm_idlepdp is counted by only pmap_page_protect() with
                 * VM_PROT_NONE. it is not correct because without considering
                 * pmap_enter(), but useful hint to just sweep.
                 */
                if (wirecount == 1)
                        pm->pm_idlepdp++;
                return false;
        }

        /* if no reference, free pdp */
        removed = false;
        while (wirecount == 1) {
                pd_entry_t *ptep_in_parent, opte __diagused;
                ptep_in_parent = VM_PAGE_TO_MD(pg)->mdpg_ptep_parent;
                if (ptep_in_parent == NULL) {
                        /* no parent */
                        pmap_free_pdp(pm, pg);
                        removed = true;
                        break;
                }

                /* unlink from parent */
                opte = atomic_swap_64(ptep_in_parent, 0);
                KASSERT(lxpde_valid(opte));
                wirecount = atomic_add_32_nv(&pg->wire_count, -1); /* 1 -> 0 */
                KASSERT(wirecount == 0);
                pmap_free_pdp(pm, pg);
                removed = true;

                /* L3->L2->L1. no need for L0 */
                pdppa = AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep_in_parent));
                if (pdppa == pm->pm_l0table_pa)
                        break;

                pg = PHYS_TO_VM_PAGE(pdppa);
                KASSERT(pg != NULL);
                KASSERTMSG(pg->wire_count >= 1,
                    "wire_count=%d", pg->wire_count);
                /* decrement wire_count of parent */
                wirecount = atomic_add_32_nv(&pg->wire_count, -1);
                KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1),
                    "pm=%p, pg=%p, wire_count=%d",
                    pm, pg, pg->wire_count);
        }

        return removed;
}

/*
 * traverse L0 -> L1 -> L2 -> L3 table with growing pdp if needed.
 */
static int
_pmap_get_pdp(struct pmap *pm, vaddr_t va, bool kenter, int flags,
    paddr_t *pap, struct vm_page **pgp)
{
        pd_entry_t *l0, *l1, *l2;
        struct vm_page *pdppg, *pdppg0;
        paddr_t pdppa, pdppa0;
        unsigned int idx;
        pd_entry_t pde;

        KASSERT(kenter || mutex_owned(&pm->pm_lock));

        l0 = pm->pm_l0table;

        idx = l0pde_index(va);
        pde = l0[idx];
        if (!l0pde_valid(pde)) {
                KASSERTMSG(!kenter || IN_MODULE_VA(va) || PMAP_EFIVA_P(va),
                    "%s va %" PRIxVADDR, kenter ? "kernel" : "user", va);
                /* no need to increment L0 occupancy. L0 page never freed */
                pdppa = pmap_alloc_pdp(pm, &pdppg, flags, false);  /* L1 pdp */
                if (pdppa == POOL_PADDR_INVALID) {
                        return ENOMEM;
                }
                atomic_swap_64(&l0[idx], pdppa | L0_TABLE);
                _pmap_pdp_setparent(pm, pdppg, &l0[idx]);
        } else {
                pdppa = l0pde_pa(pde);
                pdppg = NULL;
        }
        l1 = (void *)AARCH64_PA_TO_KVA(pdppa);

        idx = l1pde_index(va);
        pde = l1[idx];
        if (!l1pde_valid(pde)) {
                KASSERTMSG(!kenter || IN_MODULE_VA(va) || PMAP_EFIVA_P(va),
                    "%s va %" PRIxVADDR, kenter ? "kernel" : "user", va);
                pdppa0 = pdppa;
                pdppg0 = pdppg;
                pdppa = pmap_alloc_pdp(pm, &pdppg, flags, false);  /* L2 pdp */
                if (pdppa == POOL_PADDR_INVALID) {
                        return ENOMEM;
                }
                atomic_swap_64(&l1[idx], pdppa | L1_TABLE);
                _pmap_pdp_addref(pm, pdppa0, pdppg0);   /* L1 occupancy++ */
                _pmap_pdp_setparent(pm, pdppg, &l1[idx]);
        } else {
                pdppa = l1pde_pa(pde);
                pdppg = NULL;
        }
        l2 = (void *)AARCH64_PA_TO_KVA(pdppa);

        idx = l2pde_index(va);
        pde = l2[idx];
        if (!l2pde_valid(pde)) {
                KASSERTMSG(!kenter || IN_MODULE_VA(va) || PMAP_EFIVA_P(va),
                    "%s va %" PRIxVADDR, kenter ? "kernel" : "user", va);
                pdppa0 = pdppa;
                pdppg0 = pdppg;
                pdppa = pmap_alloc_pdp(pm, &pdppg, flags, false);  /* L3 pdp */
                if (pdppa == POOL_PADDR_INVALID) {
                        return ENOMEM;
                }
                atomic_swap_64(&l2[idx], pdppa | L2_TABLE);
                _pmap_pdp_addref(pm, pdppa0, pdppg0);   /* L2 occupancy++ */
                _pmap_pdp_setparent(pm, pdppg, &l2[idx]);
        } else {
                pdppa = l2pde_pa(pde);
                pdppg = NULL;
        }
        *pap = pdppa;
        *pgp = pdppg;
        return 0;
}

static int
_pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot,
    u_int flags, bool kenter)
{
        struct vm_page *pdppg;
        struct pmap_page *pp, *opp, *pps[2];
        struct pv_entry *spv, *opv = NULL;
        pt_entry_t attr, pte, opte, *ptep;
        pd_entry_t *l3;
        paddr_t pdppa;
        uint32_t mdattr;
        unsigned int idx;
        int error = 0;
#if defined(EFI_RUNTIME)
        const bool efirt_p = pm == pmap_efirt();
#else
        const bool efirt_p = false;
#endif
        const bool kernel_p = pm == pmap_kernel();
        const bool user = !kernel_p && !efirt_p;
        bool need_sync_icache, need_enter_pv;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pm=%p, kentermode=%d", pm, kenter, 0, 0);
        UVMHIST_LOG(pmaphist, "va=%016lx, pa=%016lx, prot=%08x, flags=%08x",
            va, pa, prot, flags);

        KASSERT_PM_ADDR(pm, va);
        KASSERT(!IN_DIRECTMAP_ADDR(va));
        KASSERT((prot & VM_PROT_ALL) != VM_PROT_NONE);
        KASSERT(pa < AARCH64_MAX_PA);

#ifdef PMAPCOUNTERS
        PMAP_COUNT(mappings);
        if (_pmap_color(va) == _pmap_color(pa)) {
                if (user) {
                        PMAP_COUNT(user_mappings);
                } else {
                        PMAP_COUNT(kern_mappings);
                }
        } else if (flags & PMAP_WIRED) {
                if (user) {
                        PMAP_COUNT(user_mappings_bad_wired);
                } else {
                        PMAP_COUNT(kern_mappings_bad_wired);
                }
        } else {
                if (user) {
                        PMAP_COUNT(user_mappings_bad);
                } else {
                        PMAP_COUNT(kern_mappings_bad);
                }
        }
#endif

        if (kenter) {
                pp = NULL;
                spv = NULL;
                need_enter_pv = false;
        } else {
                struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
                if (pg != NULL) {
                        pp = VM_PAGE_TO_PP(pg);
                        PMAP_COUNT(managed_mappings);
                } else {
#ifdef __HAVE_PMAP_PV_TRACK
                        pp = pmap_pv_tracked(pa);
#ifdef PMAPCOUNTERS
                        if (pp != NULL)
                                PMAP_COUNT(pvmanaged_mappings);
                        else
                                PMAP_COUNT(unmanaged_mappings);
#endif
#else
                        pp = NULL;
                        PMAP_COUNT(unmanaged_mappings);
#endif /* __HAVE_PMAP_PV_TRACK */
                }

                if (pp != NULL) {
                        /*
                         * allocate pv in advance of pm_lock().
                         */
                        spv = pool_cache_get(&_pmap_pv_pool, PR_NOWAIT);
                        need_enter_pv = true;
                } else {
                        spv = NULL;
                        need_enter_pv = false;
                }

                pm_lock(pm);
                if (pm->pm_idlepdp >= PDPSWEEP_TRIGGER) {
                        _pmap_sweep_pdp(pm);
                }
        }

        /*
         * traverse L0 -> L1 -> L2 -> L3 table with growing pdp if needed.
         */
        error = _pmap_get_pdp(pm, va, kenter, flags, &pdppa, &pdppg);
        if (error != 0) {
                if (flags & PMAP_CANFAIL) {
                        goto fail0;
                }
                panic("%s: cannot allocate L3 table error=%d", __func__,
                    error);
        }

        l3 = (void *)AARCH64_PA_TO_KVA(pdppa);

        idx = l3pte_index(va);
        ptep = &l3[idx];        /* as PTE */
        opte = *ptep;
        need_sync_icache = (prot & VM_PROT_EXECUTE) && !efirt_p;

        /* for lock ordering for old page and new page */
        pps[0] = pp;
        pps[1] = NULL;

        /* remap? */
        if (l3pte_valid(opte)) {
                bool need_remove_pv;

                KASSERT(!kenter);       /* pmap_kenter_pa() cannot override */
                if (opte & LX_BLKPAG_OS_WIRED) {
                        _pmap_adj_wired_count(pm, -1);
                }
                _pmap_adj_resident_count(pm, -1);
#ifdef PMAPCOUNTERS
                PMAP_COUNT(remappings);
                if (user) {
                        PMAP_COUNT(user_mappings_changed);
                } else {
                        PMAP_COUNT(kern_mappings_changed);
                }
#endif
                UVMHIST_LOG(pmaphist,
                    "va=%016lx has already mapped."
                    " old-pa=%016lx new-pa=%016lx, old-pte=%016llx",
                    va, l3pte_pa(opte), pa, opte);

                if (pa == l3pte_pa(opte)) {
                        /* old and new pte have same pa, no need to update pv */
                        need_remove_pv = (pp == NULL);
                        need_enter_pv = false;
                        if (need_sync_icache && l3pte_executable(opte, user))
                                need_sync_icache = false;
                } else {
                        need_remove_pv = true;
                }

                if (need_remove_pv &&
                    ((opp = phys_to_pp(l3pte_pa(opte))) != NULL)) {
                        /*
                         * need to lock both pp and opp(old pp)
                         * against deadlock, and 'pp' maybe NULL.
                         */
                        if (pp < opp) {
                                pps[0] = pp;
                                pps[1] = opp;
                        } else {
                                pps[0] = opp;
                                pps[1] = pp;
                        }
                        if (pps[0] != NULL)
                                pmap_pv_lock(pps[0]);
                        if (pps[1] != NULL)
                                pmap_pv_lock(pps[1]);
                        opv = _pmap_remove_pv(opp, pm, va, opte);
                } else {
                        if (pp != NULL)
                                pmap_pv_lock(pp);
                }
                opte = atomic_swap_64(ptep, 0);
        } else {
                if (pp != NULL)
                        pmap_pv_lock(pp);
        }

        if (!l3pte_valid(opte))
                _pmap_pdp_addref(pm, pdppa, pdppg);     /* L3 occupancy++ */

        /*
         * read permission is treated as an access permission internally.
         * require to add PROT_READ even if only PROT_WRITE or PROT_EXEC
         */
        if (prot & (VM_PROT_WRITE | VM_PROT_EXECUTE))
                prot |= VM_PROT_READ;
        if (flags & (VM_PROT_WRITE | VM_PROT_EXECUTE))
                flags |= VM_PROT_READ;

        mdattr = VM_PROT_READ | VM_PROT_WRITE;
        if (need_enter_pv) {
                KASSERT(!kenter);
                error = _pmap_enter_pv(pp, pm, &spv, va, ptep, pa, flags);
                if (error != 0) {
                        /*
                         * If pmap_enter() fails,
                         * it must not leave behind an existing pmap entry.
                         */
                        if (lxpde_valid(opte)) {
                                KASSERT((vaddr_t)l3 == trunc_page((vaddr_t)ptep));
                                _pmap_pdp_delref(pm, AARCH64_KVA_TO_PA((vaddr_t)l3),
                                    true);
                                struct pmap_asid_info * const pai = PMAP_PAI(pm,
                                    cpu_tlb_info(ci));

                                AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va);
                        }
                        PMAP_COUNT(pv_entry_cannotalloc);
                        if (flags & PMAP_CANFAIL)
                                goto fail1;
                        panic("pmap_enter: failed to allocate pv_entry");
                }
        }

        if (pp != NULL) {
                /* update referenced/modified flags */
                KASSERT(!kenter);
                pp->pp_pv.pv_va |= (flags & (VM_PROT_READ | VM_PROT_WRITE));
                mdattr &= (uint32_t)pp->pp_pv.pv_va;
        }

#ifdef PMAPCOUNTERS
        switch (flags & PMAP_CACHE_MASK) {
        case PMAP_NOCACHE:
        case PMAP_NOCACHE_OVR:
                PMAP_COUNT(uncached_mappings);
                break;
        }
#endif

        attr = L3_PAGE | (kenter ? 0 : LX_BLKPAG_NG);
        attr = _pmap_pte_adjust_prot(attr, prot, mdattr, user);
        attr = _pmap_pte_adjust_cacheflags(attr, flags);
        if (VM_MAXUSER_ADDRESS > va && !efirt_p)
                attr |= LX_BLKPAG_APUSER;
        if (flags & PMAP_WIRED)
                attr |= LX_BLKPAG_OS_WIRED;
#ifdef MULTIPROCESSOR
        attr |= LX_BLKPAG_SH_IS;
#endif

        pte = pa | attr;

        struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
        const tlb_asid_t asid = pai->pai_asid;

        if (need_sync_icache) {
                /* non-exec -> exec */
                UVMHIST_LOG(pmaphist,
                    "icache_sync: pm=%p, va=%016lx, pte: %016lx -> %016lx",
                    pm, va, opte, pte);

                if (!l3pte_readable(pte)) {
                        PTE_ICACHE_SYNC_PAGE(pte, ptep, asid, va);
                        atomic_swap_64(ptep, pte);
                        AARCH64_TLBI_BY_ASID_VA(asid, va);
                } else {
                        atomic_swap_64(ptep, pte);
                        AARCH64_TLBI_BY_ASID_VA(asid, va);
                        cpu_icache_sync_range(va, PAGE_SIZE);
                }
        } else {
                atomic_swap_64(ptep, pte);
                AARCH64_TLBI_BY_ASID_VA(asid, va);
        }

        if (pte & LX_BLKPAG_OS_WIRED) {
                _pmap_adj_wired_count(pm, 1);
        }
        _pmap_adj_resident_count(pm, 1);

 fail1:
        if (pps[1] != NULL)
                pmap_pv_unlock(pps[1]);
        if (pps[0] != NULL)
                pmap_pv_unlock(pps[0]);
 fail0:
        if (!kenter) {
                pm_unlock(pm);

                /* spare pv was not used. discard */
                if (spv != NULL)
                        pool_cache_put(&_pmap_pv_pool, spv);

                if (opv != NULL)
                        pool_cache_put(&_pmap_pv_pool, opv);
        }

        return error;
}

int
pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
{
        return _pmap_enter(pm, va, pa, prot, flags, false);
}


bool
pmap_remove_all(struct pmap *pm)
{
        struct pmap_page *pp;
        struct pv_entry *pv, *pvtmp, *opv, *pvtofree = NULL;
        pt_entry_t pte, *ptep;
        paddr_t pa;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pm=%p", pm, 0, 0, 0);

        KASSERT(pm != pmap_kernel());

        UVMHIST_LOG(pmaphist, "pm=%p, asid=%d", pm,
            PMAP_PAI(pm, cpu_tlb_info(ci))->pai_asid, 0, 0);

        pm_lock(pm);

        LIST_FOREACH_SAFE(pv, &pm->pm_pvlist, pv_proc, pvtmp) {
                ptep = pv->pv_ptep;
                pte = *ptep;

                KASSERTMSG(lxpde_valid(pte),
                    "pte is not valid: pmap=%p, va=%016lx",
                    pm, pv->pv_va);

                pa = lxpde_pa(pte);
                pp = phys_to_pp(pa);

                KASSERTMSG(pp != NULL,
                    "no pmap_page of physical address:%016lx, "
                    "pmap=%p, va=%016lx",
                    pa, pm, pv->pv_va);

                pmap_pv_lock(pp);
                opv = _pmap_remove_pv(pp, pm, trunc_page(pv->pv_va), pte);
                pmap_pv_unlock(pp);
                if (opv != NULL) {
                        opv->pv_next = pvtofree;
                        pvtofree = opv;
                }
        }
        /* all PTE should now be cleared */
        pm->pm_stats.wired_count = 0;
        pm->pm_stats.resident_count = 0;

        /* clear L0 page table page */
        pmap_zero_page(pm->pm_l0table_pa);

        aarch64_tlbi_by_asid(PMAP_PAI(pm, cpu_tlb_info(ci))->pai_asid);

        /* free L1-L3 page table pages, but not L0 */
        _pmap_free_pdp_all(pm, false);

        pm_unlock(pm);

        for (pv = pvtofree; pv != NULL; pv = pvtmp) {
                pvtmp = pv->pv_next;
                pool_cache_put(&_pmap_pv_pool, pv);
        }

        return true;
}

static void
_pmap_remove(struct pmap *pm, vaddr_t sva, vaddr_t eva, bool kremove,
    struct pv_entry **pvtofree)
{
        pt_entry_t pte, *ptep = NULL;
        struct pmap_page *pp;
        struct pv_entry *opv;
        paddr_t pa;
        vaddr_t va;
        vsize_t blocksize = 0;
        bool pdpremoved;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pm=%p, sva=%016lx, eva=%016lx, kremove=%d",
            pm, sva, eva, kremove);

        KASSERT(kremove || mutex_owned(&pm->pm_lock));

        for (va = sva; (va < eva) && (pm->pm_stats.resident_count != 0);
            va = (va + blocksize) & ~(blocksize - 1)) {

                /* va is belong to the same L3 table as before? */
                if ((blocksize == L3_SIZE) && ((va & L3INDEXMASK) != 0))
                        ptep++;
                else
                        ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);

                pte = *ptep;
                if (!lxpde_valid(pte))
                        continue;

                if (!kremove) {
                        pa = lxpde_pa(pte);
                        pp = phys_to_pp(pa);
                        if (pp != NULL) {

                                pmap_pv_lock(pp);
                                opv = _pmap_remove_pv(pp, pm, va, pte);
                                pmap_pv_unlock(pp);
                                if (opv != NULL) {
                                        opv->pv_next = *pvtofree;
                                        *pvtofree = opv;
                                }
                        }
                }

                pte = atomic_swap_64(ptep, 0);
                if (!lxpde_valid(pte))
                        continue;
                struct pmap_asid_info * const pai = PMAP_PAI(pm,
                    cpu_tlb_info(ci));

                pdpremoved = _pmap_pdp_delref(pm,
                    AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep)), true);
                AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va);

                if (pdpremoved) {
                        /*
                         * this Ln page table page has been removed.
                         * skip to next Ln table
                         */
                        blocksize *= Ln_ENTRIES;
                }

                if ((pte & LX_BLKPAG_OS_WIRED) != 0) {
                        _pmap_adj_wired_count(pm, -1);
                }
                _pmap_adj_resident_count(pm, -1);
        }
}

void
pmap_remove(struct pmap *pm, vaddr_t sva, vaddr_t eva)
{
        struct pv_entry *pvtofree = NULL;
        struct pv_entry *pv, *pvtmp;

        KASSERT_PM_ADDR(pm, sva);
        KASSERT(!IN_DIRECTMAP_ADDR(sva));

        pm_lock(pm);
        _pmap_remove(pm, sva, eva, false, &pvtofree);
        pm_unlock(pm);

        for (pv = pvtofree; pv != NULL; pv = pvtmp) {
                pvtmp = pv->pv_next;
                pool_cache_put(&_pmap_pv_pool, pv);
        }
}

static void
pmap_page_remove(struct pmap_page *pp, vm_prot_t prot)
{
        struct pv_entry *pv, *pvtmp;
        struct pv_entry *pvtofree = NULL;
        struct pmap *pm;
        pt_entry_t opte;

        /* remove all pages reference to this physical page */
        pmap_pv_lock(pp);
        for (pv = &pp->pp_pv; pv != NULL;) {
                if ((pm = pv->pv_pmap) == NULL) {
                        KASSERT(pv == &pp->pp_pv);
                        pv = pp->pp_pv.pv_next;
                        continue;
                }
                if (!pm_reverse_lock(pm, pp)) {
                        /* now retry */
                        pv = &pp->pp_pv;
                        continue;
                }
                opte = atomic_swap_64(pv->pv_ptep, 0);
                struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
                const vaddr_t va = trunc_page(pv->pv_va);

                if (lxpde_valid(opte)) {
                        _pmap_pdp_delref(pm,
                            AARCH64_KVA_TO_PA(trunc_page(
                            (vaddr_t)pv->pv_ptep)), false);
                        AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va);

                        if ((opte & LX_BLKPAG_OS_WIRED) != 0) {
                                _pmap_adj_wired_count(pm, -1);
                        }
                        _pmap_adj_resident_count(pm, -1);
                }
                pvtmp = _pmap_remove_pv(pp, pm, va, opte);
                if (pvtmp == NULL) {
                        KASSERT(pv == &pp->pp_pv);
                } else {
                        KASSERT(pv == pvtmp);
                        KASSERT(pp->pp_pv.pv_next == pv->pv_next);
                        pv->pv_next = pvtofree;
                        pvtofree = pv;
                }
                pm_unlock(pm);
                pv = pp->pp_pv.pv_next;
        }
        pmap_pv_unlock(pp);

        for (pv = pvtofree; pv != NULL; pv = pvtmp) {
                pvtmp = pv->pv_next;
                pool_cache_put(&_pmap_pv_pool, pv);
        }
}

#ifdef __HAVE_PMAP_PV_TRACK
void
pmap_pv_protect(paddr_t pa, vm_prot_t prot)
{
        struct pmap_page *pp;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pa=%016lx, prot=%08x", pa, prot, 0, 0);

        pp = pmap_pv_tracked(pa);
        if (pp == NULL)
                panic("pmap_pv_protect: page not pv-tracked: %#" PRIxPADDR, pa);

        KASSERT(prot == VM_PROT_NONE);
        pmap_page_remove(pp, prot);
}
#endif

void
pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
{
        struct pv_entry *pv;
        struct pmap_page *pp;
        struct pmap *pm;

        KASSERT((prot & VM_PROT_READ) || !(prot & VM_PROT_WRITE));

        pp = VM_PAGE_TO_PP(pg);

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pg=%p, pp=%p, pa=%016lx, prot=%08x",
            pg, pp, VM_PAGE_TO_PHYS(pg), prot);

        /* do an unlocked check first */
        if (atomic_load_relaxed(&pp->pp_pv.pv_pmap) == NULL &&
            atomic_load_relaxed(&pp->pp_pv.pv_next) == NULL) {
                return;
        }

        if ((prot & (VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE)) ==
            VM_PROT_NONE) {
                pmap_page_remove(pp, prot);
        } else {
                pmap_pv_lock(pp);
                pv = &pp->pp_pv;
                while (pv != NULL) {
                        if ((pm = pv->pv_pmap) == NULL) {
                                KASSERT(pv == &pp->pp_pv);
                                pv = pv->pv_next;
                                continue;
                        }
                        if (!pm_reverse_lock(pm, pp)) {
                                /* retry */
                                pv = &pp->pp_pv;
                                continue;
                        }
                        _pmap_protect_pv(pp, pv, prot);
                        pm_unlock(pm);
                        pv = pv->pv_next;
                }
                pmap_pv_unlock(pp);
        }
}

void
pmap_unwire(struct pmap *pm, vaddr_t va)
{
        pt_entry_t pte, *ptep;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pm=%p, va=%016lx", pm, va, 0, 0);

        PMAP_COUNT(unwire);

        KASSERT_PM_ADDR(pm, va);
        KASSERT(!IN_DIRECTMAP_ADDR(va));

        pm_lock(pm);
        ptep = _pmap_pte_lookup_l3(pm, va);
        if (ptep != NULL) {
                pte = *ptep;
                if (!l3pte_valid(pte) ||
                    ((pte & LX_BLKPAG_OS_WIRED) == 0)) {
                        /* invalid pte, or pte is not wired */
                        PMAP_COUNT(unwire_failure);
                        pm_unlock(pm);
                        return;
                }

                pte &= ~LX_BLKPAG_OS_WIRED;
                atomic_swap_64(ptep, pte);

                _pmap_adj_wired_count(pm, -1);
        }
        pm_unlock(pm);
}

bool
pmap_fault_fixup(struct pmap *pm, vaddr_t va, vm_prot_t accessprot, bool user)
{
        struct pmap_page *pp;
        pt_entry_t *ptep, pte;
        vm_prot_t pmap_prot;
        paddr_t pa;
        bool fixed = false;

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pm=%p, va=%016lx, accessprot=%08x",
            pm, va, accessprot, 0);

#if 0
        KASSERT_PM_ADDR(pm, va);
#else
        if ((pm == pmap_kernel() &&
            !(IN_RANGE(va, VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS))) ||
            (pm != pmap_kernel() &&
            !(IN_RANGE(va, VM_MIN_ADDRESS, VM_MAX_ADDRESS)))) {

                UVMHIST_LOG(pmaphist,
                    "pmap space and va mismatch: kernel=%jd, va=%016lx",
                    pm == pmap_kernel(), va, 0, 0);
                return false;
        }
#endif

        pm_lock(pm);

        ptep = _pmap_pte_lookup_l3(pm, va);
        if (ptep == NULL) {
                UVMHIST_LOG(pmaphist, "pte_lookup failure: va=%016lx",
                    va, 0, 0, 0);
                goto done;
        }

        pte = *ptep;
        if (!l3pte_valid(pte)) {
                UVMHIST_LOG(pmaphist, "invalid pte: %016llx: va=%016lx",
                    pte, va, 0, 0);
                goto done;
        }

        pa = l3pte_pa(*ptep);
        pp = phys_to_pp(pa);
        if (pp == NULL) {
                UVMHIST_LOG(pmaphist, "pmap_page not found: va=%016lx",
                    va, 0, 0, 0);
                goto done;
        }

        /*
         * Get the prot specified by pmap_enter().
         * A valid pte is considered a readable page.
         * If DBM is 1, it is considered a writable page.
         */
        pmap_prot = VM_PROT_READ;
        if ((pte & (LX_BLKPAG_OS_WRITE | LX_BLKPAG_DBM)) != 0)
                pmap_prot |= VM_PROT_WRITE;

        if (l3pte_executable(pte, pm != pmap_kernel()))
                pmap_prot |= VM_PROT_EXECUTE;

        UVMHIST_LOG(pmaphist, "va=%016lx, pmapprot=%08x, accessprot=%08x",
            va, pmap_prot, accessprot, 0);

        /* ignore except read/write */
        accessprot &= (VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE);

        /* PROT_EXEC requires implicit PROT_READ */
        if (accessprot & VM_PROT_EXECUTE)
                accessprot |= VM_PROT_READ;

        /* no permission to read/write/execute for this page */
        if ((pmap_prot & accessprot) != accessprot) {
                UVMHIST_LOG(pmaphist, "no permission to access", 0, 0, 0, 0);
                goto done;
        }

        /* pte is readable and writable, but occurred fault? probably copy(9) */
        if ((pte & LX_BLKPAG_AF) && ((pte & LX_BLKPAG_AP) == LX_BLKPAG_AP_RW))
                goto done;

        pmap_pv_lock(pp);
        if ((pte & LX_BLKPAG_AF) == 0) {
                /* pte has no AF bit, set referenced and AF bit */
                UVMHIST_LOG(pmaphist,
                    "REFERENCED:"
                    " va=%016lx, pa=%016lx, pte_prot=%08x, accessprot=%08x",
                    va, pa, pmap_prot, accessprot);
                pp->pp_pv.pv_va |= VM_PROT_READ;        /* set referenced */
                pte |= LX_BLKPAG_AF;

                PMAP_COUNT(fixup_referenced);
        }
        if ((accessprot & VM_PROT_WRITE) &&
            ((pte & LX_BLKPAG_AP) == LX_BLKPAG_AP_RO)) {
                /* pte is not RW. set modified and RW */

                UVMHIST_LOG(pmaphist, "MODIFIED:"
                    " va=%016lx, pa=%016lx, pte_prot=%08x, accessprot=%08x",
                    va, pa, pmap_prot, accessprot);
                pp->pp_pv.pv_va |= VM_PROT_WRITE;       /* set modified */
                pte &= ~LX_BLKPAG_AP;
                pte |= LX_BLKPAG_AP_RW;

                PMAP_COUNT(fixup_modified);
        }
        pmap_pv_unlock(pp);

        atomic_swap_64(ptep, pte);
        struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
        AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va);

        fixed = true;

 done:
        pm_unlock(pm);
        return fixed;
}

bool
pmap_clear_modify(struct vm_page *pg)
{
        struct pv_entry *pv;
        struct pmap_page * const pp = VM_PAGE_TO_PP(pg);
        pt_entry_t *ptep, pte, opte;
        vaddr_t va;
#ifdef ARMV81_HAFDBS
        bool modified;
#endif

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pg=%p, flags=%08x",
            pg, (int)(pp->pp_pv.pv_va & (PAGE_SIZE - 1)), 0, 0);

        PMAP_COUNT(clear_modify);

        /*
         * if this is a new page, assert it has no mappings and simply zap
         * the stored attributes without taking any locks.
         */
        if ((pg->flags & PG_FAKE) != 0) {
                KASSERT(atomic_load_relaxed(&pp->pp_pv.pv_pmap) == NULL);
                KASSERT(atomic_load_relaxed(&pp->pp_pv.pv_next) == NULL);
                atomic_store_relaxed(&pp->pp_pv.pv_va, 0);
                return false;
        }

        pmap_pv_lock(pp);

        if (
#ifdef ARMV81_HAFDBS
            aarch64_hafdbs_enabled != ID_AA64MMFR1_EL1_HAFDBS_AD &&
#endif
            (pp->pp_pv.pv_va & VM_PROT_WRITE) == 0) {
                pmap_pv_unlock(pp);
                return false;
        }
#ifdef ARMV81_HAFDBS
        modified = ((pp->pp_pv.pv_va & VM_PROT_WRITE) != 0);
#endif
        pp->pp_pv.pv_va &= ~(vaddr_t)VM_PROT_WRITE;

        for (pv = &pp->pp_pv; pv != NULL; pv = pv->pv_next) {
                if (pv->pv_pmap == NULL) {
                        KASSERT(pv == &pp->pp_pv);
                        continue;
                }

                PMAP_COUNT(clear_modify_pages);

                va = trunc_page(pv->pv_va);

                ptep = pv->pv_ptep;
                opte = pte = *ptep;
 tryagain:
                if (!l3pte_valid(pte))
                        continue;
                if ((pte & LX_BLKPAG_AP) == LX_BLKPAG_AP_RO)
                        continue;
#ifdef ARMV81_HAFDBS
                modified = true;
#endif
                /* clear write permission */
                pte &= ~LX_BLKPAG_AP;
                pte |= LX_BLKPAG_AP_RO;

                /* XXX: possible deadlock if using PM_LOCK(). this is racy */
                if ((pte = atomic_cas_64(ptep, opte, pte)) != opte) {
                        opte = pte;
                        goto tryagain;
                }

                struct pmap * const pm = pv->pv_pmap;
                struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
                AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va);

                UVMHIST_LOG(pmaphist,
                    "va=%016llx, ptep=%p, pa=%016lx, RW -> RO",
                    va, ptep, l3pte_pa(pte), 0);
        }

        pmap_pv_unlock(pp);

#ifdef ARMV81_HAFDBS
        return modified;
#else
        return true;
#endif
}

bool
pmap_clear_reference(struct vm_page *pg)
{
        struct pv_entry *pv;
        struct pmap_page * const pp = VM_PAGE_TO_PP(pg);
        pt_entry_t *ptep, pte, opte;
        vaddr_t va;
#ifdef ARMV81_HAFDBS
        bool referenced;
#endif

        UVMHIST_FUNC(__func__);
        UVMHIST_CALLARGS(pmaphist, "pg=%p, pp=%p, flags=%08x",
            pg, pp, (int)(pp->pp_pv.pv_va & (PAGE_SIZE - 1)), 0);

        pmap_pv_lock(pp);

        if (
#ifdef ARMV81_HAFDBS
            aarch64_hafdbs_enabled == ID_AA64MMFR1_EL1_HAFDBS_NONE &&
#endif
            (pp->pp_pv.pv_va & VM_PROT_READ) == 0) {
                pmap_pv_unlock(pp);
                return false;
        }
#ifdef ARMV81_HAFDBS
        referenced = ((pp->pp_pv.pv_va & VM_PROT_READ) != 0);
#endif
        pp->pp_pv.pv_va &= ~(vaddr_t)VM_PROT_READ;

        PMAP_COUNT(clear_reference);
        for (pv = &pp->pp_pv; pv != NULL; pv = pv->pv_next) {
                if (pv->pv_pmap == NULL) {
                        KASSERT(pv == &pp->pp_pv);
                        continue;
                }

                PMAP_COUNT(clear_reference_pages);

                va = trunc_page(pv->pv_va);

                ptep = pv->pv_ptep;
                opte = pte = *ptep;
 tryagain:
                if (!l3pte_valid(pte))
                        continue;
                if ((pte & LX_BLKPAG_AF) == 0)
                        continue;
#ifdef ARMV81_HAFDBS
                referenced = true;
#endif
                /* clear access permission */
                pte &= ~LX_BLKPAG_AF;

                /* XXX: possible deadlock if using PM_LOCK(). this is racy */
                if ((pte = atomic_cas_64(ptep, opte, pte)) != opte) {
                        opte = pte;
                        goto tryagain;
                }

                struct pmap * const pm = pv->pv_pmap;
                struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
                AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va);

                UVMHIST_LOG(pmaphist, "va=%016llx, ptep=%p, pa=%016lx, unse AF",
                    va, ptep, l3pte_pa(pte), 0);
        }

        pmap_pv_unlock(pp);

#ifdef ARMV81_HAFDBS
        return referenced;
#else
        return true;
#endif
}

bool
pmap_is_modified(struct vm_page *pg)
{
        struct pmap_page * const pp = VM_PAGE_TO_PP(pg);

        if (pp->pp_pv.pv_va & VM_PROT_WRITE)
                return true;

#ifdef ARMV81_HAFDBS
        /* check hardware dirty flag on each pte */
        if (aarch64_hafdbs_enabled == ID_AA64MMFR1_EL1_HAFDBS_AD) {
                struct pv_entry *pv;
                pt_entry_t *ptep, pte;

                pmap_pv_lock(pp);
                for (pv = &pp->pp_pv; pv != NULL; pv = pv->pv_next) {
                        if (pv->pv_pmap == NULL) {
                                KASSERT(pv == &pp->pp_pv);
                                continue;
                        }

                        ptep = pv->pv_ptep;
                        pte = *ptep;
                        if (!l3pte_valid(pte))
                                continue;

                        if ((pte & LX_BLKPAG_AP) == LX_BLKPAG_AP_RW) {
                                pp->pp_pv.pv_va |= VM_PROT_WRITE;
                                pmap_pv_unlock(pp);
                                return true;
                        }
                }
                pmap_pv_unlock(pp);
        }
#endif

        return false;
}

bool
pmap_is_referenced(struct vm_page *pg)
{
        struct pmap_page * const pp = VM_PAGE_TO_PP(pg);

        if (pp->pp_pv.pv_va & VM_PROT_READ)
                return true;

#ifdef ARMV81_HAFDBS
        /* check hardware access flag on each pte */
        if (aarch64_hafdbs_enabled != ID_AA64MMFR1_EL1_HAFDBS_NONE) {
                struct pv_entry *pv;
                pt_entry_t *ptep, pte;

                pmap_pv_lock(pp);
                for (pv = &pp->pp_pv; pv != NULL; pv = pv->pv_next) {
                        if (pv->pv_pmap == NULL) {
                                KASSERT(pv == &pp->pp_pv);
                                continue;
                        }

                        ptep = pv->pv_ptep;
                        pte = *ptep;
                        if (!l3pte_valid(pte))
                                continue;

                        if (pte & LX_BLKPAG_AF) {
                                pp->pp_pv.pv_va |= VM_PROT_READ;
                                pmap_pv_unlock(pp);
                                return true;
                        }
                }
                pmap_pv_unlock(pp);
        }
#endif

        return false;
}

/* get pointer to kernel segment L2 or L3 table entry */
pt_entry_t *
kvtopte(vaddr_t va)
{
        KASSERT(IN_RANGE(va, VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS));

        return _pmap_pte_lookup_bs(pmap_kernel(), va, NULL);
}

#ifdef DDB
void
pmap_db_pmap_print(struct pmap *pm,
    void (*pr)(const char *, ...) __printflike(1, 2))
{
        struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));

        pr(" pm_asid       = %d\n", pai->pai_asid);
        pr(" pm_l0table    = %p\n", pm->pm_l0table);
        pr(" pm_l0table_pa = %lx\n", pm->pm_l0table_pa);
        pr(" pm_activated  = %d\n\n", pm->pm_activated);
}
#endif /* DDB */

/***************************** PMAP DEBUGGING ********************************/

#ifdef PMAP_DEBUG

void    pmap_test_mod_ref(void);

void
pmap_test_mod_ref(void)
{
        int val;
        bool mod, ref;
        bool exp_mod, exp_ref;

        vaddr_t va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
            UVM_KMF_VAONLY | UVM_KMF_NOWAIT);

        if (va == 0) {
                printf("%s: couldn't allocate a VA to use\n",
                    __func__);
                return;
        }

        struct vm_page *pg = uvm_pagealloc(NULL, 0, NULL, 0);
        if (pg == NULL) {
                printf("%s: couldn't allocate page\n", __func__);
                return;
        }
        pg->flags &= ~(PG_BUSY|PG_FAKE);        /* never busy */

        struct lwp *l = curlwp;
        pmap_deactivate(l);

        const paddr_t pa = VM_PAGE_TO_PHYS(pg);
        volatile int * const loc = (volatile int *)va;

        /* Initialize page and mod/ref state to pristine. */
        pmap_zero_page(pa);
        pmap_clear_modify(pg);
        pmap_clear_reference(pg);

        mod = pmap_is_modified(pg);
        ref = pmap_is_referenced(pg);
        exp_mod = false;
        exp_ref = false;
        printf("%s: validating pristine page: mod=%d(%d) ref=%d(%d) (%s)\n",
            __func__,
            mod, exp_mod, ref, exp_ref,
            mod == exp_mod && ref == exp_ref ? "OK" : "FAIL");

        /* Enter non-seeded R/W mapping. */
        pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0);
        pmap_update(pmap_kernel());

        mod = pmap_is_modified(pg);
        ref = pmap_is_referenced(pg);
        exp_mod = false;
        exp_ref = false;
        printf("%s: enter(ALL, 0): mod=%d(%d) ref=%d(%d) (%s)\n",
            __func__,
            mod, exp_mod, ref, exp_ref,
            mod == exp_mod && ref == exp_ref ? "OK" : "FAIL");

        /* reference page */
        val = *loc;

        mod = pmap_is_modified(pg);
        ref = pmap_is_referenced(pg);
        exp_mod = false;
        exp_ref = true;
        printf("%s: ref 1: val=%d: mod=%d(%d) ref=%d(%d) (%s)\n",
            __func__,
            val,
            mod, exp_mod, ref, exp_ref,
            mod == exp_mod && ref == exp_ref ? "OK" : "FAIL");

        /* validate clear behavior. */
        exp_mod = mod;
        exp_ref = ref;
        mod = pmap_clear_modify(pg);
        ref = pmap_clear_reference(pg);
        printf("%s: checking clear 1: mod=%d(%d) ref=%d(%d) (%s)\n",
            __func__,
            mod, exp_mod, ref, exp_ref,
            mod == exp_mod && ref == exp_ref ? "OK" : "FAIL");

        mod = pmap_is_modified(pg);
        ref = pmap_is_referenced(pg);
        exp_mod = false;
        exp_ref = false;
        printf("%s: checking clear 2: mod=%d(%d) ref=%d(%d) (%s)\n",
            __func__,
            mod, exp_mod, ref, exp_ref,
            mod == exp_mod && ref == exp_ref ? "OK" : "FAIL");

        /* reference page again */
        val = *loc;

        mod = pmap_is_modified(pg);
        ref = pmap_is_referenced(pg);
        exp_mod = false;
        exp_ref = true;
        printf("%s: ref 2: val=%d: mod=%d(%d) ref=%d(%d) (%s)\n",
            __func__,
            val,
            mod, exp_mod, ref, exp_ref,
            mod == exp_mod && ref == exp_ref ? "OK" : "FAIL");

        /* modify page */
        *loc = 0xff;

        mod = pmap_is_modified(pg);
        ref = pmap_is_referenced(pg);
        exp_mod = true;
        exp_ref = true;
        printf("%s: mod 1: mod=%d(%d) ref=%d(%d) (%s)\n",
            __func__,
            mod, exp_mod, ref, exp_ref,
            mod == exp_mod && ref == exp_ref ? "OK" : "FAIL");

        /* write-protect page */
        pmap_page_protect(pg, UVM_PROT_READ);

        mod = pmap_clear_modify(pg);
        ref = pmap_clear_reference(pg);
        exp_mod = true;
        exp_ref = true;
        printf("%s: mod 2: mod=%d(%d) ref=%d(%d) (%s)\n",
            __func__,
            mod, exp_mod, ref, exp_ref,
            mod == exp_mod && ref == exp_ref ? "OK" : "FAIL");

        mod = pmap_is_modified(pg);
        ref = pmap_is_referenced(pg);
        exp_mod = false;
        exp_ref = false;
        printf("%s: checking clear 3: mod=%d(%d) ref=%d(%d) (%s)\n",
            __func__,
            mod, exp_mod, ref, exp_ref,
            mod == exp_mod && ref == exp_ref ? "OK" : "FAIL");

        /* modify page again */
        pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0);
        *loc = 0xaa;

        mod = pmap_is_modified(pg);
        ref = pmap_is_referenced(pg);
        exp_mod = true;
        exp_ref = true;
        printf("%s: mod 3: mod=%d(%d) ref=%d(%d) (%s)\n",
            __func__,
            mod, exp_mod, ref, exp_ref,
            mod == exp_mod && ref == exp_ref ? "OK" : "FAIL");

        /* remove all mappings of page */
        pmap_page_protect(pg, UVM_PROT_NONE);

        mod = pmap_clear_modify(pg);
        ref = pmap_clear_reference(pg);
        exp_mod = true;
        exp_ref = true;
        printf("%s: mod 4: mod=%d(%d) ref=%d(%d) (%s)\n",
            __func__,
            mod, exp_mod, ref, exp_ref,
            mod == exp_mod && ref == exp_ref ? "OK" : "FAIL");

        /* all done. */
        pmap_remove(pmap_kernel(), va, va + PAGE_SIZE);
        pmap_update(pmap_kernel());

        printf("%s: done\n", __func__);

        pmap_activate(l);

        uvm_km_free(kernel_map, va, PAGE_SIZE, UVM_KMF_VAONLY);
}

#endif /* PMAP_DEBUG */