crypto/krb5/src/include/gssrpc/pmap_prot.h

93

extern bool_t xdr_pmap(XDR *, struct pmap *);

crypto/krb5/src/include/gssrpc/pmap_prot.h

96

struct pmap pml_map;

crypto/krb5/src/lib/rpc/pmap_clnt.c

152

struct pmap parms;

crypto/krb5/src/lib/rpc/pmap_clnt.c

71

struct pmap parms;

crypto/krb5/src/lib/rpc/pmap_getport.c

72

struct pmap parms;

crypto/krb5/src/lib/rpc/pmap_prot.c

49

xdr_pmap(XDR *xdrs, struct pmap *regs)

crypto/openssl/crypto/x509/v3_pmaps.c

101

sk_POLICY_MAPPING_push(pmaps, pmap); /* no failure as it was reserved */

crypto/openssl/crypto/x509/v3_pmaps.c

50

POLICY_MAPPING *pmap;

crypto/openssl/crypto/x509/v3_pmaps.c

56

pmap = sk_POLICY_MAPPING_value(pmaps, i);

crypto/openssl/crypto/x509/v3_pmaps.c

57

i2t_ASN1_OBJECT(obj_tmp1, 80, pmap->issuerDomainPolicy);

crypto/openssl/crypto/x509/v3_pmaps.c

58

i2t_ASN1_OBJECT(obj_tmp2, 80, pmap->subjectDomainPolicy);

crypto/openssl/crypto/x509/v3_pmaps.c

67

POLICY_MAPPING *pmap = NULL;

crypto/openssl/crypto/x509/v3_pmaps.c

93

pmap = POLICY_MAPPING_new();

crypto/openssl/crypto/x509/v3_pmaps.c

94

if (pmap == NULL) {

crypto/openssl/crypto/x509/v3_pmaps.c

98

pmap->issuerDomainPolicy = obj1;

crypto/openssl/crypto/x509/v3_pmaps.c

99

pmap->subjectDomainPolicy = obj2;

include/rpc/pmap_prot.h

94

struct pmap pml_map;

include/rpc/pmap_prot.h

99

extern bool_t xdr_pmap(XDR *, struct pmap *);

lib/libc/rpc/pmap_getport.c

70

struct pmap parms;

lib/libc/rpc/pmap_prot.c

50

xdr_pmap(XDR *xdrs, struct pmap *regs)

lib/libc/rpc/rpcb_clnt.c

733

struct pmap pmapparms;

sys/amd64/amd64/genassym.c

73

ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));

sys/amd64/amd64/mp_machdep.c

601

smp_targeted_tlb_shootdown_native(pmap_t pmap, vm_offset_t addr1,

sys/amd64/amd64/mp_machdep.c

622

CPU_COPY(pmap_invalidate_cpu_mask(pmap), &mask);

sys/amd64/amd64/mp_machdep.c

641

PCPU_SET(smp_tlb_pmap, pmap);

sys/amd64/amd64/mp_machdep.c

667

curcpu_cb(pmap, addr1, addr2);

sys/amd64/amd64/mp_machdep.c

686

curcpu_cb(pmap, addr1, addr2);

sys/amd64/amd64/mp_machdep.c

692

smp_masked_invltlb(pmap_t pmap, smp_invl_cb_t curcpu_cb)

sys/amd64/amd64/mp_machdep.c

694

if (invlpgb_works && pmap == kernel_pmap) {

sys/amd64/amd64/mp_machdep.c

708

smp_targeted_tlb_shootdown(pmap, 0, 0, curcpu_cb, invl_op_tlb);

sys/amd64/amd64/mp_machdep.c

715

smp_masked_invlpg(vm_offset_t addr, pmap_t pmap, smp_invl_cb_t curcpu_cb)

sys/amd64/amd64/mp_machdep.c

717

if (invlpgb_works && pmap == kernel_pmap) {

sys/amd64/amd64/mp_machdep.c

724

smp_targeted_tlb_shootdown(pmap, addr, 0, curcpu_cb, invl_op_pg);

sys/amd64/amd64/mp_machdep.c

731

smp_masked_invlpg_range(vm_offset_t addr1, vm_offset_t addr2, pmap_t pmap,

sys/amd64/amd64/mp_machdep.c

734

if (invlpgb_works && pmap == kernel_pmap) {

sys/amd64/amd64/mp_machdep.c

767

smp_targeted_tlb_shootdown(pmap, addr1, addr2, curcpu_cb,

sys/amd64/amd64/pmap.c

10017

pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/amd64/amd64/pmap.c

10025

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

10026

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

10027

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

10028

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

10030

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

10034

pdpe = pmap_pdpe(pmap, addr);

sys/amd64/amd64/pmap.c

10044

pdep = pmap_pde(pmap, addr);

sys/amd64/amd64/pmap.c

10073

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

10078

pmap_pcid_alloc(pmap_t pmap, struct pmap_pcid *pcidp)

sys/amd64/amd64/pmap.c

10109

pmap_pcid_alloc_checked(pmap_t pmap, struct pmap_pcid *pcidp)

sys/amd64/amd64/pmap.c

10113

cached = pmap_pcid_alloc(pmap, pcidp);

sys/amd64/amd64/pmap.c

10115

("pmap %p cpu %d pcid %#x", pmap, PCPU_GET(cpuid), pcidp->pm_pcid));

sys/amd64/amd64/pmap.c

10116

KASSERT(pcidp->pm_pcid != PMAP_PCID_KERN || pmap == kernel_pmap,

sys/amd64/amd64/pmap.c

10118

pmap, PCPU_GET(cpuid), pcidp->pm_pcid));

sys/amd64/amd64/pmap.c

10123

pmap_activate_sw_pti_post(struct thread *td, pmap_t pmap)

sys/amd64/amd64/pmap.c

10126

PCPU_GET(tssp)->tss_rsp0 = pmap->pm_ucr3 != PMAP_NO_CR3 ?

sys/amd64/amd64/pmap.c

10131

pmap_activate_sw_pcid_pti(struct thread *td, pmap_t pmap, u_int cpuid)

sys/amd64/amd64/pmap.c

10149

pcidp = zpcpu_get_cpu(pmap->pm_pcidp, cpuid);

sys/amd64/amd64/pmap.c

10150

cached = pmap_pcid_alloc_checked(pmap, pcidp);

sys/amd64/amd64/pmap.c

10152

if ((cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3)

sys/amd64/amd64/pmap.c

10153

load_cr3(pmap->pm_cr3 | pcidp->pm_pcid);

sys/amd64/amd64/pmap.c

10154

PCPU_SET(curpmap, pmap);

sys/amd64/amd64/pmap.c

10155

kcr3 = pmap->pm_cr3 | pcidp->pm_pcid;

sys/amd64/amd64/pmap.c

10156

ucr3 = pmap->pm_ucr3 | pcidp->pm_pcid | PMAP_PCID_USER_PT;

sys/amd64/amd64/pmap.c

10158

if (!cached && pmap->pm_ucr3 != PMAP_NO_CR3)

sys/amd64/amd64/pmap.c

10166

pmap_activate_sw_pti_post(td, pmap);

sys/amd64/amd64/pmap.c

10170

pmap_activate_sw_pcid_nopti(struct thread *td __unused, pmap_t pmap,

sys/amd64/amd64/pmap.c

10179

pcidp = zpcpu_get_cpu(pmap->pm_pcidp, cpuid);

sys/amd64/amd64/pmap.c

10180

cached = pmap_pcid_alloc_checked(pmap, pcidp);

sys/amd64/amd64/pmap.c

10182

if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3)

sys/amd64/amd64/pmap.c

10183

load_cr3(pmap->pm_cr3 | pcidp->pm_pcid | cached);

sys/amd64/amd64/pmap.c

10184

PCPU_SET(curpmap, pmap);

sys/amd64/amd64/pmap.c

10190

pmap_activate_sw_nopcid_nopti(struct thread *td __unused, pmap_t pmap,

sys/amd64/amd64/pmap.c

10194

load_cr3(pmap->pm_cr3);

sys/amd64/amd64/pmap.c

10195

PCPU_SET(curpmap, pmap);

sys/amd64/amd64/pmap.c

10199

pmap_activate_sw_nopcid_pti(struct thread *td, pmap_t pmap,

sys/amd64/amd64/pmap.c

10203

pmap_activate_sw_nopcid_nopti(td, pmap, cpuid);

sys/amd64/amd64/pmap.c

10204

PCPU_SET(kcr3, pmap->pm_cr3);

sys/amd64/amd64/pmap.c

10205

PCPU_SET(ucr3, pmap->pm_ucr3);

sys/amd64/amd64/pmap.c

10206

pmap_activate_sw_pti_post(td, pmap);

sys/amd64/amd64/pmap.c

10226

pmap_t oldpmap, pmap;

sys/amd64/amd64/pmap.c

10230

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/amd64/amd64/pmap.c

10231

if (oldpmap == pmap) {

sys/amd64/amd64/pmap.c

10237

CPU_SET_ATOMIC(cpuid, &pmap->pm_active);

sys/amd64/amd64/pmap.c

10238

pmap_activate_sw_mode(td, pmap, cpuid);

sys/amd64/amd64/pmap.c

10268

pmap_activate_boot(pmap_t pmap)

sys/amd64/amd64/pmap.c

10277

MPASS(pmap != kernel_pmap);

sys/amd64/amd64/pmap.c

10280

CPU_SET_ATOMIC(cpuid, &pmap->pm_active);

sys/amd64/amd64/pmap.c

10281

PCPU_SET(curpmap, pmap);

sys/amd64/amd64/pmap.c

10283

kcr3 = pmap->pm_cr3;

sys/amd64/amd64/pmap.c

10285

kcr3 |= pmap_get_pcid(pmap) | CR3_PCID_SAVE;

sys/amd64/amd64/pmap.c

10294

pmap_active_cpus(pmap_t pmap, cpuset_t *res)

sys/amd64/amd64/pmap.c

10296

*res = pmap->pm_active;

sys/amd64/amd64/pmap.c

10347

pmap_emulate_accessed_dirty(pmap_t pmap, vm_offset_t va, int ftype)

sys/amd64/amd64/pmap.c

10360

if (!pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

10363

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

10364

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

10365

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

10366

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

10370

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

10372

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

10420

pmap_promote_pde(pmap, pde, va, mpte, &lock)) {

sys/amd64/amd64/pmap.c

10437

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

10442

pmap_get_mapping(pmap_t pmap, vm_offset_t va, uint64_t *ptr, int *num)

sys/amd64/amd64/pmap.c

10451

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

10452

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

10454

pml4 = pmap_pml4e(pmap, va);

sys/amd64/amd64/pmap.c

10475

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

11422

pmap_pkru_assign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, u_int keyidx,

sys/amd64/amd64/pmap.c

11428

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11429

MPASS(pmap->pm_type == PT_X86);

sys/amd64/amd64/pmap.c

11432

!rangeset_check_empty(&pmap->pm_pkru, sva, eva))

sys/amd64/amd64/pmap.c

11439

error = rangeset_insert(&pmap->pm_pkru, sva, eva, ppr);

sys/amd64/amd64/pmap.c

11446

pmap_pkru_deassign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

11449

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11450

MPASS(pmap->pm_type == PT_X86);

sys/amd64/amd64/pmap.c

11452

return (rangeset_remove(&pmap->pm_pkru, sva, eva));

sys/amd64/amd64/pmap.c

11456

pmap_pkru_deassign_all(pmap_t pmap)

sys/amd64/amd64/pmap.c

11459

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11460

if (pmap->pm_type == PT_X86 &&

sys/amd64/amd64/pmap.c

11462

rangeset_remove_all(&pmap->pm_pkru);

sys/amd64/amd64/pmap.c

11471

pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte)

sys/amd64/amd64/pmap.c

11477

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11478

KASSERT(pmap->pm_type != PT_X86 || (*pte & X86_PG_PKU_MASK) == 0,

sys/amd64/amd64/pmap.c

11480

if (pmap->pm_type != PT_X86 ||

sys/amd64/amd64/pmap.c

11485

ppr = rangeset_containing(&pmap->pm_pkru, sva);

sys/amd64/amd64/pmap.c

11487

return (rangeset_empty(&pmap->pm_pkru, sva, eva));

sys/amd64/amd64/pmap.c

11490

if ((ppr = rangeset_beginning(&pmap->pm_pkru, va)) == NULL ||

sys/amd64/amd64/pmap.c

11499

pmap_pkru_get(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

11503

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11504

if (pmap->pm_type != PT_X86 ||

sys/amd64/amd64/pmap.c

11508

ppr = rangeset_containing(&pmap->pm_pkru, va);

sys/amd64/amd64/pmap.c

11524

pmap_pkru_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

11527

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11528

if (pmap->pm_type == PT_X86 &&

sys/amd64/amd64/pmap.c

11530

rangeset_remove_pred(&pmap->pm_pkru, sva, eva,

sys/amd64/amd64/pmap.c

11550

pmap_pkru_update_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/amd64/pmap.c

11560

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11561

MPASS(pmap->pm_type == PT_X86);

sys/amd64/amd64/pmap.c

11565

pml4e = pmap_pml4e(pmap, va);

sys/amd64/amd64/pmap.c

11600

} else if (!pmap_demote_pde(pmap, pde, va)) {

sys/amd64/amd64/pmap.c

11621

pmap_invalidate_range(pmap, sva, eva);

sys/amd64/amd64/pmap.c

11625

pmap_pkru_check_uargs(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/amd64/pmap.c

11629

if (pmap->pm_type != PT_X86 || keyidx > PMAP_MAX_PKRU_IDX ||

sys/amd64/amd64/pmap.c

11640

pmap_pkru_set(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, u_int keyidx,

sys/amd64/amd64/pmap.c

11647

error = pmap_pkru_check_uargs(pmap, sva, eva, keyidx, flags);

sys/amd64/amd64/pmap.c

11651

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

11652

error = pmap_pkru_assign(pmap, sva, eva, keyidx, flags);

sys/amd64/amd64/pmap.c

11654

pmap_pkru_update_range(pmap, sva, eva, keyidx);

sys/amd64/amd64/pmap.c

11655

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

11664

pmap_pkru_clear(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

11670

error = pmap_pkru_check_uargs(pmap, sva, eva, 0, 0);

sys/amd64/amd64/pmap.c

11674

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

11675

error = pmap_pkru_deassign(pmap, sva, eva);

sys/amd64/amd64/pmap.c

11677

pmap_pkru_update_range(pmap, sva, eva, 0);

sys/amd64/amd64/pmap.c

11678

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

12175

pmap_t pmap;

sys/amd64/amd64/pmap.c

12190

pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace);

sys/amd64/amd64/pmap.c

12192

pmap = PCPU_GET(curpmap);

sys/amd64/amd64/pmap.c

12194

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

12197

if (pmap_is_la57(pmap)) {

sys/amd64/amd64/pmap.c

12198

pml5 = pmap_pml5e(pmap, va);

sys/amd64/amd64/pmap.c

12206

pml4 = pmap_pml4e(pmap, va);

sys/amd64/amd64/pmap.c

12300

pmap_t pmap;

sys/amd64/amd64/pmap.c

12307

pmap = (pmap_t)addr;

sys/amd64/amd64/pmap.c

12309

pmap = PCPU_GET(curpmap);

sys/amd64/amd64/pmap.c

12311

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

12313

if (pmap_is_la57(pmap)) {

sys/amd64/amd64/pmap.c

12314

pml5 = pmap->pm_pmltop;

sys/amd64/amd64/pmap.c

12328

(vm_offset_t)pmap->pm_pmltop)), NUP4ML4E, PG_V);

sys/amd64/amd64/pmap.c

1324

static void free_pv_entry(pmap_t pmap, pv_entry_t pv);

sys/amd64/amd64/pmap.c

1325

static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp);

sys/amd64/amd64/pmap.c

1328

static void reserve_pv_entries(pmap_t pmap, int needed,

sys/amd64/amd64/pmap.c

1330

static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/amd64/amd64/pmap.c

1332

static bool pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde,

sys/amd64/amd64/pmap.c

1335

static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/amd64/amd64/pmap.c

1338

static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);

sys/amd64/amd64/pmap.c

1339

static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,

sys/amd64/amd64/pmap.c

1342

static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);

sys/amd64/amd64/pmap.c

1345

static bool pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);

sys/amd64/amd64/pmap.c

1346

static bool pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde,

sys/amd64/amd64/pmap.c

1348

static bool pmap_demote_pde_mpte(pmap_t pmap, pd_entry_t *pde,

sys/amd64/amd64/pmap.c

1350

static bool pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe,

sys/amd64/amd64/pmap.c

1352

static int pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/amd64/amd64/pmap.c

1354

static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,

sys/amd64/amd64/pmap.c

1356

static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,

sys/amd64/amd64/pmap.c

1359

static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/amd64/amd64/pmap.c

1365

static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,

sys/amd64/amd64/pmap.c

1371

static bool pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,

sys/amd64/amd64/pmap.c

1374

static bool pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,

sys/amd64/amd64/pmap.c

1382

static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,

sys/amd64/amd64/pmap.c

1384

static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,

sys/amd64/amd64/pmap.c

1386

static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);

sys/amd64/amd64/pmap.c

1387

static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,

sys/amd64/amd64/pmap.c

1389

static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/amd64/pmap.c

1392

static bool pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,

sys/amd64/amd64/pmap.c

1394

static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,

sys/amd64/amd64/pmap.c

1398

static pd_entry_t *pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp,

sys/amd64/amd64/pmap.c

1400

static vm_page_t pmap_allocpte_alloc(pmap_t pmap, vm_pindex_t ptepindex,

sys/amd64/amd64/pmap.c

1402

static vm_page_t pmap_allocpte_nosleep(pmap_t pmap, vm_pindex_t ptepindex,

sys/amd64/amd64/pmap.c

1404

static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,

sys/amd64/amd64/pmap.c

1407

static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/amd64/amd64/pmap.c

1447

pmap_pml5e(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1450

MPASS(pmap_is_la57(pmap));

sys/amd64/amd64/pmap.c

1451

return (&pmap->pm_pmltop[pmap_pml5e_index(va)]);

sys/amd64/amd64/pmap.c

1455

pmap_pml5e_u(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1458

MPASS(pmap_is_la57(pmap));

sys/amd64/amd64/pmap.c

1459

return (&pmap->pm_pmltopu[pmap_pml5e_index(va)]);

sys/amd64/amd64/pmap.c

1474

pmap_pml4e(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1480

if (pmap_is_la57(pmap)) {

sys/amd64/amd64/pmap.c

1481

pml5e = pmap_pml5e(pmap, va);

sys/amd64/amd64/pmap.c

1482

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

1487

pml4e = pmap->pm_pmltop;

sys/amd64/amd64/pmap.c

1493

pmap_pml4e_u(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1495

MPASS(!pmap_is_la57(pmap));

sys/amd64/amd64/pmap.c

1496

return (&pmap->pm_pmltopu[pmap_pml4e_index(va)]);

sys/amd64/amd64/pmap.c

1511

pmap_pdpe(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1516

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

1517

pml4e = pmap_pml4e(pmap, va);

sys/amd64/amd64/pmap.c

1537

pmap_pde(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1542

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

1543

pdpe = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

1547

("pmap_pde for 1G page, pmap %p va %#lx", pmap, va));

sys/amd64/amd64/pmap.c

1565

pmap_pte(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1570

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

1571

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

1580

pmap_resident_count_adj(pmap_t pmap, int count)

sys/amd64/amd64/pmap.c

1583

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

1584

KASSERT(pmap->pm_stats.resident_count + count >= 0,

sys/amd64/amd64/pmap.c

1585

("pmap %p resident count underflow %ld %d", pmap,

sys/amd64/amd64/pmap.c

1586

pmap->pm_stats.resident_count, count));

sys/amd64/amd64/pmap.c

1587

pmap->pm_stats.resident_count += count;

sys/amd64/amd64/pmap.c

1591

pmap_pt_page_count_pinit(pmap_t pmap, int count)

sys/amd64/amd64/pmap.c

1593

KASSERT(pmap->pm_stats.resident_count + count >= 0,

sys/amd64/amd64/pmap.c

1594

("pmap %p resident count underflow %ld %d", pmap,

sys/amd64/amd64/pmap.c

1595

pmap->pm_stats.resident_count, count));

sys/amd64/amd64/pmap.c

1596

pmap->pm_stats.resident_count += count;

sys/amd64/amd64/pmap.c

1600

pmap_pt_page_count_adj(pmap_t pmap, int count)

sys/amd64/amd64/pmap.c

1602

if (pmap == kernel_pmap)

sys/amd64/amd64/pmap.c

1605

if (pmap != NULL)

sys/amd64/amd64/pmap.c

1606

pmap_resident_count_adj(pmap, count);

sys/amd64/amd64/pmap.c

177

pmap_type_guest(pmap_t pmap)

sys/amd64/amd64/pmap.c

180

return ((pmap->pm_type == PT_EPT) || (pmap->pm_type == PT_RVI));

sys/amd64/amd64/pmap.c

184

pmap_emulate_ad_bits(pmap_t pmap)

sys/amd64/amd64/pmap.c

187

return ((pmap->pm_flags & PMAP_EMULATE_AD_BITS) != 0);

sys/amd64/amd64/pmap.c

191

pmap_valid_bit(pmap_t pmap)

sys/amd64/amd64/pmap.c

195

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

201

if (pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

207

panic("pmap_valid_bit: invalid pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

214

pmap_rw_bit(pmap_t pmap)

sys/amd64/amd64/pmap.c

218

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

224

if (pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

230

panic("pmap_rw_bit: invalid pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

2384

pmap_allow_2m_x_page(pmap_t pmap, bool executable)

sys/amd64/amd64/pmap.c

2387

return (pmap->pm_type != PT_EPT || !executable ||

sys/amd64/amd64/pmap.c

239

pmap_global_bit(pmap_t pmap)

sys/amd64/amd64/pmap.c

243

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

252

panic("pmap_global_bit: invalid pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

259

pmap_accessed_bit(pmap_t pmap)

sys/amd64/amd64/pmap.c

263

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

269

if (pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

2712

pmap_swap_pat(pmap_t pmap, pt_entry_t entry)

sys/amd64/amd64/pmap.c

2716

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

2734

panic("pmap_switch_pat_bits: bad pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

2741

pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)

sys/amd64/amd64/pmap.c

275

panic("pmap_accessed_bit: invalid pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

2753

pmap_cache_bits(pmap_t pmap, int mode, bool is_pde)

sys/amd64/amd64/pmap.c

2757

if (!pmap_is_valid_memattr(pmap, mode))

sys/amd64/amd64/pmap.c

2760

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

2784

panic("unsupported pmap type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

2791

pmap_cache_mask(pmap_t pmap, bool is_pde)

sys/amd64/amd64/pmap.c

2795

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

2804

panic("pmap_cache_mask: invalid pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

2811

pmap_pat_index(pmap_t pmap, pt_entry_t pte, bool is_pde)

sys/amd64/amd64/pmap.c

2816

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

282

pmap_modified_bit(pmap_t pmap)

sys/amd64/amd64/pmap.c

2846

pmap_ps_enabled(pmap_t pmap)

sys/amd64/amd64/pmap.c

2849

return (pg_ps_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0);

sys/amd64/amd64/pmap.c

2853

pmap_update_pde_store(pmap_t pmap, pd_entry_t *pde, pd_entry_t newpde)

sys/amd64/amd64/pmap.c

2856

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

286

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

2875

atomic_add_long(&pmap->pm_eptgen, 1);

sys/amd64/amd64/pmap.c

2878

panic("pmap_update_pde_store: bad pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

2891

pmap_update_pde_invalidate(pmap_t pmap, vm_offset_t va, pd_entry_t newpde)

sys/amd64/amd64/pmap.c

2895

if (pmap_type_guest(pmap))

sys/amd64/amd64/pmap.c

2898

KASSERT(pmap->pm_type == PT_X86,

sys/amd64/amd64/pmap.c

2899

("pmap_update_pde_invalidate: invalid type %d", pmap->pm_type));

sys/amd64/amd64/pmap.c

2901

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

2905

pmap_invlpg(pmap, va);

sys/amd64/amd64/pmap.c

292

if (pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

298

panic("pmap_modified_bit: invalid pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

305

pmap_pku_mask_bit(pmap_t pmap)

sys/amd64/amd64/pmap.c

308

return (pmap->pm_type == PT_X86 ? X86_PG_PKU_MASK : 0);

sys/amd64/amd64/pmap.c

3085

pmap_invalidate_ept(pmap_t pmap)

sys/amd64/amd64/pmap.c

3091

KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active),

sys/amd64/amd64/pmap.c

3116

atomic_add_long(&pmap->pm_eptgen, 1);

sys/amd64/amd64/pmap.c

3118

goal = smr_advance(pmap->pm_eptsmr);

sys/amd64/amd64/pmap.c

312

safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte)

sys/amd64/amd64/pmap.c

3123

ipinum = pmap->pm_flags & PMAP_NESTED_IPIMASK;

sys/amd64/amd64/pmap.c

3124

ipi_selected(pmap->pm_active, ipinum);

sys/amd64/amd64/pmap.c

3131

smr_wait(pmap->pm_eptsmr, goal);

sys/amd64/amd64/pmap.c

3135

pmap_invalidate_preipi_pcid(pmap_t pmap)

sys/amd64/amd64/pmap.c

3143

if (pmap != PCPU_GET(curpmap))

sys/amd64/amd64/pmap.c

3148

pcidp = zpcpu_get_cpu(pmap->pm_pcidp, i);

sys/amd64/amd64/pmap.c

315

if (!pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

3167

pmap_invalidate_preipi_nopcid(pmap_t pmap __unused)

sys/amd64/amd64/pmap.c

3179

pmap_invalidate_page_pcid_cb(pmap_t pmap, vm_offset_t va,

sys/amd64/amd64/pmap.c

318

KASSERT(pmap->pm_type == PT_EPT, ("invalid pm_type %d", pmap->pm_type));

sys/amd64/amd64/pmap.c

3199

if (pmap->pm_ucr3 == PMAP_NO_CR3 ||

sys/amd64/amd64/pmap.c

3203

pcid = pmap_get_pcid(pmap);

sys/amd64/amd64/pmap.c

3210

kcr3 = pmap->pm_cr3 | pcid | CR3_PCID_SAVE;

sys/amd64/amd64/pmap.c

3211

ucr3 = pmap->pm_ucr3 | pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;

sys/amd64/amd64/pmap.c

3217

pmap_invalidate_page_pcid_invpcid_cb(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

3219

pmap_invalidate_page_pcid_cb(pmap, va, true);

sys/amd64/amd64/pmap.c

3223

pmap_invalidate_page_pcid_noinvpcid_cb(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

3225

pmap_invalidate_page_pcid_cb(pmap, va, false);

sys/amd64/amd64/pmap.c

3229

pmap_invalidate_page_nopcid_cb(pmap_t pmap __unused, vm_offset_t va __unused)

sys/amd64/amd64/pmap.c

3242

pmap_invalidate_page_curcpu_cb(pmap_t pmap, vm_offset_t va,

sys/amd64/amd64/pmap.c

3245

if (pmap == kernel_pmap) {

sys/amd64/amd64/pmap.c

3247

} else if (pmap == PCPU_GET(curpmap)) {

sys/amd64/amd64/pmap.c

3249

pmap_invalidate_page_cb(pmap, va);

sys/amd64/amd64/pmap.c

3254

pmap_invalidate_page(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

3256

if (pmap_type_guest(pmap)) {

sys/amd64/amd64/pmap.c

3257

pmap_invalidate_ept(pmap);

sys/amd64/amd64/pmap.c

3261

KASSERT(pmap->pm_type == PT_X86,

sys/amd64/amd64/pmap.c

3262

("pmap_invalidate_page: invalid type %d", pmap->pm_type));

sys/amd64/amd64/pmap.c

3264

pmap_invalidate_preipi(pmap);

sys/amd64/amd64/pmap.c

3265

smp_masked_invlpg(va, pmap, pmap_invalidate_page_curcpu_cb);

sys/amd64/amd64/pmap.c

3272

pmap_invalidate_range_pcid_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/amd64/pmap.c

3281

if (pmap != PCPU_GET(curpmap) ||

sys/amd64/amd64/pmap.c

3282

pmap->pm_ucr3 == PMAP_NO_CR3 ||

sys/amd64/amd64/pmap.c

3286

pcid = pmap_get_pcid(pmap);

sys/amd64/amd64/pmap.c

3293

kcr3 = pmap->pm_cr3 | pcid | CR3_PCID_SAVE;

sys/amd64/amd64/pmap.c

3294

ucr3 = pmap->pm_ucr3 | pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;

sys/amd64/amd64/pmap.c

3300

pmap_invalidate_range_pcid_invpcid_cb(pmap_t pmap, vm_offset_t sva,

sys/amd64/amd64/pmap.c

3303

pmap_invalidate_range_pcid_cb(pmap, sva, eva, true);

sys/amd64/amd64/pmap.c

3307

pmap_invalidate_range_pcid_noinvpcid_cb(pmap_t pmap, vm_offset_t sva,

sys/amd64/amd64/pmap.c

3310

pmap_invalidate_range_pcid_cb(pmap, sva, eva, false);

sys/amd64/amd64/pmap.c

3314

pmap_invalidate_range_nopcid_cb(pmap_t pmap __unused, vm_offset_t sva __unused,

sys/amd64/amd64/pmap.c

332

((pmap->pm_flags & PMAP_SUPPORTS_EXEC_ONLY) != 0))

sys/amd64/amd64/pmap.c

3329

pmap_invalidate_range_curcpu_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

3333

if (pmap == kernel_pmap) {

sys/amd64/amd64/pmap.c

3342

} else if (pmap == PCPU_GET(curpmap)) {

sys/amd64/amd64/pmap.c

3345

pmap_invalidate_range_cb(pmap, sva, eva);

sys/amd64/amd64/pmap.c

3350

pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

3353

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

3357

if (pmap_type_guest(pmap)) {

sys/amd64/amd64/pmap.c

3358

pmap_invalidate_ept(pmap);

sys/amd64/amd64/pmap.c

3362

KASSERT(pmap->pm_type == PT_X86,

sys/amd64/amd64/pmap.c

3363

("pmap_invalidate_range: invalid type %d", pmap->pm_type));

sys/amd64/amd64/pmap.c

3365

pmap_invalidate_preipi(pmap);

sys/amd64/amd64/pmap.c

3366

smp_masked_invlpg_range(sva, eva, pmap,

sys/amd64/amd64/pmap.c

3371

pmap_invalidate_all_cb_template(pmap_t pmap, bool pmap_pcid_enabled1,

sys/amd64/amd64/pmap.c

3378

if (pmap == kernel_pmap) {

sys/amd64/amd64/pmap.c

3385

} else if (pmap == PCPU_GET(curpmap)) {

sys/amd64/amd64/pmap.c

3389

pcid = pmap_get_pcid(pmap);

sys/amd64/amd64/pmap.c

3396

kcr3 = pmap->pm_cr3 | pcid;

sys/amd64/amd64/pmap.c

3399

if (pmap->pm_ucr3 != PMAP_NO_CR3)

sys/amd64/amd64/pmap.c

3408

pmap_invalidate_all_pcid_invpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused,

sys/amd64/amd64/pmap.c

3411

pmap_invalidate_all_cb_template(pmap, true, true);

sys/amd64/amd64/pmap.c

3415

pmap_invalidate_all_pcid_noinvpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused,

sys/amd64/amd64/pmap.c

3418

pmap_invalidate_all_cb_template(pmap, true, false);

sys/amd64/amd64/pmap.c

3422

pmap_invalidate_all_nopcid_invpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused,

sys/amd64/amd64/pmap.c

3425

pmap_invalidate_all_cb_template(pmap, false, true);

sys/amd64/amd64/pmap.c

3429

pmap_invalidate_all_nopcid_noinvpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused,

sys/amd64/amd64/pmap.c

3432

pmap_invalidate_all_cb_template(pmap, false, false);

sys/amd64/amd64/pmap.c

3446

pmap_invalidate_all(pmap_t pmap)

sys/amd64/amd64/pmap.c

3448

if (pmap_type_guest(pmap)) {

sys/amd64/amd64/pmap.c

3449

pmap_invalidate_ept(pmap);

sys/amd64/amd64/pmap.c

3453

KASSERT(pmap->pm_type == PT_X86,

sys/amd64/amd64/pmap.c

3454

("pmap_invalidate_all: invalid type %d", pmap->pm_type));

sys/amd64/amd64/pmap.c

3456

pmap_invalidate_preipi(pmap);

sys/amd64/amd64/pmap.c

3457

smp_masked_invltlb(pmap, pmap_invalidate_all_curcpu_cb);

sys/amd64/amd64/pmap.c

3461

pmap_invalidate_cache_curcpu_cb(pmap_t pmap __unused, vm_offset_t va __unused,

sys/amd64/amd64/pmap.c

3476

pmap_t pmap;

sys/amd64/amd64/pmap.c

3489

pmap_update_pde_store(act->pmap, act->pde, act->newpde);

sys/amd64/amd64/pmap.c

3498

pmap_update_pde_invalidate(act->pmap, act->va, act->newpde);

sys/amd64/amd64/pmap.c

3510

pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)

sys/amd64/amd64/pmap.c

3520

if (pmap == kernel_pmap || pmap_type_guest(pmap))

sys/amd64/amd64/pmap.c

3523

active = pmap->pm_active;

sys/amd64/amd64/pmap.c

3529

act.pmap = pmap;

sys/amd64/amd64/pmap.c

3537

pmap_update_pde_store(pmap, pde, newpde);

sys/amd64/amd64/pmap.c

3539

pmap_update_pde_invalidate(pmap, va, newpde);

sys/amd64/amd64/pmap.c

3545

pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde)

sys/amd64/amd64/pmap.c

3560

pmap_invalidate_range(pmap, va, va + NBPDR - 1);

sys/amd64/amd64/pmap.c

3562

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

3749

pmap_extract(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

3757

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

3758

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

3759

pdpe = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

3777

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

3789

pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/amd64/amd64/pmap.c

3797

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

3798

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

3799

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

3801

pdpep = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

3831

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

4035

pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/amd64/amd64/pmap.c

4039

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

404

struct pmap kernel_pmap_store = {

sys/amd64/amd64/pmap.c

4043

return (vm_radix_insert(&pmap->pm_root, mpte));

sys/amd64/amd64/pmap.c

4053

pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

4056

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

4057

return (vm_radix_remove(&pmap->pm_root, pmap_pde_pindex(va)));

sys/amd64/amd64/pmap.c

4067

pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/amd64/amd64/pmap.c

4072

_pmap_unwire_ptp(pmap, va, m, free);

sys/amd64/amd64/pmap.c

4079

_pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/amd64/amd64/pmap.c

4087

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

4094

MPASS(pmap_is_la57(pmap));

sys/amd64/amd64/pmap.c

4095

pml5 = pmap_pml5e(pmap, va);

sys/amd64/amd64/pmap.c

4097

if (pmap->pm_pmltopu != NULL && va <= VM_MAXUSER_ADDRESS) {

sys/amd64/amd64/pmap.c

4098

pml5 = pmap_pml5e_u(pmap, va);

sys/amd64/amd64/pmap.c

4103

pml4 = pmap_pml4e(pmap, va);

sys/amd64/amd64/pmap.c

4105

if (!pmap_is_la57(pmap) && pmap->pm_pmltopu != NULL &&

sys/amd64/amd64/pmap.c

4107

pml4 = pmap_pml4e_u(pmap, va);

sys/amd64/amd64/pmap.c

4112

pdp = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

4116

pd = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

4121

pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME);

sys/amd64/amd64/pmap.c

4122

pmap_unwire_ptp(pmap, va, pdpg, free);

sys/amd64/amd64/pmap.c

4125

pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME);

sys/amd64/amd64/pmap.c

4126

pmap_unwire_ptp(pmap, va, pdppg, free);

sys/amd64/amd64/pmap.c

4127

} else if (m->pindex < NUPDE + NUPDPE + NUPML4E && pmap_is_la57(pmap)) {

sys/amd64/amd64/pmap.c

4129

pml4pg = PHYS_TO_VM_PAGE(*pmap_pml5e(pmap, va) & PG_FRAME);

sys/amd64/amd64/pmap.c

4130

pmap_unwire_ptp(pmap, va, pml4pg, free);

sys/amd64/amd64/pmap.c

4133

pmap_pt_page_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

4147

pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde,

sys/amd64/amd64/pmap.c

4156

return (pmap_unwire_ptp(pmap, va, mpte, free));

sys/amd64/amd64/pmap.c

4164

pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte)

sys/amd64/amd64/pmap.c

4169

if (pmap_unwire_ptp(pmap, va, mpte, &free)) {

sys/amd64/amd64/pmap.c

4175

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

4181

pmap_pinit_pcids(pmap_t pmap, uint32_t pcid, int gen)

sys/amd64/amd64/pmap.c

4187

pcidp = zpcpu_get_cpu(pmap->pm_pcidp, i);

sys/amd64/amd64/pmap.c

4194

pmap_pinit0(pmap_t pmap)

sys/amd64/amd64/pmap.c

4199

PMAP_LOCK_INIT(pmap);

sys/amd64/amd64/pmap.c

420

static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,

sys/amd64/amd64/pmap.c

4200

pmap->pm_pmltop = kernel_pmap->pm_pmltop;

sys/amd64/amd64/pmap.c

4201

pmap->pm_pmltopu = NULL;

sys/amd64/amd64/pmap.c

4202

pmap->pm_cr3 = kernel_pmap->pm_cr3;

sys/amd64/amd64/pmap.c

4204

pmap->pm_ucr3 = PMAP_NO_CR3;

sys/amd64/amd64/pmap.c

4205

vm_radix_init(&pmap->pm_root);

sys/amd64/amd64/pmap.c

4206

CPU_ZERO(&pmap->pm_active);

sys/amd64/amd64/pmap.c

4207

TAILQ_INIT(&pmap->pm_pvchunk);

sys/amd64/amd64/pmap.c

4208

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/amd64/amd64/pmap.c

4209

pmap->pm_flags = pmap_flags;

sys/amd64/amd64/pmap.c

4210

pmap->pm_pcidp = uma_zalloc_pcpu(pcpu_zone_8, M_WAITOK);

sys/amd64/amd64/pmap.c

4211

pmap_pinit_pcids(pmap, PMAP_PCID_KERN + 1, 1);

sys/amd64/amd64/pmap.c

4212

pmap_activate_boot(pmap);

sys/amd64/amd64/pmap.c

4316

pmap_alloc_pt_page(pmap_t pmap, vm_pindex_t pindex, int flags)

sys/amd64/amd64/pmap.c

4324

pmap_pt_page_count_adj(pmap, 1);

sys/amd64/amd64/pmap.c

4329

pmap_free_pt_page(pmap_t pmap, vm_page_t m, bool zerofilled)

sys/amd64/amd64/pmap.c

4345

pmap_pt_page_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

4355

pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags)

sys/amd64/amd64/pmap.c

4360

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/amd64/amd64/pmap.c

4377

pmap_pt_page_count_pinit(pmap, 1);

sys/amd64/amd64/pmap.c

4380

pmap->pm_pmltop = (pml5_entry_t *)PHYS_TO_DMAP(pmltop_phys);

sys/amd64/amd64/pmap.c

4383

if (pmap->pm_pcidp == NULL)

sys/amd64/amd64/pmap.c

4384

pmap->pm_pcidp = uma_zalloc_pcpu(pcpu_zone_8,

sys/amd64/amd64/pmap.c

4386

pmap_pinit_pcids(pmap, PMAP_PCID_NONE, 0);

sys/amd64/amd64/pmap.c

4388

pmap->pm_cr3 = PMAP_NO_CR3; /* initialize to an invalid value */

sys/amd64/amd64/pmap.c

4389

pmap->pm_ucr3 = PMAP_NO_CR3;

sys/amd64/amd64/pmap.c

4390

pmap->pm_pmltopu = NULL;

sys/amd64/amd64/pmap.c

4392

pmap->pm_type = pm_type;

sys/amd64/amd64/pmap.c

4402

pmap->pm_cr3 = pmltop_phys;

sys/amd64/amd64/pmap.c

4403

if (pmap_is_la57(pmap))

sys/amd64/amd64/pmap.c

4415

pmap_pt_page_count_pinit(pmap, 1);

sys/amd64/amd64/pmap.c

4416

pmap->pm_pmltopu = (pml4_entry_t *)PHYS_TO_DMAP(

sys/amd64/amd64/pmap.c

4418

if (pmap_is_la57(pmap))

sys/amd64/amd64/pmap.c

442

pmap_is_la57(pmap_t pmap)

sys/amd64/amd64/pmap.c

4422

pmap->pm_ucr3 = VM_PAGE_TO_PHYS(pmltop_pgu);

sys/amd64/amd64/pmap.c

4425

rangeset_init(&pmap->pm_pkru, pkru_dup_range,

sys/amd64/amd64/pmap.c

4426

pkru_free_range, pmap, M_NOWAIT);

sys/amd64/amd64/pmap.c

4431

pmap->pm_eptsmr = smr_create("pmap", 0, 0);

sys/amd64/amd64/pmap.c

4435

vm_radix_init(&pmap->pm_root);

sys/amd64/amd64/pmap.c

4436

CPU_ZERO(&pmap->pm_active);

sys/amd64/amd64/pmap.c

4437

TAILQ_INIT(&pmap->pm_pvchunk);

sys/amd64/amd64/pmap.c

4438

pmap->pm_flags = flags;

sys/amd64/amd64/pmap.c

4439

pmap->pm_eptgen = 0;

sys/amd64/amd64/pmap.c

444

if (pmap->pm_type == PT_X86)

sys/amd64/amd64/pmap.c

4445

pmap_pinit(pmap_t pmap)

sys/amd64/amd64/pmap.c

4448

return (pmap_pinit_type(pmap, PT_X86, pmap_flags));

sys/amd64/amd64/pmap.c

4452

pmap_allocpte_free_unref(pmap_t pmap, vm_offset_t va, pt_entry_t *pte)

sys/amd64/amd64/pmap.c

4461

_pmap_unwire_ptp(pmap, va, mpg, &free);

sys/amd64/amd64/pmap.c

4462

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

4467

pmap_allocpte_getpml4(pmap_t pmap, struct rwlock **lockp, vm_offset_t va,

sys/amd64/amd64/pmap.c

4477

if (!pmap_is_la57(pmap))

sys/amd64/amd64/pmap.c

4478

return (&pmap->pm_pmltop[pmap_pml4e_index(va)]);

sys/amd64/amd64/pmap.c

4480

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

4482

pml5 = &pmap->pm_pmltop[pml5index];

sys/amd64/amd64/pmap.c

4484

if (pmap_allocpte_nosleep(pmap, pmap_pml5e_pindex(va), lockp,

sys/amd64/amd64/pmap.c

4504

pmap_allocpte_getpdp(pmap_t pmap, struct rwlock **lockp, vm_offset_t va,

sys/amd64/amd64/pmap.c

4513

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

4515

pml4 = pmap_allocpte_getpml4(pmap, lockp, va, false);

sys/amd64/amd64/pmap.c

4521

if (pmap_allocpte_nosleep(pmap, pmap_pml4e_pindex(va), lockp,

sys/amd64/amd64/pmap.c

4523

if (pmap_is_la57(pmap))

sys/amd64/amd64/pmap.c

4524

pmap_allocpte_free_unref(pmap, va,

sys/amd64/amd64/pmap.c

4525

pmap_pml5e(pmap, va));

sys/amd64/amd64/pmap.c

4578

pmap_allocpte_nosleep(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp,

sys/amd64/amd64/pmap.c

4589

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

4591

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

4592

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

4593

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

4594

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

4599

m = pmap_alloc_pt_page(pmap, ptepindex,

sys/amd64/amd64/pmap.c

4609

MPASS(pmap_is_la57(pmap));

sys/amd64/amd64/pmap.c

4612

pml5 = &pmap->pm_pmltop[pml5index];

sys/amd64/amd64/pmap.c

4614

("pmap %p va %#lx pml5 %#lx", pmap, va, *pml5));

sys/amd64/amd64/pmap.c

4617

if (pmap->pm_pmltopu != NULL && pml5index < NUPML5E) {

sys/amd64/amd64/pmap.c

4618

MPASS(pmap->pm_ucr3 != PMAP_NO_CR3);

sys/amd64/amd64/pmap.c

4621

pml5u = &pmap->pm_pmltopu[pml5index];

sys/amd64/amd64/pmap.c

4628

pml4 = pmap_allocpte_getpml4(pmap, lockp, va, true);

sys/amd64/amd64/pmap.c

4630

pmap_free_pt_page(pmap, m, true);

sys/amd64/amd64/pmap.c

4634

("pmap %p va %#lx pml4 %#lx", pmap, va, *pml4));

sys/amd64/amd64/pmap.c

4637

if (!pmap_is_la57(pmap) && pmap->pm_pmltopu != NULL &&

sys/amd64/amd64/pmap.c

4639

MPASS(pmap->pm_ucr3 != PMAP_NO_CR3);

sys/amd64/amd64/pmap.c

4650

pml4u = &pmap->pm_pmltopu[pml4index];

sys/amd64/amd64/pmap.c

4656

pdp = pmap_allocpte_getpdp(pmap, lockp, va, true);

sys/amd64/amd64/pmap.c

4658

pmap_free_pt_page(pmap, m, true);

sys/amd64/amd64/pmap.c

4662

("pmap %p va %#lx pdp %#lx", pmap, va, *pdp));

sys/amd64/amd64/pmap.c

4666

pdp = pmap_allocpte_getpdp(pmap, lockp, va, false);

sys/amd64/amd64/pmap.c

4668

pmap_free_pt_page(pmap, m, true);

sys/amd64/amd64/pmap.c

4673

if (pmap_allocpte_nosleep(pmap, pmap_pdpe_pindex(va),

sys/amd64/amd64/pmap.c

4675

pmap_allocpte_free_unref(pmap, va,

sys/amd64/amd64/pmap.c

4676

pmap_pml4e(pmap, va));

sys/amd64/amd64/pmap.c

4677

pmap_free_pt_page(pmap, m, true);

sys/amd64/amd64/pmap.c

4690

("pmap %p va %#lx pd %#lx", pmap, va, *pd));

sys/amd64/amd64/pmap.c

4707

pmap_allocpte_alloc(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp,

sys/amd64/amd64/pmap.c

4712

m = pmap_allocpte_nosleep(pmap, ptepindex, lockp, va);

sys/amd64/amd64/pmap.c

4715

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

4718

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

4724

pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp,

sys/amd64/amd64/pmap.c

4732

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

4735

pdpe = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

4747

pdpg = pmap_allocpte_alloc(pmap, NUPDE + pdpindex, lockp, va);

sys/amd64/amd64/pmap.c

4764

pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)

sys/amd64/amd64/pmap.c

4770

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

4780

pd = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

4787

if (!pmap_demote_pde_locked(pmap, pd, va, lockp)) {

sys/amd64/amd64/pmap.c

4808

m = pmap_allocpte_alloc(pmap, ptepindex, lockp, va);

sys/amd64/amd64/pmap.c

4825

pmap_release(pmap_t pmap)

sys/amd64/amd64/pmap.c

4830

KASSERT(vm_radix_is_empty(&pmap->pm_root),

sys/amd64/amd64/pmap.c

4832

pmap));

sys/amd64/amd64/pmap.c

4833

KASSERT(CPU_EMPTY(&pmap->pm_active),

sys/amd64/amd64/pmap.c

4834

("releasing active pmap %p", pmap));

sys/amd64/amd64/pmap.c

4836

m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_pmltop));

sys/amd64/amd64/pmap.c

4838

if (pmap_is_la57(pmap)) {

sys/amd64/amd64/pmap.c

4840

pmap->pm_pmltop[i] = 0;

sys/amd64/amd64/pmap.c

4843

pmap->pm_pmltop[KPML4BASE + i] = 0;

sys/amd64/amd64/pmap.c

4846

pmap->pm_pmltop[KASANPML4I + i] = 0;

sys/amd64/amd64/pmap.c

4850

pmap->pm_pmltop[KMSANSHADPML4I + i] = 0;

sys/amd64/amd64/pmap.c

4852

pmap->pm_pmltop[KMSANORIGPML4I + i] = 0;

sys/amd64/amd64/pmap.c

4855

pmap->pm_pmltop[DMPML4I + i] = 0;

sys/amd64/amd64/pmap.c

4856

pmap->pm_pmltop[PML4PML4I] = 0; /* Recursive Mapping */

sys/amd64/amd64/pmap.c

4858

pmap->pm_pmltop[LMSPML4I + i] = 0;

sys/amd64/amd64/pmap.c

4862

pmap_pt_page_count_pinit(pmap, -1);

sys/amd64/amd64/pmap.c

4864

if (pmap->pm_pmltopu != NULL) {

sys/amd64/amd64/pmap.c

4865

m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->

sys/amd64/amd64/pmap.c

4868

pmap_pt_page_count_pinit(pmap, -1);

sys/amd64/amd64/pmap.c

4870

if (pmap->pm_type == PT_X86 &&

sys/amd64/amd64/pmap.c

4872

rangeset_fini(&pmap->pm_pkru);

sys/amd64/amd64/pmap.c

4874

KASSERT(pmap->pm_stats.resident_count == 0,

sys/amd64/amd64/pmap.c

4876

pmap, pmap->pm_stats.resident_count));

sys/amd64/amd64/pmap.c

5173

reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap, bool start_di)

sys/amd64/amd64/pmap.c

5176

if (pmap == NULL)

sys/amd64/amd64/pmap.c

5178

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

5179

if (pmap != locked_pmap)

sys/amd64/amd64/pmap.c

5180

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

5204

pmap_t next_pmap, pmap;

sys/amd64/amd64/pmap.c

5217

pmap = NULL;

sys/amd64/amd64/pmap.c

5257

if (pmap != next_pmap) {

sys/amd64/amd64/pmap.c

5259

reclaim_pv_chunk_leave_pmap(pmap, locked_pmap,

sys/amd64/amd64/pmap.c

5261

pmap = next_pmap;

sys/amd64/amd64/pmap.c

5263

if (pmap > locked_pmap) {

sys/amd64/amd64/pmap.c

5265

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

5270

} else if (pmap != locked_pmap) {

sys/amd64/amd64/pmap.c

5271

if (PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

5277

pmap = NULL; /* pmap is not locked */

sys/amd64/amd64/pmap.c

5287

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

5288

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

5289

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

5290

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

5305

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

5313

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

5332

pmap_unuse_pt(pmap, va, *pde, &free);

sys/amd64/amd64/pmap.c

5341

pmap_resident_count_adj(pmap, -freed);

sys/amd64/amd64/pmap.c

5345

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5357

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5360

if (pmap == locked_pmap)

sys/amd64/amd64/pmap.c

5365

if (pvc->active_reclaims == 1 && pmap != NULL) {

sys/amd64/amd64/pmap.c

5384

reclaim_pv_chunk_leave_pmap(pmap, locked_pmap, start_di);

sys/amd64/amd64/pmap.c

5416

free_pv_entry(pmap_t pmap, pv_entry_t pv)

sys/amd64/amd64/pmap.c

5421

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5432

if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {

sys/amd64/amd64/pmap.c

5433

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5434

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5438

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5504

get_pv_entry(pmap_t pmap, struct rwlock **lockp)

sys/amd64/amd64/pmap.c

5512

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5515

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/amd64/amd64/pmap.c

5528

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5529

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,

sys/amd64/amd64/pmap.c

5544

m = reclaim_pv_chunk(pmap, lockp);

sys/amd64/amd64/pmap.c

5553

pc->pc_pmap = pmap;

sys/amd64/amd64/pmap.c

5562

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5604

reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)

sys/amd64/amd64/pmap.c

5613

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5626

TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {

sys/amd64/amd64/pmap.c

5643

m = reclaim_pv_chunk(pmap, lockp);

sys/amd64/amd64/pmap.c

5653

pc->pc_pmap = pmap;

sys/amd64/amd64/pmap.c

5657

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5686

pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

5691

if (pmap == PV_PMAP(pv) && va == pv->pv_va) {

sys/amd64/amd64/pmap.c

5706

pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/amd64/amd64/pmap.c

5716

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5728

pv = pmap_pvh_remove(pvh, pmap, va);

sys/amd64/amd64/pmap.c

5737

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/amd64/amd64/pmap.c

5755

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5756

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5760

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5761

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5774

pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/amd64/amd64/pmap.c

5795

pv = pmap_pvh_remove(&m->md, pmap, va);

sys/amd64/amd64/pmap.c

5805

pmap_pvh_free(&m->md, pmap, va);

sys/amd64/amd64/pmap.c

5816

pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

5820

pv = pmap_pvh_remove(pvh, pmap, va);

sys/amd64/amd64/pmap.c

5822

free_pv_entry(pmap, pv);

sys/amd64/amd64/pmap.c

5830

pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/amd64/amd64/pmap.c

5835

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5837

if ((pv = get_pv_entry(pmap, NULL)) != NULL) {

sys/amd64/amd64/pmap.c

5853

pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, u_int flags,

sys/amd64/amd64/pmap.c

5860

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5862

if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?

sys/amd64/amd64/pmap.c

5893

pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)

sys/amd64/amd64/pmap.c

5899

rv = pmap_demote_pde_locked(pmap, pde, va, &lock);

sys/amd64/amd64/pmap.c

5933

pmap_demote_pde_abort(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,

sys/amd64/amd64/pmap.c

5941

pmap_remove_pde(pmap, pde, sva, true, &free, lockp);

sys/amd64/amd64/pmap.c

5942

if ((oldpde & pmap_global_bit(pmap)) == 0)

sys/amd64/amd64/pmap.c

5943

pmap_invalidate_pde_page(pmap, sva, oldpde);

sys/amd64/amd64/pmap.c

5946

va, pmap);

sys/amd64/amd64/pmap.c

5950

pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,

sys/amd64/amd64/pmap.c

5953

return (pmap_demote_pde_mpte(pmap, pde, va, lockp, NULL));

sys/amd64/amd64/pmap.c

5957

pmap_demote_pde_mpte(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,

sys/amd64/amd64/pmap.c

5967

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

5968

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

5969

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

5970

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

5971

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

5972

PG_PTE_CACHE = pmap_cache_mask(pmap, false);

sys/amd64/amd64/pmap.c

5973

PG_PKU_MASK = pmap_pku_mask_bit(pmap);

sys/amd64/amd64/pmap.c

5975

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5989

pmap_demote_pde_abort(pmap, va, pde, oldpde,

sys/amd64/amd64/pmap.c

5993

mpte = pmap_remove_pt_page(pmap, va);

sys/amd64/amd64/pmap.c

5996

} else if ((mpte = pmap_remove_pt_page(pmap, va)) == NULL) {

sys/amd64/amd64/pmap.c

6019

mpte = pmap_alloc_pt_page(pmap, pmap_pde_pindex(va),

sys/amd64/amd64/pmap.c

6028

pmap_demote_pde_abort(pmap, va, pde, oldpde,

sys/amd64/amd64/pmap.c

6043

newpte = pmap_swap_pat(pmap, newpte);

sys/amd64/amd64/pmap.c

6070

reserve_pv_entries(pmap, NPTEPG - 1, lockp);

sys/amd64/amd64/pmap.c

6080

pmap_update_pde(pmap, va, pde, newpde);

sys/amd64/amd64/pmap.c

6088

pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));

sys/amd64/amd64/pmap.c

6094

pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME, lockp);

sys/amd64/amd64/pmap.c

6098

va, pmap);

sys/amd64/amd64/pmap.c

6106

pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)

sys/amd64/amd64/pmap.c

6112

KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));

sys/amd64/amd64/pmap.c

6113

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6114

mpte = pmap_remove_pt_page(pmap, va);

sys/amd64/amd64/pmap.c

6131

pmap_update_pde(pmap, va, pde, newpde);

sys/amd64/amd64/pmap.c

6138

pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));

sys/amd64/amd64/pmap.c

6145

pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, bool demote_kpde,

sys/amd64/amd64/pmap.c

6154

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

6155

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

6156

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

6157

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

6159

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6164

pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;

sys/amd64/amd64/pmap.c

6167

pmap_resident_count_adj(pmap, -NBPDR / PAGE_SIZE);

sys/amd64/amd64/pmap.c

6171

pmap_pvh_free(pvh, pmap, sva);

sys/amd64/amd64/pmap.c

6185

if (pmap != kernel_pmap) {

sys/amd64/amd64/pmap.c

6186

mpte = pmap_remove_pt_page(pmap, sva);

sys/amd64/amd64/pmap.c

6190

pmap_pt_page_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

6197

pmap_remove_kernel_pde(pmap, pdq, sva);

sys/amd64/amd64/pmap.c

6199

mpte = vm_radix_lookup(&pmap->pm_root, pmap_pde_pindex(sva));

sys/amd64/amd64/pmap.c

6205

return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free));

sys/amd64/amd64/pmap.c

6212

pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,

sys/amd64/amd64/pmap.c

6219

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

6220

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

6221

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

6223

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6226

pmap->pm_stats.wired_count -= 1;

sys/amd64/amd64/pmap.c

6227

pmap_resident_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

6235

pmap_pvh_free(&m->md, pmap, va);

sys/amd64/amd64/pmap.c

6244

return (pmap_unuse_pt(pmap, va, ptepde, free));

sys/amd64/amd64/pmap.c

6251

pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,

sys/amd64/amd64/pmap.c

6257

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

6258

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6265

pmap_remove_pte(pmap, pte, va, *pde, free, &lock);

sys/amd64/amd64/pmap.c

6268

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

6275

pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/amd64/pmap.c

6282

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6283

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

6290

pmap_invalidate_range(pmap, va, sva);

sys/amd64/amd64/pmap.c

6299

if (pmap_remove_pte(pmap, pte, sva, *pde, free, lockp)) {

sys/amd64/amd64/pmap.c

6305

pmap_invalidate_range(pmap, va, sva);

sys/amd64/amd64/pmap.c

6310

pmap_remove1(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, bool map_delete)

sys/amd64/amd64/pmap.c

6323

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

6324

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

6333

if (pmap->pm_stats.resident_count <= 1 + (pmap->pm_pmltopu != NULL ?

sys/amd64/amd64/pmap.c

6341

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

6343

pmap_pkru_on_remove(pmap, sva, eva);

sys/amd64/amd64/pmap.c

6351

pde = pmap_pde(pmap, sva);

sys/amd64/amd64/pmap.c

6353

pmap_remove_page(pmap, sva, pde, &free);

sys/amd64/amd64/pmap.c

6360

if (pmap->pm_stats.resident_count == 0)

sys/amd64/amd64/pmap.c

6363

if (pmap_is_la57(pmap)) {

sys/amd64/amd64/pmap.c

6364

pml5e = pmap_pml5e(pmap, sva);

sys/amd64/amd64/pmap.c

637

static bool pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/amd64/pmap.c

6373

pml4e = pmap_pml4e(pmap, sva);

sys/amd64/amd64/pmap.c

639

static pt_entry_t pmap_pkru_get(pmap_t pmap, vm_offset_t va);

sys/amd64/amd64/pmap.c

6393

MPASS(pmap != kernel_pmap); /* XXXKIB */

sys/amd64/amd64/pmap.c

6397

pmap_resident_count_adj(pmap, -NBPDP / PAGE_SIZE);

sys/amd64/amd64/pmap.c

6398

mt = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, sva) & PG_FRAME);

sys/amd64/amd64/pmap.c

6399

pmap_unwire_ptp(pmap, sva, mt, &free);

sys/amd64/amd64/pmap.c

640

static void pmap_pkru_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);

sys/amd64/amd64/pmap.c

6434

pmap_remove_pde(pmap, pde, sva, true, &free,

sys/amd64/amd64/pmap.c

6437

} else if (!pmap_demote_pde_locked(pmap, pde, sva,

sys/amd64/amd64/pmap.c

644

static int pmap_pkru_deassign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);

sys/amd64/amd64/pmap.c

645

static void pmap_pkru_deassign_all(pmap_t pmap);

sys/amd64/amd64/pmap.c

6453

if (pmap_remove_ptes(pmap, sva, va_next, pde, &free, &lock))

sys/amd64/amd64/pmap.c

6460

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

6461

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

6473

pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

6475

pmap_remove1(pmap, sva, eva, false);

sys/amd64/amd64/pmap.c

6485

pmap_map_delete(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

6487

pmap_remove1(pmap, sva, eva, true);

sys/amd64/amd64/pmap.c

6508

pmap_t pmap;

sys/amd64/amd64/pmap.c

6525

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

6526

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

6529

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

6532

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

6537

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

6538

(void)pmap_demote_pde_locked(pmap, pde, va, &lock);

sys/amd64/amd64/pmap.c

6539

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

6542

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

6543

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

6547

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

6550

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

6554

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

6555

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

6556

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

6557

pmap_resident_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

6558

pde = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

6564

pmap->pm_stats.wired_count--;

sys/amd64/amd64/pmap.c

6573

pmap_unuse_pt(pmap, pv->pv_va, *pde, &free);

sys/amd64/amd64/pmap.c

6574

pmap_invalidate_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

6577

free_pv_entry(pmap, pv);

sys/amd64/amd64/pmap.c

6578

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

6590

pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)

sys/amd64/amd64/pmap.c

6597

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

6598

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

6599

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

6601

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6639

pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)

sys/amd64/amd64/pmap.c

6652

pmap_remove(pmap, sva, eva);

sys/amd64/amd64/pmap.c

6660

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

6661

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

6662

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

6663

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

6686

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

6688

pml4e = pmap_pml4e(pmap, sva);

sys/amd64/amd64/pmap.c

6710

MPASS(pmap != kernel_pmap); /* XXXKIB */

sys/amd64/amd64/pmap.c

6750

if (pmap_protect_pde(pmap, pde, sva, prot))

sys/amd64/amd64/pmap.c

6764

if (ptpaddr == pbits || !pmap_demote_pde(pmap, pde,

sys/amd64/amd64/pmap.c

6800

pmap_invalidate_page(pmap, sva);

sys/amd64/amd64/pmap.c

6807

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

6808

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

6812

pmap_pde_ept_executable(pmap_t pmap, pd_entry_t pde)

sys/amd64/amd64/pmap.c

6815

if (pmap->pm_type != PT_EPT)

sys/amd64/amd64/pmap.c

6829

pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, vm_page_t mpte,

sys/amd64/amd64/pmap.c

6837

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6838

if (!pmap_ps_enabled(pmap))

sys/amd64/amd64/pmap.c

6841

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

6842

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

6843

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

6844

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

6845

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

6846

PG_PKU_MASK = pmap_pku_mask_bit(pmap);

sys/amd64/amd64/pmap.c

6847

PG_PTE_CACHE = pmap_cache_mask(pmap, false);

sys/amd64/amd64/pmap.c

6856

if (!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap, newpde)))

sys/amd64/amd64/pmap.c

6861

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

6891

" in pmap %p", va & ~PDRMASK, pmap);

sys/amd64/amd64/pmap.c

6906

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

6920

(va & ~PDRMASK), pmap);

sys/amd64/amd64/pmap.c

6925

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

6944

KASSERT((newpde & PG_A) != 0 || safe_to_clear_referenced(pmap, newpde),

sys/amd64/amd64/pmap.c

6962

if (pmap_insert_pt_page(pmap, mpte, true, allpte_PG_A != 0)) {

sys/amd64/amd64/pmap.c

6966

pmap);

sys/amd64/amd64/pmap.c

6974

pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME, lockp);

sys/amd64/amd64/pmap.c

6979

newpde = pmap_swap_pat(pmap, newpde);

sys/amd64/amd64/pmap.c

6985

pmap_update_pde(pmap, va, pde, PG_PS | newpde);

sys/amd64/amd64/pmap.c

6991

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

6997

pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags,

sys/amd64/amd64/pmap.c

7003

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

7016

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7020

if (!pmap_pkru_same(pmap, va, va + pagesizes[psind], &pten))

sys/amd64/amd64/pmap.c

7024

pml4e = pmap_pml4e(pmap, va);

sys/amd64/amd64/pmap.c

7026

mp = pmap_allocpte_alloc(pmap, pmap_pml4e_pindex(va),

sys/amd64/amd64/pmap.c

7045

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

7047

mp = pmap_allocpte_alloc(pmap, pmap_pdpe_pindex(va),

sys/amd64/amd64/pmap.c

7058

pdpe = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

7071

pmap->pm_stats.wired_count += pagesizes[psind] / PAGE_SIZE;

sys/amd64/amd64/pmap.c

7073

pmap->pm_stats.wired_count -= pagesizes[psind] / PAGE_SIZE;

sys/amd64/amd64/pmap.c

7075

pmap_resident_count_adj(pmap, pagesizes[psind] / PAGE_SIZE);

sys/amd64/amd64/pmap.c

7082

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

7084

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

7105

pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/amd64/amd64/pmap.c

7118

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

7119

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

7120

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

7121

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7122

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

7149

if (pmap == kernel_pmap)

sys/amd64/amd64/pmap.c

7151

newpte |= pmap_cache_bits(pmap, m->md.pat_mode, psind > 0);

sys/amd64/amd64/pmap.c

7165

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

7169

rv = pmap_enter_largepage(pmap, va, newpte | PG_PS, flags,

sys/amd64/amd64/pmap.c

7177

rv = pmap_enter_pde(pmap, va, newpte | PG_PS, flags, m, &lock);

sys/amd64/amd64/pmap.c

7187

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

7189

pmap_demote_pde_locked(pmap, pde, va, &lock))) {

sys/amd64/amd64/pmap.c

7201

mpte = pmap_allocpte_alloc(pmap, pmap_pde_pindex(va),

sys/amd64/amd64/pmap.c

7213

if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86)

sys/amd64/amd64/pmap.c

7214

newpte |= pmap_pkru_get(pmap, va);

sys/amd64/amd64/pmap.c

7227

pmap->pm_stats.wired_count++;

sys/amd64/amd64/pmap.c

7229

pmap->pm_stats.wired_count--;

sys/amd64/amd64/pmap.c

7281

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

7285

pv = pmap_pvh_remove(&om->md, pmap, va);

sys/amd64/amd64/pmap.c

7289

free_pv_entry(pmap, pv);

sys/amd64/amd64/pmap.c

7300

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

7308

pmap->pm_stats.wired_count++;

sys/amd64/amd64/pmap.c

7309

pmap_resident_count_adj(pmap, 1);

sys/amd64/amd64/pmap.c

7317

pv = get_pv_entry(pmap, &lock);

sys/amd64/amd64/pmap.c

7352

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

7366

(void)pmap_promote_pde(pmap, pde, va, mpte, &lock);

sys/amd64/amd64/pmap.c

7373

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

7384

pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/amd64/amd64/pmap.c

7390

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

7391

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7393

pmap_cache_bits(pmap, m->md.pat_mode, true) | PG_PS | PG_V;

sys/amd64/amd64/pmap.c

7400

return (pmap_enter_pde(pmap, va, newpde, PMAP_ENTER_NOSLEEP |

sys/amd64/amd64/pmap.c

7441

pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags,

sys/amd64/amd64/pmap.c

7450

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

7451

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

7452

KASSERT((newpde & (pmap_modified_bit(pmap) | PG_RW)) != PG_RW,

sys/amd64/amd64/pmap.c

7457

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7458

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

7460

if (!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap,

sys/amd64/amd64/pmap.c

7463

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

7466

if ((pde = pmap_alloc_pde(pmap, va, &pdpg, (flags &

sys/amd64/amd64/pmap.c

7469

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

7478

if (!pmap_pkru_same(pmap, va, va + NBPDR, &newpde)) {

sys/amd64/amd64/pmap.c

7479

pmap_abort_ptp(pmap, va, pdpg);

sys/amd64/amd64/pmap.c

7496

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

7504

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

7518

(void)pmap_remove_pde(pmap, pde, va, false, &free,

sys/amd64/amd64/pmap.c

7521

pmap_invalidate_pde_page(pmap, va, oldpde);

sys/amd64/amd64/pmap.c

7530

if (pmap_insert_pt_page(pmap, mt, false,

sys/amd64/amd64/pmap.c

7544

if (pmap_remove_ptes(pmap, va, va + NBPDR, pde, &free,

sys/amd64/amd64/pmap.c

7546

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

7563

if ((newpde & PG_W) != 0 && pmap != kernel_pmap) {

sys/amd64/amd64/pmap.c

7564

uwptpg = pmap_alloc_pt_page(pmap, pmap_pde_pindex(va),

sys/amd64/amd64/pmap.c

7567

pmap_abort_ptp(pmap, va, pdpg);

sys/amd64/amd64/pmap.c

7570

if (pmap_insert_pt_page(pmap, uwptpg, true, false)) {

sys/amd64/amd64/pmap.c

7571

pmap_free_pt_page(pmap, uwptpg, false);

sys/amd64/amd64/pmap.c

7572

pmap_abort_ptp(pmap, va, pdpg);

sys/amd64/amd64/pmap.c

7582

if (!pmap_pv_insert_pde(pmap, va, newpde, flags, lockp)) {

sys/amd64/amd64/pmap.c

7584

pmap_abort_ptp(pmap, va, pdpg);

sys/amd64/amd64/pmap.c

7589

mt = pmap_remove_pt_page(pmap, va);

sys/amd64/amd64/pmap.c

7594

mt = pmap_remove_pt_page(pmap, va);

sys/amd64/amd64/pmap.c

7599

pmap_free_pt_page(pmap, uwptpg, false);

sys/amd64/amd64/pmap.c

7602

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

7615

pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE;

sys/amd64/amd64/pmap.c

7616

pmap_resident_count_adj(pmap, NBPDR / PAGE_SIZE);

sys/amd64/amd64/pmap.c

7626

va, pmap);

sys/amd64/amd64/pmap.c

7643

pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,

sys/amd64/amd64/pmap.c

7659

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

7663

m->psind == 1 && pmap_ps_enabled(pmap) &&

sys/amd64/amd64/pmap.c

7664

((rv = pmap_enter_2mpage(pmap, va, m, prot, &lock)) ==

sys/amd64/amd64/pmap.c

7668

mpte = pmap_enter_quick_locked(pmap, va, m, prot,

sys/amd64/amd64/pmap.c

7675

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

7688

pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/amd64/amd64/pmap.c

7693

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

7694

(void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock);

sys/amd64/amd64/pmap.c

7697

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

7701

pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/amd64/amd64/pmap.c

7710

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7711

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

7737

pdpe = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

7748

mpte = pmap_allocpte_alloc(pmap,

sys/amd64/amd64/pmap.c

7754

mpte = pmap_allocpte_alloc(pmap, ptepindex,

sys/amd64/amd64/pmap.c

7776

!pmap_try_insert_pv_entry(pmap, va, m, lockp)) {

sys/amd64/amd64/pmap.c

7778

pmap_abort_ptp(pmap, va, mpte);

sys/amd64/amd64/pmap.c

7785

pmap_resident_count_adj(pmap, 1);

sys/amd64/amd64/pmap.c

7788

pmap_cache_bits(pmap, m->md.pat_mode, false);

sys/amd64/amd64/pmap.c

7794

newpte |= PG_U | pmap_pkru_get(pmap, va);

sys/amd64/amd64/pmap.c

7807

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

7813

if (pmap_promote_pde(pmap, pde, va, mpte, lockp))

sys/amd64/amd64/pmap.c

7842

pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,

sys/amd64/amd64/pmap.c

7852

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

7853

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

7854

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7855

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

7861

if (!pmap_ps_enabled(pmap))

sys/amd64/amd64/pmap.c

7899

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

7900

for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, true);

sys/amd64/amd64/pmap.c

7902

pde = pmap_alloc_pde(pmap, addr, &pdpg, NULL);

sys/amd64/amd64/pmap.c

7917

pmap_resident_count_adj(pmap, NBPDR / PAGE_SIZE);

sys/amd64/amd64/pmap.c

7928

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

7945

pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

7953

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7954

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

7955

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

7957

pml4e = pmap_pml4e(pmap, sva);

sys/amd64/amd64/pmap.c

7976

MPASS(pmap != kernel_pmap); /* XXXKIB */

sys/amd64/amd64/pmap.c

7979

pmap->pm_stats.wired_count -= NBPDP / PAGE_SIZE;

sys/amd64/amd64/pmap.c

8000

pmap->pm_stats.wired_count -= NBPDR /

sys/amd64/amd64/pmap.c

8003

} else if (!pmap_demote_pde(pmap, pde, sva))

sys/amd64/amd64/pmap.c

8022

pmap->pm_stats.wired_count--;

sys/amd64/amd64/pmap.c

8025

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8337

pmap_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/amd64/amd64/pmap.c

8351

if (PV_PMAP(pv) == pmap) {

sys/amd64/amd64/pmap.c

8362

if (PV_PMAP(pv) == pmap) {

sys/amd64/amd64/pmap.c

8386

pmap_t pmap;

sys/amd64/amd64/pmap.c

8398

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8399

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8402

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8405

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8409

pte = pmap_pte(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8412

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8417

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8418

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8422

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8426

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8430

pte = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8433

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8487

pmap_remove_pages(pmap_t pmap)

sys/amd64/amd64/pmap.c

8513

KASSERT(pmap == PCPU_GET(curpmap), ("non-current pmap %p", pmap));

sys/amd64/amd64/pmap.c

8521

CPU_AND(&other_cpus, &other_cpus, &pmap->pm_active);

sys/amd64/amd64/pmap.c

8523

KASSERT(CPU_EMPTY(&other_cpus), ("pmap active %p", pmap));

sys/amd64/amd64/pmap.c

8528

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

8529

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

8530

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

8535

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8536

TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {

sys/amd64/amd64/pmap.c

8550

pte = pmap_pdpe(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8628

pmap_resident_count_adj(pmap, -NBPDR / PAGE_SIZE);

sys/amd64/amd64/pmap.c

8638

mpte = pmap_remove_pt_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8642

pmap_pt_page_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

8649

pmap_resident_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

8660

pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);

sys/amd64/amd64/pmap.c

8670

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

8676

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

8677

pmap_pkru_deassign_all(pmap);

sys/amd64/amd64/pmap.c

8679

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8691

pmap_t pmap;

sys/amd64/amd64/pmap.c

8700

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8701

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8704

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8707

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8711

pte = pmap_pte(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8714

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

8715

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

8719

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

8720

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

8724

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8731

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8732

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8736

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8740

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8744

pte = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8747

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

8748

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

8752

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

8753

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

8757

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8795

pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/amd64/amd64/pmap.c

8801

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

8808

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8809

pde = pmap_pde(pmap, addr);

sys/amd64/amd64/pmap.c

8814

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8840

pmap_t pmap;

sys/amd64/amd64/pmap.c

8861

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8862

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8865

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8868

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8872

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

8874

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

8876

(void)pmap_demote_pde_locked(pmap, pde, va, &lock);

sys/amd64/amd64/pmap.c

8880

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8883

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8884

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8888

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8892

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8896

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

8897

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

8898

pde = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8910

pmap_invalidate_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8912

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8944

pmap_t pmap;

sys/amd64/amd64/pmap.c

8970

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8971

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8974

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8977

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8981

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

8982

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

8983

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

8985

pde = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9015

(uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&

sys/amd64/amd64/pmap.c

9017

if (safe_to_clear_referenced(pmap, oldpde)) {

sys/amd64/amd64/pmap.c

9019

pmap_invalidate_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9021

} else if (pmap_demote_pde_locked(pmap, pde,

sys/amd64/amd64/pmap.c

9035

pmap_remove_pte(pmap, pte, va, *pde,

sys/amd64/amd64/pmap.c

9037

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

9058

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9075

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

9076

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

9080

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

9083

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9087

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

9088

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

9089

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

9090

pde = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9098

if (safe_to_clear_referenced(pmap, *pte)) {

sys/amd64/amd64/pmap.c

9100

pmap_invalidate_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9109

pmap_remove_pte(pmap, pte, pv->pv_va,

sys/amd64/amd64/pmap.c

9111

pmap_invalidate_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9122

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9143

pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)

sys/amd64/amd64/pmap.c

9163

if (pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

9166

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

9167

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

9168

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

9169

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

9170

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

9173

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

9175

pml4e = pmap_pml4e(pmap, sva);

sys/amd64/amd64/pmap.c

9203

if (!pmap_demote_pde_locked(pmap, pde, sva, &lock)) {

sys/amd64/amd64/pmap.c

9232

pmap_remove_pte(pmap, pte, va, *pde, NULL,

sys/amd64/amd64/pmap.c

9270

pmap_invalidate_range(pmap, va, sva);

sys/amd64/amd64/pmap.c

9275

pmap_invalidate_range(pmap, va, sva);

sys/amd64/amd64/pmap.c

9278

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

9279

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9290

pmap_t pmap;

sys/amd64/amd64/pmap.c

9310

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

9311

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

9314

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

9317

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9321

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

9322

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

9324

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

9328

pmap_demote_pde_locked(pmap, pde, va, &lock) &&

sys/amd64/amd64/pmap.c

9338

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

9340

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9343

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

9344

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

9348

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

9351

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9355

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

9356

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

9357

pde = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9363

pmap_invalidate_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9365

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9531

pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va, vm_page_t m)

sys/amd64/amd64/pmap.c

9539

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

9540

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

9541

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

9542

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

9544

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

9549

pdpg = pmap_alloc_pt_page(pmap, va >> PDPSHIFT,

sys/amd64/amd64/pmap.c

9554

va, pmap);

sys/amd64/amd64/pmap.c

9560

pmap_pt_page_count_adj(pmap, 1);

sys/amd64/amd64/pmap.c

9587

pmap_invalidate_page(pmap, (vm_offset_t)vtopde(va));

sys/amd64/amd64/pmap.c

9591

" in pmap %p", va, pmap);

sys/amd64/include/pcpu.h

59

struct pmap *pc_curpmap; \

sys/amd64/include/pmap.h

346

typedef struct pmap *pmap_t;

sys/amd64/include/pmap.h

349

extern struct pmap kernel_pmap_store;

sys/amd64/include/pmap.h

352

#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)

sys/amd64/include/pmap.h

353

#define PMAP_LOCK_ASSERT(pmap, type) \

sys/amd64/include/pmap.h

354

mtx_assert(&(pmap)->pm_mtx, (type))

sys/amd64/include/pmap.h

355

#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)

sys/amd64/include/pmap.h

356

#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \

sys/amd64/include/pmap.h

358

#define PMAP_LOCKED(pmap) mtx_owned(&(pmap)->pm_mtx)

sys/amd64/include/pmap.h

359

#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)

sys/amd64/include/pmap.h

360

#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)

sys/amd64/include/pmap.h

361

#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)

sys/amd64/include/pmap.h

363

int pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags);

sys/amd64/include/pmap.h

364

int pmap_emulate_accessed_dirty(pmap_t pmap, vm_offset_t va, int ftype);

sys/amd64/include/pmap.h

390

void pmap_activate_boot(pmap_t pmap);

sys/amd64/include/pmap.h

394

int pmap_cache_bits(pmap_t pmap, int mode, bool is_pde);

sys/amd64/include/pmap.h

418

bool pmap_ps_enabled(pmap_t pmap);

sys/amd64/include/pmap.h

427

void pmap_get_mapping(pmap_t pmap, vm_offset_t va, uint64_t *ptr, int *num);

sys/amd64/include/pmap.h

437

int pmap_pkru_clear(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);

sys/amd64/include/pmap.h

438

int pmap_pkru_set(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/include/pmap.h

455

pmap_invalidate_cpu_mask(pmap_t pmap)

sys/amd64/include/pmap.h

457

return (&pmap->pm_active);

sys/amd64/include/pmap.h

468

pmap_invlpg(pmap_t pmap, vm_offset_t va)

sys/amd64/include/pmap.h

470

if (pmap == kernel_pmap && PCPU_GET(pcid_invlpg_workaround)) {

sys/amd64/include/pmap.h

483

pmap_get_pcid(pmap_t pmap)

sys/amd64/include/pmap.h

488

pcidp = zpcpu_get(pmap->pm_pcidp);

sys/amd64/include/pmap.h

511

typedef void (*smp_invl_local_cb_t)(struct pmap *, vm_offset_t addr1,

sys/amd64/include/vmm.h

166

struct pmap;

sys/amd64/include/vmm.h

177

DECLARE_VMMOPS_FUNC(void *, init, (struct vm *vm, struct pmap *pmap));

sys/amd64/include/vmm.h

179

struct pmap *pmap, struct vm_eventinfo *info));

sys/amd64/vmm/amd/npt.c

67

npt_pinit(pmap_t pmap)

sys/amd64/vmm/amd/npt.c

70

return (pmap_pinit_type(pmap, PT_RVI, npt_flags));

sys/amd64/vmm/amd/svm.c

1944

svm_pmap_activate(struct svm_vcpu *vcpu, pmap_t pmap)

sys/amd64/vmm/amd/svm.c

1952

CPU_SET_ATOMIC(cpu, &pmap->pm_active);

sys/amd64/vmm/amd/svm.c

1953

smr_enter(pmap->pm_eptsmr);

sys/amd64/vmm/amd/svm.c

1993

eptgen = atomic_load_long(&pmap->pm_eptgen);

sys/amd64/vmm/amd/svm.c

2047

svm_pmap_deactivate(pmap_t pmap)

sys/amd64/vmm/amd/svm.c

2049

smr_exit(pmap->pm_eptsmr);

sys/amd64/vmm/amd/svm.c

2050

CPU_CLR_ATOMIC(curcpu, &pmap->pm_active);

sys/amd64/vmm/amd/svm.c

2125

svm_run(void *vcpui, register_t rip, pmap_t pmap, struct vm_eventinfo *evinfo)

sys/amd64/vmm/amd/svm.c

2234

svm_pmap_activate(vcpu, pmap);

sys/amd64/vmm/amd/svm.c

2246

svm_pmap_deactivate(pmap);

sys/amd64/vmm/amd/svm.c

591

svm_init(struct vm *vm, pmap_t pmap)

sys/amd64/vmm/amd/svm.c

607

svm_sc->nptp = vtophys(pmap->pm_pmltop);

sys/amd64/vmm/intel/ept.c

173

ept_pinit(pmap_t pmap)

sys/amd64/vmm/intel/ept.c

176

return (pmap_pinit_type(pmap, PT_EPT, ept_pmap_flags));

sys/amd64/vmm/intel/vmx.c

1056

vmx_init(struct vm *vm, pmap_t pmap)

sys/amd64/vmm/intel/vmx.c

1064

vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pmltop));

sys/amd64/vmm/intel/vmx.c

1127

vmx->pmap = pmap;

sys/amd64/vmm/intel/vmx.c

1245

vcpu->ctx.pmap = vmx->pmap;

sys/amd64/vmm/intel/vmx.c

1289

vmx_invvpid(struct vmx *vmx, struct vmx_vcpu *vcpu, pmap_t pmap, int running)

sys/amd64/vmm/intel/vmx.c

1327

if (atomic_load_long(&pmap->pm_eptgen) == vmx->eptgen[curcpu]) {

sys/amd64/vmm/intel/vmx.c

1346

vmx_set_pcpu_defaults(struct vmx *vmx, struct vmx_vcpu *vcpu, pmap_t pmap)

sys/amd64/vmm/intel/vmx.c

1361

vmx_invvpid(vmx, vcpu, pmap, 1);

sys/amd64/vmm/intel/vmx.c

3013

vmx_pmap_activate(struct vmx *vmx, pmap_t pmap)

sys/amd64/vmm/intel/vmx.c

3020

CPU_SET_ATOMIC(cpu, &pmap->pm_active);

sys/amd64/vmm/intel/vmx.c

3021

smr_enter(pmap->pm_eptsmr);

sys/amd64/vmm/intel/vmx.c

3022

eptgen = atomic_load_long(&pmap->pm_eptgen);

sys/amd64/vmm/intel/vmx.c

3031

vmx_pmap_deactivate(struct vmx *vmx, pmap_t pmap)

sys/amd64/vmm/intel/vmx.c

3033

smr_exit(pmap->pm_eptsmr);

sys/amd64/vmm/intel/vmx.c

3034

CPU_CLR_ATOMIC(curcpu, &pmap->pm_active);

sys/amd64/vmm/intel/vmx.c

3038

vmx_run(void *vcpui, register_t rip, pmap_t pmap, struct vm_eventinfo *evinfo)

sys/amd64/vmm/intel/vmx.c

3059

KASSERT(vmxctx->pmap == pmap,

sys/amd64/vmm/intel/vmx.c

3060

("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));

sys/amd64/vmm/intel/vmx.c

3077

vmx_set_pcpu_defaults(vmx, vcpu, pmap);

sys/amd64/vmm/intel/vmx.c

3190

vmx_pmap_activate(vmx, pmap);

sys/amd64/vmm/intel/vmx.c

3195

vmx_pmap_deactivate(vmx, pmap);

sys/amd64/vmm/intel/vmx.c

3442

pmap_t pmap;

sys/amd64/vmm/intel/vmx.c

3498

pmap = vcpu->ctx.pmap;

sys/amd64/vmm/intel/vmx.c

3499

vmx_invvpid(vmx, vcpu, pmap, running);

sys/amd64/vmm/intel/vmx.h

147

pmap_t pmap;

sys/amd64/vmm/intel/vmx.h

38

struct pmap;

sys/amd64/vmm/intel/vmx.h

86

struct pmap *pmap;

sys/amd64/vmm/intel/vmx_genassym.c

77

ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));

sys/amd64/vmm/intel/vmx_genassym.c

78

ASSYM(PM_EPTGEN, offsetof(struct pmap, pm_eptgen));

sys/amd64/vmm/vmm.c

1132

pmap_t pmap;

sys/amd64/vmm/vmm.c

1142

pmap = vmspace_pmap(vm_vmspace(vm));

sys/amd64/vmm/vmm.c

1150

KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active),

sys/amd64/vmm/vmm.c

1161

error = vmmops_run(vcpu->cookie, vcpu->nextrip, pmap, &evinfo);

sys/amd64/vmm/vmm.c

133

DEFINE_VMMOPS_IFUNC(void *, init, (struct vm *vm, struct pmap *pmap))

sys/amd64/vmm/vmm.c

134

DEFINE_VMMOPS_IFUNC(int, run, (void *vcpui, register_t rip, struct pmap *pmap,

sys/amd64/vmm/vmm.c

470

pmap_t pmap;

sys/amd64/vmm/vmm.c

477

pmap = vmspace_pmap(vm_vmspace(vm));

sys/amd64/vmm/vmm.c

491

hpa = pmap_extract(pmap, gpa);

sys/arm/arm/busdma_machdep.c

158

static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, pmap_t pmap,

sys/arm/arm/busdma_machdep.c

813

_bus_dmamap_count_pages(bus_dma_tag_t dmat, pmap_t pmap, bus_dmamap_t map,

sys/arm/arm/busdma_machdep.c

836

if (__predict_true(pmap == kernel_pmap))

sys/arm/arm/busdma_machdep.c

839

paddr = pmap_extract(pmap, vaddr);

sys/arm/arm/busdma_machdep.c

943

bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,

sys/arm/arm/busdma_machdep.c

972

_bus_dmamap_count_pages(dmat, pmap, map, buf, buflen, flags);

sys/arm/arm/busdma_machdep.c

990

if (__predict_true(pmap == kernel_pmap)) {

sys/arm/arm/busdma_machdep.c

994

curaddr = pmap_extract(pmap, vaddr);

sys/arm/arm/db_interface.c

160

struct pmap *pmap;

sys/arm/arm/db_interface.c

162

if (!p || !p->p_vmspace || !p->p_vmspace->vm_map.pmap ||

sys/arm/arm/db_interface.c

169

pmap = kernel_pmap;

sys/arm/arm/db_interface.c

171

pmap = p->p_vmspace->vm_map.pmap;

sys/arm/arm/db_interface.c

173

return (pmap_extract(pmap, addr) == 0);

sys/arm/arm/genassym.c

123

ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));

sys/arm/arm/pmap-v6.c

1361

pmap_t pmap;

sys/arm/arm/pmap-v6.c

1365

LIST_FOREACH(pmap, &allpmaps, pm_list) {

sys/arm/arm/pmap-v6.c

1366

pte2p = pmap_pt2tab_entry(pmap, va);

sys/arm/arm/pmap-v6.c

1379

pmap_t pmap;

sys/arm/arm/pmap-v6.c

1383

LIST_FOREACH(pmap, &allpmaps, pm_list) {

sys/arm/arm/pmap-v6.c

1384

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

1488

pmap_tlb_flush(pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

1491

if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))

sys/arm/arm/pmap-v6.c

1496

pmap_tlb_flush_range(pmap_t pmap, vm_offset_t sva, vm_size_t size)

sys/arm/arm/pmap-v6.c

1499

if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))

sys/arm/arm/pmap-v6.c

1564

SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,

sys/arm/arm/pmap-v6.c

1586

pmap_ps_enabled(pmap_t pmap __unused)

sys/arm/arm/pmap-v6.c

1620

pmap_ttb_get(pmap_t pmap)

sys/arm/arm/pmap-v6.c

1623

return (vtophys(pmap->pm_pt1) | ttb_flags);

sys/arm/arm/pmap-v6.c

1702

pmap_pt2pg_init(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/arm/arm/pmap-v6.c

1719

if (pmap == kernel_pmap)

sys/arm/arm/pmap-v6.c

1722

pte2p = pmap_pt2tab_entry(pmap, va);

sys/arm/arm/pmap-v6.c

1870

pmap_is_current(pmap_t pmap)

sys/arm/arm/pmap-v6.c

1873

return (pmap == kernel_pmap ||

sys/arm/arm/pmap-v6.c

1874

(pmap == vmspace_pmap(curthread->td_proc->p_vmspace)));

sys/arm/arm/pmap-v6.c

1882

pmap_pte2(pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

1887

pte1 = pte1_load(pmap_pte1(pmap, va));

sys/arm/arm/pmap-v6.c

1892

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

1930

pmap_pte2_quick(pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

1935

pte1 = pte1_load(pmap_pte1(pmap, va));

sys/arm/arm/pmap-v6.c

1940

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

1975

pmap_extract(pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

1981

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

1982

pte1 = pte1_load(pmap_pte1(pmap, va));

sys/arm/arm/pmap-v6.c

1986

pte2p = pmap_pte2(pmap, va);

sys/arm/arm/pmap-v6.c

1991

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

2003

pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/arm/arm/pmap-v6.c

2011

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

2012

pte1 = pte1_load(pmap_pte1(pmap, va));

sys/arm/arm/pmap-v6.c

2021

pte2p = pmap_pte2(pmap, va);

sys/arm/arm/pmap-v6.c

2032

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

2164

pmap_pinit0(pmap_t pmap)

sys/arm/arm/pmap-v6.c

2166

PDEBUG(1, printf("%s: pmap = %p\n", __func__, pmap));

sys/arm/arm/pmap-v6.c

2168

PMAP_LOCK_INIT(pmap);

sys/arm/arm/pmap-v6.c

2179

pmap->pm_pt1 = kern_pt1;

sys/arm/arm/pmap-v6.c

2180

pmap->pm_pt2tab = kern_pt2tab;

sys/arm/arm/pmap-v6.c

2181

CPU_ZERO(&pmap->pm_active);

sys/arm/arm/pmap-v6.c

2182

PCPU_SET(curpmap, pmap);

sys/arm/arm/pmap-v6.c

2183

TAILQ_INIT(&pmap->pm_pvchunk);

sys/arm/arm/pmap-v6.c

2184

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/arm/arm/pmap-v6.c

2185

CPU_SET(0, &pmap->pm_active);

sys/arm/arm/pmap-v6.c

2215

pmap_pinit(pmap_t pmap)

sys/arm/arm/pmap-v6.c

2222

PDEBUG(6, printf("%s: pmap = %p, pm_pt1 = %p\n", __func__, pmap,

sys/arm/arm/pmap-v6.c

2223

pmap->pm_pt1));

sys/arm/arm/pmap-v6.c

2245

if (pmap->pm_pt1 == NULL) {

sys/arm/arm/pmap-v6.c

2246

pmap->pm_pt1 = kmem_alloc_contig(NB_IN_PT1,

sys/arm/arm/pmap-v6.c

2248

if (pmap->pm_pt1 == NULL)

sys/arm/arm/pmap-v6.c

2251

if (pmap->pm_pt2tab == NULL) {

sys/arm/arm/pmap-v6.c

2262

pmap->pm_pt2tab = kmem_alloc_attr(NB_IN_PT2TAB,

sys/arm/arm/pmap-v6.c

2264

if (pmap->pm_pt2tab == NULL) {

sys/arm/arm/pmap-v6.c

2270

kmem_free(pmap->pm_pt1, NB_IN_PT1);

sys/arm/arm/pmap-v6.c

2271

pmap->pm_pt1 = NULL;

sys/arm/arm/pmap-v6.c

2289

pte1_copy_nosync(kern_pt1, pmap->pm_pt1, KERNBASE,

sys/arm/arm/pmap-v6.c

2291

pte1_copy_nosync(kern_pt1, pmap->pm_pt1, vm_max_kernel_address,

sys/arm/arm/pmap-v6.c

2293

pt2tab_copy_nosync(kern_pt2tab, pmap->pm_pt2tab, KERNBASE,

sys/arm/arm/pmap-v6.c

2295

pt2tab_copy_nosync(kern_pt2tab, pmap->pm_pt2tab, vm_max_kernel_address,

sys/arm/arm/pmap-v6.c

2297

LIST_INSERT_HEAD(&allpmaps, pmap, pm_list);

sys/arm/arm/pmap-v6.c

2305

pt2tab_pa = vtophys(pmap->pm_pt2tab);

sys/arm/arm/pmap-v6.c

2306

pte2p = pmap_pt2tab_entry(pmap, (vm_offset_t)PT2MAP);

sys/arm/arm/pmap-v6.c

2312

pte1p = pmap_pte1(pmap, (vm_offset_t)PT2MAP);

sys/arm/arm/pmap-v6.c

2320

pte1_sync_range(pmap->pm_pt1, NB_IN_PT1);

sys/arm/arm/pmap-v6.c

2321

pte2_sync_range(pmap->pm_pt2tab, NB_IN_PT2TAB);

sys/arm/arm/pmap-v6.c

2323

CPU_ZERO(&pmap->pm_active);

sys/arm/arm/pmap-v6.c

2324

TAILQ_INIT(&pmap->pm_pvchunk);

sys/arm/arm/pmap-v6.c

2325

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/arm/arm/pmap-v6.c

2348

pmap_release(pmap_t pmap)

sys/arm/arm/pmap-v6.c

2353

KASSERT(pmap->pm_stats.resident_count == 0,

sys/arm/arm/pmap-v6.c

2355

pmap->pm_stats.resident_count));

sys/arm/arm/pmap-v6.c

2356

KASSERT(pt2tab_user_is_empty(pmap->pm_pt2tab),

sys/arm/arm/pmap-v6.c

2358

KASSERT(CPU_EMPTY(&pmap->pm_active),

sys/arm/arm/pmap-v6.c

2359

("%s: pmap %p is active on some CPU(s)", __func__, pmap));

sys/arm/arm/pmap-v6.c

2362

LIST_REMOVE(pmap, pm_list);

sys/arm/arm/pmap-v6.c

2368

bzero((char *)pmap->pm_pt1 + start, end - start);

sys/arm/arm/pmap-v6.c

2372

bzero((char *)pmap->pm_pt2tab + start, end - start);

sys/arm/arm/pmap-v6.c

2501

_pmap_allocpte2(pmap_t pmap, vm_offset_t va, u_int flags)

sys/arm/arm/pmap-v6.c

2510

pte1p = pmap->pm_pt1 + pte1_idx;

sys/arm/arm/pmap-v6.c

2516

pte2 = pt2tab_load(pmap_pt2tab_entry(pmap, va));

sys/arm/arm/pmap-v6.c

2524

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

2528

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

2538

pmap->pm_stats.resident_count++;

sys/arm/arm/pmap-v6.c

2539

pt2pg_pa = pmap_pt2pg_init(pmap, va, m);

sys/arm/arm/pmap-v6.c

2553

pmap_allocpte2(pmap_t pmap, vm_offset_t va, u_int flags)

sys/arm/arm/pmap-v6.c

2561

pte1p = pmap->pm_pt1 + pte1_idx;

sys/arm/arm/pmap-v6.c

2569

(void)pmap_demote_pte1(pmap, pte1p, va);

sys/arm/arm/pmap-v6.c

257

struct pmap kernel_pmap_store;

sys/arm/arm/pmap-v6.c

258

LIST_HEAD(pmaplist, pmap);

sys/arm/arm/pmap-v6.c

2591

m = _pmap_allocpte2(pmap, va, flags);

sys/arm/arm/pmap-v6.c

2623

pmap_unwire_pt2pg(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/arm/arm/pmap-v6.c

2630

("%s: pmap %p PT2PG %p wired", __func__, pmap, m));

sys/arm/arm/pmap-v6.c

2639

("%s: pmap %p va %#x PT2PG %p bad index", __func__, pmap, va, m));

sys/arm/arm/pmap-v6.c

2640

pte1p = pmap->pm_pt1 + m->pindex;

sys/arm/arm/pmap-v6.c

2643

("%s: pmap %p PT2 %u (PG %p) wired", __func__, pmap, i, m));

sys/arm/arm/pmap-v6.c

2650

pmap_tlb_flush(pmap, (m->pindex + i) << PTE1_SHIFT);

sys/arm/arm/pmap-v6.c

2656

pmap, va, opte1, i));

sys/arm/arm/pmap-v6.c

2663

pte2p = pmap_pt2tab_entry(pmap, va);

sys/arm/arm/pmap-v6.c

2665

pmap_tlb_flush(pmap, pt2map_pt2pg(va));

sys/arm/arm/pmap-v6.c

2668

pmap->pm_stats.resident_count--;

sys/arm/arm/pmap-v6.c

2686

pmap_unwire_pt2(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/arm/arm/pmap-v6.c

2696

pmap_unwire_pt2pg(pmap, va, m);

sys/arm/arm/pmap-v6.c

2709

pmap_unwire_pt2_all(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm/arm/pmap-v6.c

2744

pmap_unwire_pt2pg(pmap, va, m);

sys/arm/arm/pmap-v6.c

2754

pmap_unuse_pt2(pmap_t pmap, vm_offset_t va, struct spglist *free)

sys/arm/arm/pmap-v6.c

2761

pte1 = pte1_load(pmap_pte1(pmap, va));

sys/arm/arm/pmap-v6.c

2763

return (pmap_unwire_pt2(pmap, va, mpte, free));

sys/arm/arm/pmap-v6.c

2843

pmap_t pmap;

sys/arm/arm/pmap-v6.c

2853

pmap = NULL;

sys/arm/arm/pmap-v6.c

2860

if (pmap != pc->pc_pmap) {

sys/arm/arm/pmap-v6.c

2861

if (pmap != NULL) {

sys/arm/arm/pmap-v6.c

2862

if (pmap != locked_pmap)

sys/arm/arm/pmap-v6.c

2863

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

2865

pmap = pc->pc_pmap;

sys/arm/arm/pmap-v6.c

2867

if (pmap > locked_pmap)

sys/arm/arm/pmap-v6.c

2868

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

2869

else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {

sys/arm/arm/pmap-v6.c

2870

pmap = NULL;

sys/arm/arm/pmap-v6.c

2886

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

2889

pte2p = pmap_pte2(pmap, va);

sys/arm/arm/pmap-v6.c

2898

pmap, va));

sys/arm/arm/pmap-v6.c

2899

pmap_tlb_flush(pmap, va);

sys/arm/arm/pmap-v6.c

2915

pmap_unuse_pt2(pmap, va, &free);

sys/arm/arm/pmap-v6.c

2924

pmap->pm_stats.resident_count -= freed;

sys/arm/arm/pmap-v6.c

2928

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

2931

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,

sys/arm/arm/pmap-v6.c

2939

if (pmap == locked_pmap)

sys/arm/arm/pmap-v6.c

2956

if (pmap != NULL) {

sys/arm/arm/pmap-v6.c

2957

if (pmap != locked_pmap)

sys/arm/arm/pmap-v6.c

2958

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

2992

free_pv_entry(pmap_t pmap, pv_entry_t pv)

sys/arm/arm/pmap-v6.c

2998

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3013

if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=

sys/arm/arm/pmap-v6.c

3015

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

3016

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,

sys/arm/arm/pmap-v6.c

3021

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

3030

get_pv_entry(pmap_t pmap, bool try)

sys/arm/arm/pmap-v6.c

3040

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3049

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/arm/arm/pmap-v6.c

3066

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

3067

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

3084

m = pmap_pv_reclaim(pmap);

sys/arm/arm/pmap-v6.c

3092

pc->pc_pmap = pmap;

sys/arm/arm/pmap-v6.c

3098

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

3108

pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/arm/arm/pmap-v6.c

3113

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3114

pv = get_pv_entry(pmap, false);

sys/arm/arm/pmap-v6.c

3120

pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

3126

if (pmap == PV_PMAP(pv) && va == pv->pv_va) {

sys/arm/arm/pmap-v6.c

3135

pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

3139

pv = pmap_pvh_remove(pvh, pmap, va);

sys/arm/arm/pmap-v6.c

3141

free_pv_entry(pmap, pv);

sys/arm/arm/pmap-v6.c

3145

pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)

sys/arm/arm/pmap-v6.c

3150

pmap_pvh_free(&m->md, pmap, va);

sys/arm/arm/pmap-v6.c

3159

pmap_pv_demote_pte1(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)

sys/arm/arm/pmap-v6.c

3176

pv = pmap_pvh_remove(pvh, pmap, va);

sys/arm/arm/pmap-v6.c

3187

pmap_insert_entry(pmap, va, m);

sys/arm/arm/pmap-v6.c

3193

pmap_pv_promote_pte1(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)

sys/arm/arm/pmap-v6.c

3213

pv = pmap_pvh_remove(&m->md, pmap, va);

sys/arm/arm/pmap-v6.c

3222

pmap_pvh_free(&m->md, pmap, va);

sys/arm/arm/pmap-v6.c

3231

pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/arm/arm/pmap-v6.c

3236

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3238

(pv = get_pv_entry(pmap, true)) != NULL) {

sys/arm/arm/pmap-v6.c

3250

pmap_pv_insert_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t pte1, u_int flags)

sys/arm/arm/pmap-v6.c

3259

(pv = get_pv_entry(pmap, noreclaim)) == NULL)

sys/arm/arm/pmap-v6.c

3268

pmap_tlb_flush_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t npte1)

sys/arm/arm/pmap-v6.c

3273

pmap_tlb_flush_range(pmap, pte1_trunc(va), PTE1_SIZE);

sys/arm/arm/pmap-v6.c

3275

pmap_tlb_flush(pmap, pte1_trunc(va));

sys/arm/arm/pmap-v6.c

328

static bool pmap_demote_pte1(pmap_t pmap, pt1_entry_t *pte1p,

sys/arm/arm/pmap-v6.c

3294

pmap_t pmap;

sys/arm/arm/pmap-v6.c

330

static int pmap_enter_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t pte1,

sys/arm/arm/pmap-v6.c

3301

pmap = PCPU_GET(curpmap);

sys/arm/arm/pmap-v6.c

3302

if (pmap == NULL)

sys/arm/arm/pmap-v6.c

3303

pmap = kernel_pmap;

sys/arm/arm/pmap-v6.c

3312

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

3329

LIST_FOREACH(pmap, &allpmaps, pm_list) {

sys/arm/arm/pmap-v6.c

3330

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

3375

pmap_change_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va,

sys/arm/arm/pmap-v6.c

3379

if (pmap == kernel_pmap) {

sys/arm/arm/pmap-v6.c

3402

pmap_tlb_flush_pte1(pmap, va, npte1);

sys/arm/arm/pmap-v6.c

3409

pmap_change_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va,

sys/arm/arm/pmap-v6.c

3413

if (pmap == kernel_pmap) {

sys/arm/arm/pmap-v6.c

3427

pmap_tlb_flush_pte1(pmap, va, npte1);

sys/arm/arm/pmap-v6.c

3448

pmap_promote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va)

sys/arm/arm/pmap-v6.c

3457

pmap, va, pte1_load(pte1p), pte1p));

sys/arm/arm/pmap-v6.c

3459

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3466

fpte2p = pmap_pte2_quick(pmap, pte1_trunc(va));

sys/arm/arm/pmap-v6.c

3472

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3475

if (pte2_is_managed(fpte2) && pmap == kernel_pmap) {

sys/arm/arm/pmap-v6.c

3478

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3502

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3515

__func__, pteva, pmap);

sys/arm/arm/pmap-v6.c

3520

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3548

pmap_pv_promote_pte1(pmap, va, pte1_pa(npte1));

sys/arm/arm/pmap-v6.c

3553

pmap_change_pte1(pmap, pte1p, va, npte1);

sys/arm/arm/pmap-v6.c

3557

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3560

__func__, pmap, va, npte1, pte1_load(pte1p), pte1p));

sys/arm/arm/pmap-v6.c

3581

pmap_remove_kernel_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va)

sys/arm/arm/pmap-v6.c

3588

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3589

m = pmap_pt2_page(pmap, va);

sys/arm/arm/pmap-v6.c

3623

pmap_remove_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva,

sys/arm/arm/pmap-v6.c

3631

PDEBUG(6, printf("%s(%p): va %#x pte1 %#x at %p\n", __func__, pmap, sva,

sys/arm/arm/pmap-v6.c

3634

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3644

pmap_tlb_flush(pmap, sva);

sys/arm/arm/pmap-v6.c

3647

pmap->pm_stats.wired_count -= PTE1_SIZE / PAGE_SIZE;

sys/arm/arm/pmap-v6.c

3648

pmap->pm_stats.resident_count -= PTE1_SIZE / PAGE_SIZE;

sys/arm/arm/pmap-v6.c

3651

pmap_pvh_free(pvh, pmap, sva);

sys/arm/arm/pmap-v6.c

3664

if (pmap == kernel_pmap) {

sys/arm/arm/pmap-v6.c

3669

pmap_remove_kernel_pte1(pmap, pte1p, sva);

sys/arm/arm/pmap-v6.c

3675

m = pmap_pt2_page(pmap, sva);

sys/arm/arm/pmap-v6.c

3677

pmap_unwire_pt2_all(pmap, sva, m, free);

sys/arm/arm/pmap-v6.c

3700

pmap_demote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va)

sys/arm/arm/pmap-v6.c

3710

pmap, va, pte1_load(pte1p), pte1p));

sys/arm/arm/pmap-v6.c

3712

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3717

if ((opte1 & PTE1_A) == 0 || (m = pmap_pt2_page(pmap, va)) == NULL) {

sys/arm/arm/pmap-v6.c

3729

pmap_remove_pte1(pmap, pte1p, pte1_trunc(va), &free);

sys/arm/arm/pmap-v6.c

3732

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3737

pmap->pm_stats.resident_count++;

sys/arm/arm/pmap-v6.c

3746

pt2pg_pa = pmap_pt2pg_init(pmap, va, m);

sys/arm/arm/pmap-v6.c

3770

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

3836

pmap_change_pte1(pmap, pte1p, va, npte1);

sys/arm/arm/pmap-v6.c

3848

pmap_pv_demote_pte1(pmap, va, pte1_pa(opte1));

sys/arm/arm/pmap-v6.c

3852

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3855

__func__, pmap, va, npte1, pte1_load(pte1p), pte1p));

sys/arm/arm/pmap-v6.c

3872

pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/arm/arm/pmap-v6.c

3908

if (pmap != kernel_pmap)

sys/arm/arm/pmap-v6.c

3912

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

3919

rv = pmap_enter_pte1(pmap, va, PTE1_PA(pa) | ATTR_TO_L1(npte2) |

sys/arm/arm/pmap-v6.c

3929

mpte2 = pmap_allocpte2(pmap, va, flags);

sys/arm/arm/pmap-v6.c

3938

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

3941

pte2p = pmap_pte2_quick(pmap, va);

sys/arm/arm/pmap-v6.c

3959

pmap->pm_stats.wired_count++;

sys/arm/arm/pmap-v6.c

3961

pmap->pm_stats.wired_count--;

sys/arm/arm/pmap-v6.c

3982

__func__, pmap, va, opte2, opa, pa, flags, prot));

sys/arm/arm/pmap-v6.c

3992

pmap->pm_stats.wired_count--;

sys/arm/arm/pmap-v6.c

3997

pv = pmap_pvh_remove(&om->md, pmap, va);

sys/arm/arm/pmap-v6.c

4005

pmap->pm_stats.resident_count++;

sys/arm/arm/pmap-v6.c

4012

pv = get_pv_entry(pmap, false);

sys/arm/arm/pmap-v6.c

4017

free_pv_entry(pmap, pv);

sys/arm/arm/pmap-v6.c

402

pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)

sys/arm/arm/pmap-v6.c

4023

pmap->pm_stats.wired_count++;

sys/arm/arm/pmap-v6.c

4057

if ((prot & VM_PROT_EXECUTE) && pmap != kernel_pmap &&

sys/arm/arm/pmap-v6.c

4065

pmap_tlb_flush(pmap, va);

sys/arm/arm/pmap-v6.c

4090

panic("%s: pmap %p va %#x opte2 %x npte2 %x !!", __func__, pmap,

sys/arm/arm/pmap-v6.c

4103

pmap_promote_pte1(pmap, pte1p, va);

sys/arm/arm/pmap-v6.c

4110

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4118

pmap_remove_pte2(pmap_t pmap, pt2_entry_t *pte2p, vm_offset_t va,

sys/arm/arm/pmap-v6.c

4125

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

4129

pmap_tlb_flush(pmap, va);

sys/arm/arm/pmap-v6.c

4132

__func__, pmap, va, opte2));

sys/arm/arm/pmap-v6.c

4135

pmap->pm_stats.wired_count -= 1;

sys/arm/arm/pmap-v6.c

4136

pmap->pm_stats.resident_count -= 1;

sys/arm/arm/pmap-v6.c

4143

pmap_remove_entry(pmap, m, va);

sys/arm/arm/pmap-v6.c

4145

return (pmap_unuse_pt2(pmap, va, free));

sys/arm/arm/pmap-v6.c

4152

pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free)

sys/arm/arm/pmap-v6.c

4159

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

4160

if ((pte2p = pmap_pte2_quick(pmap, va)) == NULL ||

sys/arm/arm/pmap-v6.c

4163

pmap_remove_pte2(pmap, pte2p, va, free);

sys/arm/arm/pmap-v6.c

4173

pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm/arm/pmap-v6.c

4183

if (pmap->pm_stats.resident_count == 0)

sys/arm/arm/pmap-v6.c

4190

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4197

pte1 = pte1_load(pmap_pte1(pmap, sva));

sys/arm/arm/pmap-v6.c

4199

pmap_remove_page(pmap, sva, &free);

sys/arm/arm/pmap-v6.c

4211

if (pmap->pm_stats.resident_count == 0)

sys/arm/arm/pmap-v6.c

4214

pte1p = pmap_pte1(pmap, sva);

sys/arm/arm/pmap-v6.c

4230

pmap_remove_pte1(pmap, pte1p, sva, &free);

sys/arm/arm/pmap-v6.c

4232

} else if (!pmap_demote_pte1(pmap, pte1p, sva)) {

sys/arm/arm/pmap-v6.c

4245

" is not link", __func__, pmap, sva, pte1, pte1p));

sys/arm/arm/pmap-v6.c

4255

for (pte2p = pmap_pte2_quick(pmap, sva); sva != nextva;

sys/arm/arm/pmap-v6.c

4260

if (pmap_remove_pte2(pmap, pte2p, sva, &free))

sys/arm/arm/pmap-v6.c

4267

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4289

pmap_t pmap;

sys/arm/arm/pmap-v6.c

4305

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

4306

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4307

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

4308

(void)pmap_demote_pte1(pmap, pte1p, va);

sys/arm/arm/pmap-v6.c

4309

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4313

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

4314

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4315

pmap->pm_stats.resident_count--;

sys/arm/arm/pmap-v6.c

4316

pte1p = pmap_pte1(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

4319

pte2p = pmap_pte2_quick(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

4321

pmap_tlb_flush(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

4323

__func__, pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

4325

pmap->pm_stats.wired_count--;

sys/arm/arm/pmap-v6.c

4334

pmap_unuse_pt2(pmap, pv->pv_va, &free);

sys/arm/arm/pmap-v6.c

4336

free_pv_entry(pmap, pv);

sys/arm/arm/pmap-v6.c

4337

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4350

pmap_remove_pte1_quick(pmap_t pmap, pt1_entry_t pte1, pv_entry_t pv,

sys/arm/arm/pmap-v6.c

4371

pmap->pm_stats.resident_count -= PTE1_SIZE / PAGE_SIZE;

sys/arm/arm/pmap-v6.c

4379

mpt2pg = pmap_pt2_page(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

4381

pmap_unwire_pt2_all(pmap, pv->pv_va, mpt2pg, free);

sys/arm/arm/pmap-v6.c

4389

pmap_remove_pte2_quick(pmap_t pmap, pt2_entry_t pte2, pv_entry_t pv,

sys/arm/arm/pmap-v6.c

4408

pmap->pm_stats.resident_count--;

sys/arm/arm/pmap-v6.c

4415

pmap_unuse_pt2(pmap, pv->pv_va, free);

sys/arm/arm/pmap-v6.c

4426

pmap_remove_pages(pmap_t pmap)

sys/arm/arm/pmap-v6.c

4443

KASSERT(pmap == vmspace_pmap(curthread->td_proc->p_vmspace),

sys/arm/arm/pmap-v6.c

4444

("%s: non-current pmap %p", __func__, pmap));

sys/arm/arm/pmap-v6.c

4450

other_cpus = pmap->pm_active;

sys/arm/arm/pmap-v6.c

4454

("%s: pmap %p active on other cpus", __func__, pmap));

sys/arm/arm/pmap-v6.c

4459

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4461

TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {

sys/arm/arm/pmap-v6.c

4462

KASSERT(pc->pc_pmap == pmap, ("%s: wrong pmap %p %p",

sys/arm/arm/pmap-v6.c

4463

__func__, pmap, pc->pc_pmap));

sys/arm/arm/pmap-v6.c

4478

pte1p = pmap_pte1(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

4486

pmap_remove_pte1_quick(pmap, pte1, pv,

sys/arm/arm/pmap-v6.c

4496

pmap, pv->pv_va, pte2);

sys/arm/arm/pmap-v6.c

4505

pmap_remove_pte2_quick(pmap, pte2, pv,

sys/arm/arm/pmap-v6.c

4509

__func__, pmap, pv->pv_va, pte1);

sys/arm/arm/pmap-v6.c

4521

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

4528

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4541

pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm/arm/pmap-v6.c

4553

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

4568

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

4600

mpt2pg = _pmap_allocpte2(pmap, va,

sys/arm/arm/pmap-v6.c

4633

!pmap_try_insert_pv_entry(pmap, va, m)) {

sys/arm/arm/pmap-v6.c

4636

if (pmap_unwire_pt2(pmap, va, mpt2pg, &free)) {

sys/arm/arm/pmap-v6.c

4637

pmap_tlb_flush(pmap, va);

sys/arm/arm/pmap-v6.c

4649

pmap->pm_stats.resident_count++;

sys/arm/arm/pmap-v6.c

4660

else if (m->md.pat_mode == VM_MEMATTR_WB_WA && pmap != kernel_pmap) {

sys/arm/arm/pmap-v6.c

4673

pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/arm/arm/pmap-v6.c

4677

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4678

(void)pmap_enter_quick_locked(pmap, va, m, prot, NULL);

sys/arm/arm/pmap-v6.c

4680

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4690

pmap_enter_1mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/arm/arm/pmap-v6.c

4695

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

4702

if (pmap != kernel_pmap)

sys/arm/arm/pmap-v6.c

4704

return (pmap_enter_pte1(pmap, va, pte1, PMAP_ENTER_NOSLEEP |

sys/arm/arm/pmap-v6.c

4717

pmap_enter_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t pte1, u_int flags,

sys/arm/arm/pmap-v6.c

4730

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

4731

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

4736

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

4746

pmap_remove_pte1(pmap, pte1p, va, &free);

sys/arm/arm/pmap-v6.c

4750

for (cur = va, pte2p = pmap_pte2_quick(pmap, va);

sys/arm/arm/pmap-v6.c

4755

if (pmap_remove_pte2(pmap, pte2p, cur, &free))

sys/arm/arm/pmap-v6.c

4766

if (!pmap_pv_insert_pte1(pmap, va, pte1, flags)) {

sys/arm/arm/pmap-v6.c

4768

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

4781

pmap->pm_stats.wired_count += PTE1_SIZE / PAGE_SIZE;

sys/arm/arm/pmap-v6.c

4782

pmap->pm_stats.resident_count += PTE1_SIZE / PAGE_SIZE;

sys/arm/arm/pmap-v6.c

4789

pmap != kernel_pmap && (!pte1_is_section(opte1) ||

sys/arm/arm/pmap-v6.c

4800

pmap);

sys/arm/arm/pmap-v6.c

4817

pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,

sys/arm/arm/pmap-v6.c

4825

__func__, pmap, start, end, m_start, prot));

sys/arm/arm/pmap-v6.c

4834

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4839

pmap_enter_1mpage(pmap, va, m, prot)) {

sys/arm/arm/pmap-v6.c

4842

mpt2pg = pmap_enter_quick_locked(pmap, va, m, prot,

sys/arm/arm/pmap-v6.c

4848

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4857

pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,

sys/arm/arm/pmap-v6.c

4911

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4913

pte1p = pmap_pte1(pmap, addr);

sys/arm/arm/pmap-v6.c

4916

pmap->pm_stats.resident_count += PTE1_SIZE /

sys/arm/arm/pmap-v6.c

4923

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4931

pmap_protect_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva,

sys/arm/arm/pmap-v6.c

4938

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

4962

pmap_tlb_flush(pmap, sva);

sys/arm/arm/pmap-v6.c

4971

pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)

sys/arm/arm/pmap-v6.c

4980

pmap_remove(pmap, sva, eva);

sys/arm/arm/pmap-v6.c

4988

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

4997

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5006

pte1p = pmap_pte1(pmap, sva);

sys/arm/arm/pmap-v6.c

5022

pmap_protect_pte1(pmap, pte1p, sva, prot);

sys/arm/arm/pmap-v6.c

5028

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5033

if (!pmap_demote_pte1(pmap, pte1p, sva)) {

sys/arm/arm/pmap-v6.c

5050

" is not link", __func__, pmap, sva, pte1, pte1p));

sys/arm/arm/pmap-v6.c

5060

for (pte2p = pmap_pte2_quick(pmap, sva); sva != nextva; pte2p++,

sys/arm/arm/pmap-v6.c

5088

pmap_tlb_flush(pmap, sva);

sys/arm/arm/pmap-v6.c

5096

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5107

pmap_t pmap;

sys/arm/arm/pmap-v6.c

5115

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5116

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5117

pte1 = pte1_load(pmap_pte1(pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

5124

pte2 = pte2_load(pmap_pte2_quick(pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

5128

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5169

pmap_t pmap;

sys/arm/arm/pmap-v6.c

5176

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5177

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5178

pte1 = pte1_load(pmap_pte1(pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

5184

pte2 = pte2_load(pmap_pte2_quick(pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

5187

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5229

pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/arm/arm/pmap-v6.c

5236

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5237

pte1 = pte1_load(pmap_pte1(pmap, addr));

sys/arm/arm/pmap-v6.c

5242

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5257

pmap_t pmap;

sys/arm/arm/pmap-v6.c

5264

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5265

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5266

pte1 = pte1_load(pmap_pte1(pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

5270

pte2 = pte2_load(pmap_pte2_quick(pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

5273

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5323

pmap_t pmap;

sys/arm/arm/pmap-v6.c

5340

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5341

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5342

pte1p = pmap_pte1(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5372

(uintptr_t)pmap) & (NPTE2_IN_PG - 1)) == 0 &&

sys/arm/arm/pmap-v6.c

5375

pmap_tlb_flush(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5379

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5393

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5394

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5395

pte1p = pmap_pte1(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5399

pte2p = pmap_pte2_quick(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5405

pmap_tlb_flush(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5408

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5432

pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm/arm/pmap-v6.c

5439

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

5447

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5453

pte1p = pmap_pte1(pmap, sva);

sys/arm/arm/pmap-v6.c

5473

pmap->pm_stats.wired_count -= PTE1_SIZE /

sys/arm/arm/pmap-v6.c

5480

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5486

if (!pmap_demote_pte1(pmap, pte1p, sva))

sys/arm/arm/pmap-v6.c

5498

" is not link", __func__, pmap, sva, pte1, pte1p));

sys/arm/arm/pmap-v6.c

5508

for (pte2p = pmap_pte2_quick(pmap, sva); sva != nextva; pte2p++,

sys/arm/arm/pmap-v6.c

5523

pmap->pm_stats.wired_count--;

sys/arm/arm/pmap-v6.c

5530

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5541

pmap_t pmap;

sys/arm/arm/pmap-v6.c

5559

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5560

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5561

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

5563

(void)pmap_demote_pte1(pmap, pte1p, va);

sys/arm/arm/pmap-v6.c

5564

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5568

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5569

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5570

pte1p = pmap_pte1(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5573

pte2p = pmap_pte2_quick(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5579

pmap_tlb_flush(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5581

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5594

pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)

sys/arm/arm/pmap-v6.c

5604

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

5612

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5617

pte1p = pmap_pte1(pmap, sva);

sys/arm/arm/pmap-v6.c

5627

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5632

if (!pmap_demote_pte1(pmap, pte1p, sva)) {

sys/arm/arm/pmap-v6.c

5647

pte2p = pmap_pte2_quick(pmap, sva);

sys/arm/arm/pmap-v6.c

5650

pmap_remove_pte2(pmap, pte2p, sva, NULL);

sys/arm/arm/pmap-v6.c

5655

for (pte2p = pmap_pte2_quick(pmap, sva); sva != pdnxt; pte2p++,

sys/arm/arm/pmap-v6.c

5676

pmap_tlb_flush(pmap, sva);

sys/arm/arm/pmap-v6.c

5683

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5694

pmap_t pmap;

sys/arm/arm/pmap-v6.c

5712

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5713

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5714

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

5717

if (pmap_demote_pte1(pmap, pte1p, va) &&

sys/arm/arm/pmap-v6.c

5725

pte2p = pmap_pte2_quick(pmap, va);

sys/arm/arm/pmap-v6.c

5730

pmap_tlb_flush(pmap, va);

sys/arm/arm/pmap-v6.c

5734

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5738

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5739

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5740

pte1p = pmap_pte1(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5743

pte2p = pmap_pte2_quick(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5746

pmap_tlb_flush(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5748

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5834

pmap_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/arm/arm/pmap-v6.c

5846

if (PV_PMAP(pv) == pmap) {

sys/arm/arm/pmap-v6.c

5857

if (PV_PMAP(pv) == pmap) {

sys/arm/arm/pmap-v6.c

6202

pmap_t pmap, oldpmap;

sys/arm/arm/pmap-v6.c

6208

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/arm/arm/pmap-v6.c

6214

CPU_SET_ATOMIC(cpuid, &pmap->pm_active);

sys/arm/arm/pmap-v6.c

6217

CPU_SET(cpuid, &pmap->pm_active);

sys/arm/arm/pmap-v6.c

6220

ttb = pmap_ttb_get(pmap);

sys/arm/arm/pmap-v6.c

6227

PCPU_SET(curpmap, pmap);

sys/arm/arm/pmap-v6.c

6232

pmap_active_cpus(pmap_t pmap, cpuset_t *res)

sys/arm/arm/pmap-v6.c

6234

*res = pmap->pm_active;

sys/arm/arm/pmap-v6.c

6243

pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/arm/arm/pmap-v6.c

6251

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

6252

pte1p = pmap_pte1(pmap, addr);

sys/arm/arm/pmap-v6.c

6263

pte2p = pmap_pte2(pmap, addr);

sys/arm/arm/pmap-v6.c

6281

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6309

pmap_set_pcb_pagedir(pmap_t pmap, struct pcb *pcb)

sys/arm/arm/pmap-v6.c

6312

pcb->pcb_pagedir = pmap_ttb_get(pmap);

sys/arm/arm/pmap-v6.c

6368

pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t size)

sys/arm/arm/pmap-v6.c

6381

pa = pmap_extract(pmap, va); /* offset is preserved */

sys/arm/arm/pmap-v6.c

6410

pmap_fault(pmap_t pmap, vm_offset_t far, uint32_t fsr, int idx, bool usermode)

sys/arm/arm/pmap-v6.c

6415

if (pmap == NULL)

sys/arm/arm/pmap-v6.c

6416

pmap = kernel_pmap;

sys/arm/arm/pmap-v6.c

6424

if (__predict_false(IN_RANGE2(far, pmap->pm_pt1, NB_IN_PT1))) {

sys/arm/arm/pmap-v6.c

6433

__func__, pmap, pmap->pm_pt1, far);

sys/arm/arm/pmap-v6.c

6449

__func__, pmap, PT2MAP, far);

sys/arm/arm/pmap-v6.c

6476

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

6478

pte1 = pte1_load(pmap_pte1(pmap, far));

sys/arm/arm/pmap-v6.c

6487

pte2 = pt2tab_load(pmap_pt2tab_entry(pmap, far));

sys/arm/arm/pmap-v6.c

6490

__func__, pmap, far);

sys/arm/arm/pmap-v6.c

6501

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6513

pte1 = pte1_load(pmap_pte1(pmap, far));

sys/arm/arm/pmap-v6.c

6520

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6529

__func__, pmap, far);

sys/arm/arm/pmap-v6.c

6530

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6535

pte1p = pmap_pte1(pmap, far);

sys/arm/arm/pmap-v6.c

6539

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6547

__func__, pmap, far);

sys/arm/arm/pmap-v6.c

6548

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6562

pte1 = pte1_load(pmap_pte1(pmap, far));

sys/arm/arm/pmap-v6.c

6571

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6580

__func__, pmap, far);

sys/arm/arm/pmap-v6.c

6581

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6586

pte1p = pmap_pte1(pmap, far);

sys/arm/arm/pmap-v6.c

6592

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6601

__func__, pmap, far);

sys/arm/arm/pmap-v6.c

6602

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6612

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6649

pmap_t pmap;

sys/arm/arm/pmap-v6.c

6659

pmap = vmspace_pmap(p->p_vmspace);

sys/arm/arm/pmap-v6.c

6668

pte1 = pte1_load(&pmap->pm_pt1[i]);

sys/arm/arm/pmap-v6.c

6685

pte2p = pmap_pte2(pmap, va);

sys/arm/arm/pmap-v6.c

6716

pmap_pte2_ddb(pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

6721

pte1 = pte1_load(pmap_pte1(pmap, va));

sys/arm/arm/pmap-v6.c

6725

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

6747

dump_pmap(pmap_t pmap)

sys/arm/arm/pmap-v6.c

6750

printf("pmap %p\n", pmap);

sys/arm/arm/pmap-v6.c

6751

printf(" pm_pt1: %p\n", pmap->pm_pt1);

sys/arm/arm/pmap-v6.c

6752

printf(" pm_pt2tab: %p\n", pmap->pm_pt2tab);

sys/arm/arm/pmap-v6.c

6753

printf(" pm_active: 0x%08lX\n", pmap->pm_active.__bits[0]);

sys/arm/arm/pmap-v6.c

6759

pmap_t pmap;

sys/arm/arm/pmap-v6.c

6760

LIST_FOREACH(pmap, &allpmaps, pm_list) {

sys/arm/arm/pmap-v6.c

6761

dump_pmap(pmap);

sys/arm/arm/pmap-v6.c

6776

dump_section(pmap_t pmap, uint32_t pte1_idx)

sys/arm/arm/pmap-v6.c

6781

dump_link(pmap_t pmap, uint32_t pte1_idx, bool invalid_ok)

sys/arm/arm/pmap-v6.c

6789

pte2p = pmap_pte2_ddb(pmap, va);

sys/arm/arm/pmap-v6.c

6823

DB_SHOW_COMMAND(pmap, pmap_pmap_print)

sys/arm/arm/pmap-v6.c

6826

pmap_t pmap = (pmap_t)addr;

sys/arm/arm/pmap-v6.c

6838

if (pm == pmap) break;

sys/arm/arm/pmap-v6.c

6840

printf("given pmap %p is not in allpmaps list\n", pmap);

sys/arm/arm/pmap-v6.c

6844

pmap = PCPU_GET(curpmap);

sys/arm/arm/pmap-v6.c

6849

printf("pmap: 0x%08X\n", (uint32_t)pmap);

sys/arm/arm/pmap-v6.c

6851

printf("pt2tab: 0x%08X\n", (uint32_t)pmap->pm_pt2tab);

sys/arm/arm/pmap-v6.c

6854

pte1 = pte1_load(&pmap->pm_pt1[i]);

sys/arm/arm/pmap-v6.c

6864

dump_section(pmap, i);

sys/arm/arm/pmap-v6.c

6868

pte2 = pte2_load(pmap_pt2tab_entry(pmap, va));

sys/arm/arm/pmap-v6.c

6889

dump_link(pmap, i, invalid_ok);

sys/arm/arm/pmap-v6.c

6896

dump_pt2tab(pmap_t pmap)

sys/arm/arm/pmap-v6.c

6906

pte2 = pte2_load(&pmap->pm_pt2tab[i]);

sys/arm/arm/pmap-v6.c

6924

pmap_t pmap = (pmap_t)addr;

sys/arm/arm/pmap-v6.c

6935

pmap = PCPU_GET(curpmap);

sys/arm/arm/pmap-v6.c

6936

printf("curpmap: 0x%08X\n", (uint32_t)pmap);

sys/arm/arm/pmap-v6.c

6938

printf("pt2tab: 0x%08X\n", (uint32_t)pmap->pm_pt2tab);

sys/arm/arm/pmap-v6.c

6942

pte1 = pte1_load(&pmap->pm_pt1[i]);

sys/arm/arm/pmap-v6.c

6949

dump_section(pmap, i);

sys/arm/arm/pmap-v6.c

6951

pte2 = pte2_load(pmap_pt2tab_entry(pmap, va));

sys/arm/arm/pmap-v6.c

6959

dump_pt2tab(pmap);

sys/arm/include/pcpu.h

52

struct pmap *pc_curpmap; \

sys/arm/include/pmap.h

108

LIST_ENTRY(pmap) pm_list; /* List of all pmaps */

sys/arm/include/pmap.h

111

typedef struct pmap *pmap_t;

sys/arm/include/pmap.h

114

extern struct pmap kernel_pmap_store;

sys/arm/include/pmap.h

117

#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)

sys/arm/include/pmap.h

118

#define PMAP_LOCK_ASSERT(pmap, type) \

sys/arm/include/pmap.h

119

mtx_assert(&(pmap)->pm_mtx, (type))

sys/arm/include/pmap.h

120

#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)

sys/arm/include/pmap.h

121

#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \

sys/arm/include/pmap.h

123

#define PMAP_LOCKED(pmap) mtx_owned(&(pmap)->pm_mtx)

sys/arm/include/pmap.h

124

#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)

sys/arm/include/pmap.h

125

#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)

sys/arm/include/pmap.h

126

#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)

sys/arm/include/pmap.h

141

bool pmap_ps_enabled(pmap_t pmap);

sys/arm/include/pmap.h

172

#define pmap_map_delete(pmap, sva, eva) pmap_remove(pmap, sva, eva)

sys/arm/include/pmap_var.h

446

pmap_pte1(pmap_t pmap, vm_offset_t va)

sys/arm/include/pmap_var.h

449

return (pte1_ptr(pmap->pm_pt1, va));

sys/arm/include/pmap_var.h

460

pmap_pt2tab_entry(pmap_t pmap, vm_offset_t va)

sys/arm/include/pmap_var.h

463

return (pt2tab_entry(pmap->pm_pt2tab, va));

sys/arm/include/pmap_var.h

474

pmap_pt2_page(pmap_t pmap, vm_offset_t va)

sys/arm/include/pmap_var.h

478

pte2 = pte2_load(pmap_pt2tab_entry(pmap, va));

sys/arm64/arm64/busdma_bounce.c

119

pmap_t pmap, void *buf, bus_size_t buflen, int flags);

sys/arm64/arm64/busdma_bounce.c

678

_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,

sys/arm64/arm64/busdma_bounce.c

703

if (pmap == kernel_pmap)

sys/arm64/arm64/busdma_bounce.c

706

paddr = pmap_extract(pmap, vaddr);

sys/arm64/arm64/busdma_bounce.c

806

bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,

sys/arm64/arm64/busdma_bounce.c

827

_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);

sys/arm64/arm64/busdma_bounce.c

849

if (__predict_true(pmap == kernel_pmap)) {

sys/arm64/arm64/busdma_bounce.c

853

curaddr = pmap_extract(pmap, vaddr);

sys/arm64/arm64/pmap.c

162

#define PMAP_ASSERT_STAGE1(pmap) MPASS((pmap)->pm_stage == PM_STAGE1)

sys/arm64/arm64/pmap.c

163

#define PMAP_ASSERT_STAGE2(pmap) MPASS((pmap)->pm_stage == PM_STAGE2)

sys/arm64/arm64/pmap.c

1919

pmap_s1_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only)

sys/arm64/arm64/pmap.c

1923

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

1927

if (pmap == kernel_pmap) {

sys/arm64/arm64/pmap.c

1930

r |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));

sys/arm64/arm64/pmap.c

1942

pmap_s2_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only)

sys/arm64/arm64/pmap.c

1944

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

1946

pmap_stage2_invalidate_range(pmap_to_ttbr0(pmap), va, va + PAGE_SIZE,

sys/arm64/arm64/pmap.c

1951

pmap_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only)

sys/arm64/arm64/pmap.c

1953

if (pmap->pm_stage == PM_STAGE1)

sys/arm64/arm64/pmap.c

1954

pmap_s1_invalidate_page(pmap, va, final_only);

sys/arm64/arm64/pmap.c

1956

pmap_s2_invalidate_page(pmap, va, final_only);

sys/arm64/arm64/pmap.c

1964

pmap_s1_invalidate_strided(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/arm64/arm64/pmap.c

1969

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

1972

if (pmap == kernel_pmap) {

sys/arm64/arm64/pmap.c

1978

start = end = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));

sys/arm64/arm64/pmap.c

1997

pmap_s1_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/arm64/arm64/pmap.c

2000

pmap_s1_invalidate_strided(pmap, sva, eva, L3_SIZE, final_only);

sys/arm64/arm64/pmap.c

2004

pmap_s2_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/arm64/arm64/pmap.c

2007

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

2009

pmap_stage2_invalidate_range(pmap_to_ttbr0(pmap), sva, eva, final_only);

sys/arm64/arm64/pmap.c

2013

pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/arm64/arm64/pmap.c

2016

if (pmap->pm_stage == PM_STAGE1)

sys/arm64/arm64/pmap.c

2017

pmap_s1_invalidate_range(pmap, sva, eva, final_only);

sys/arm64/arm64/pmap.c

2019

pmap_s2_invalidate_range(pmap, sva, eva, final_only);

sys/arm64/arm64/pmap.c

2040

pmap_s1_invalidate_all(pmap_t pmap)

sys/arm64/arm64/pmap.c

2044

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

2047

if (pmap == kernel_pmap) {

sys/arm64/arm64/pmap.c

2050

r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));

sys/arm64/arm64/pmap.c

2062

pmap_s2_invalidate_all(pmap_t pmap)

sys/arm64/arm64/pmap.c

2064

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

2066

pmap_stage2_invalidate_all(pmap_to_ttbr0(pmap));

sys/arm64/arm64/pmap.c

2070

pmap_invalidate_all(pmap_t pmap)

sys/arm64/arm64/pmap.c

2072

if (pmap->pm_stage == PM_STAGE1)

sys/arm64/arm64/pmap.c

2073

pmap_s1_invalidate_all(pmap);

sys/arm64/arm64/pmap.c

2075

pmap_s2_invalidate_all(pmap);

sys/arm64/arm64/pmap.c

2085

pmap_extract(pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

2092

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

2097

pte = pmap_pte(pmap, va, &lvl);

sys/arm64/arm64/pmap.c

2123

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

2135

pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/arm64/arm64/pmap.c

2144

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

2145

pte = pmap_pte(pmap, va, &lvl);

sys/arm64/arm64/pmap.c

2163

else if (pmap->pm_stage == PM_STAGE1 &&

sys/arm64/arm64/pmap.c

2166

else if (pmap->pm_stage == PM_STAGE2 &&

sys/arm64/arm64/pmap.c

2188

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

2639

pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/arm64/arm64/pmap.c

2644

_pmap_unwire_l3(pmap, va, m, free);

sys/arm64/arm64/pmap.c

2651

_pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/arm64/arm64/pmap.c

2654

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

2662

l0 = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

2668

l1 = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

2674

l2 = pmap_l2(pmap, va);

sys/arm64/arm64/pmap.c

2677

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

2683

l1 = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

2686

pmap_unwire_l3(pmap, va, l2pg, free);

sys/arm64/arm64/pmap.c

2692

l0 = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

2695

pmap_unwire_l3(pmap, va, l1pg, free);

sys/arm64/arm64/pmap.c

2697

pmap_invalidate_page(pmap, va, false);

sys/arm64/arm64/pmap.c

2711

pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde,

sys/arm64/arm64/pmap.c

2722

return (pmap_unwire_l3(pmap, va, mpte, free));

sys/arm64/arm64/pmap.c

2730

pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte)

sys/arm64/arm64/pmap.c

2735

if (pmap_unwire_l3(pmap, va, mpte, &free))

sys/arm64/arm64/pmap.c

2740

pmap_pinit0(pmap_t pmap)

sys/arm64/arm64/pmap.c

2743

PMAP_LOCK_INIT(pmap);

sys/arm64/arm64/pmap.c

2744

bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));

sys/arm64/arm64/pmap.c

2745

pmap->pm_l0_paddr = READ_SPECIALREG(ttbr0_el1);

sys/arm64/arm64/pmap.c

2746

pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->pm_l0_paddr);

sys/arm64/arm64/pmap.c

2747

TAILQ_INIT(&pmap->pm_pvchunk);

sys/arm64/arm64/pmap.c

2748

vm_radix_init(&pmap->pm_root);

sys/arm64/arm64/pmap.c

2749

pmap->pm_cookie = COOKIE_FROM(ASID_RESERVED_FOR_PID_0, INT_MIN);

sys/arm64/arm64/pmap.c

2750

pmap->pm_stage = PM_STAGE1;

sys/arm64/arm64/pmap.c

2751

pmap->pm_levels = 4;

sys/arm64/arm64/pmap.c

2752

pmap->pm_ttbr = pmap->pm_l0_paddr;

sys/arm64/arm64/pmap.c

2753

pmap->pm_asid_set = &asids;

sys/arm64/arm64/pmap.c

2754

pmap->pm_bti = NULL;

sys/arm64/arm64/pmap.c

2756

PCPU_SET(curpmap, pmap);

sys/arm64/arm64/pmap.c

2760

pmap_pinit_stage(pmap_t pmap, enum pmap_stage stage, int levels)

sys/arm64/arm64/pmap.c

2769

pmap->pm_l0_paddr = VM_PAGE_TO_PHYS(m);

sys/arm64/arm64/pmap.c

2770

pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->pm_l0_paddr);

sys/arm64/arm64/pmap.c

2772

TAILQ_INIT(&pmap->pm_pvchunk);

sys/arm64/arm64/pmap.c

2773

vm_radix_init(&pmap->pm_root);

sys/arm64/arm64/pmap.c

2774

bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));

sys/arm64/arm64/pmap.c

2775

pmap->pm_cookie = COOKIE_FROM(-1, INT_MAX);

sys/arm64/arm64/pmap.c

2778

pmap->pm_levels = levels;

sys/arm64/arm64/pmap.c

2779

pmap->pm_stage = stage;

sys/arm64/arm64/pmap.c

2780

pmap->pm_bti = NULL;

sys/arm64/arm64/pmap.c

2783

pmap->pm_asid_set = &asids;

sys/arm64/arm64/pmap.c

2785

pmap->pm_bti = malloc(sizeof(struct rangeset), M_DEVBUF,

sys/arm64/arm64/pmap.c

2787

rangeset_init(pmap->pm_bti, bti_dup_range,

sys/arm64/arm64/pmap.c

2788

bti_free_range, pmap, M_NOWAIT);

sys/arm64/arm64/pmap.c

2792

pmap->pm_asid_set = &vmids;

sys/arm64/arm64/pmap.c

2800

pmap_alloc_asid(pmap);

sys/arm64/arm64/pmap.c

2807

if (pmap->pm_levels == 3) {

sys/arm64/arm64/pmap.c

2808

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

2809

m = _pmap_alloc_l3(pmap, NUL2E + NUL1E, NULL);

sys/arm64/arm64/pmap.c

2810

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

2812

pmap->pm_ttbr = VM_PAGE_TO_PHYS(m);

sys/arm64/arm64/pmap.c

2818

pmap_pinit(pmap_t pmap)

sys/arm64/arm64/pmap.c

2821

return (pmap_pinit_stage(pmap, PM_STAGE1, 4));

sys/arm64/arm64/pmap.c

2836

_pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)

sys/arm64/arm64/pmap.c

2840

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

2848

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

2850

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

2881

l0p = &pmap->pm_l0[l0index];

sys/arm64/arm64/pmap.c

2892

if (pmap == kernel_pmap)

sys/arm64/arm64/pmap.c

2905

l0 = &pmap->pm_l0[l0index];

sys/arm64/arm64/pmap.c

2909

if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index,

sys/arm64/arm64/pmap.c

2933

l0 = &pmap->pm_l0[l0index];

sys/arm64/arm64/pmap.c

2937

if (_pmap_alloc_l3(pmap, NUL2E + l1index,

sys/arm64/arm64/pmap.c

2952

if (_pmap_alloc_l3(pmap, NUL2E + l1index,

sys/arm64/arm64/pmap.c

2971

pmap_resident_count_inc(pmap, 1);

sys/arm64/arm64/pmap.c

2977

pmap_alloc_l2(pmap_t pmap, vm_offset_t va, vm_page_t *l2pgp,

sys/arm64/arm64/pmap.c

2988

l1 = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

3000

l2pg = _pmap_alloc_l3(pmap, NUL2E + l2pindex, lockp);

sys/arm64/arm64/pmap.c

3017

pmap_alloc_l3(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)

sys/arm64/arm64/pmap.c

303

struct pmap kernel_pmap_store;

sys/arm64/arm64/pmap.c

3035

pde = pmap_pde(pmap, va, &lvl);

sys/arm64/arm64/pmap.c

3074

m = _pmap_alloc_l3(pmap, ptepindex, lockp);

sys/arm64/arm64/pmap.c

3091

pmap_release(pmap_t pmap)

sys/arm64/arm64/pmap.c

3099

if (pmap->pm_levels != 4) {

sys/arm64/arm64/pmap.c

3100

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

3101

KASSERT(pmap->pm_stats.resident_count == 1,

sys/arm64/arm64/pmap.c

3103

pmap->pm_stats.resident_count));

sys/arm64/arm64/pmap.c

3104

KASSERT((pmap->pm_l0[0] & ATTR_DESCR_VALID) == ATTR_DESCR_VALID,

sys/arm64/arm64/pmap.c

3105

("pmap_release: Invalid l0 entry: %lx", pmap->pm_l0[0]));

sys/arm64/arm64/pmap.c

3108

m = PHYS_TO_VM_PAGE(pmap->pm_ttbr);

sys/arm64/arm64/pmap.c

3109

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

3110

rv = pmap_unwire_l3(pmap, 0, m, &freelist);

sys/arm64/arm64/pmap.c

3111

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

3116

KASSERT(pmap->pm_stats.resident_count == 0,

sys/arm64/arm64/pmap.c

3118

pmap->pm_stats.resident_count));

sys/arm64/arm64/pmap.c

3119

KASSERT(vm_radix_is_empty(&pmap->pm_root),

sys/arm64/arm64/pmap.c

3122

set = pmap->pm_asid_set;

sys/arm64/arm64/pmap.c

3131

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

3133

if (COOKIE_TO_EPOCH(pmap->pm_cookie) == set->asid_epoch) {

sys/arm64/arm64/pmap.c

3134

asid = COOKIE_TO_ASID(pmap->pm_cookie);

sys/arm64/arm64/pmap.c

3142

if (pmap->pm_bti != NULL) {

sys/arm64/arm64/pmap.c

3143

rangeset_fini(pmap->pm_bti);

sys/arm64/arm64/pmap.c

3144

free(pmap->pm_bti, M_DEVBUF);

sys/arm64/arm64/pmap.c

3148

m = PHYS_TO_VM_PAGE(pmap->pm_l0_paddr);

sys/arm64/arm64/pmap.c

3303

pmap_t next_pmap, pmap;

sys/arm64/arm64/pmap.c

3315

pmap = NULL;

sys/arm64/arm64/pmap.c

3347

if (pmap != next_pmap) {

sys/arm64/arm64/pmap.c

3348

if (pmap != NULL && pmap != locked_pmap)

sys/arm64/arm64/pmap.c

3349

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

3350

pmap = next_pmap;

sys/arm64/arm64/pmap.c

3352

if (pmap > locked_pmap) {

sys/arm64/arm64/pmap.c

3354

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

3357

} else if (pmap != locked_pmap) {

sys/arm64/arm64/pmap.c

3358

if (PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

3362

pmap = NULL; /* pmap is not locked */

sys/arm64/arm64/pmap.c

3383

pde = pmap_pde(pmap, va, &lvl);

sys/arm64/arm64/pmap.c

3391

(void)pmap_demote_l3c(pmap, pte, va);

sys/arm64/arm64/pmap.c

3394

if (pmap_pte_dirty(pmap, tpte))

sys/arm64/arm64/pmap.c

3397

pmap_s1_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

3412

pmap_unuse_pt(pmap, va, pmap_load(pde), &free);

sys/arm64/arm64/pmap.c

3421

pmap_resident_count_dec(pmap, freed);

sys/arm64/arm64/pmap.c

3425

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3437

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3440

if (pmap == locked_pmap)

sys/arm64/arm64/pmap.c

3446

if (pvc->active_reclaims == 1 && pmap != NULL) {

sys/arm64/arm64/pmap.c

3465

if (pmap != NULL && pmap != locked_pmap)

sys/arm64/arm64/pmap.c

3466

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

3498

free_pv_entry(pmap_t pmap, pv_entry_t pv)

sys/arm64/arm64/pmap.c

3503

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3514

if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {

sys/arm64/arm64/pmap.c

3515

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3516

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3520

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3585

get_pv_entry(pmap_t pmap, struct rwlock **lockp)

sys/arm64/arm64/pmap.c

3593

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3596

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/arm64/arm64/pmap.c

3609

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

361

static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,

sys/arm64/arm64/pmap.c

3610

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,

sys/arm64/arm64/pmap.c

3625

m = reclaim_pv_chunk(pmap, lockp);

sys/arm64/arm64/pmap.c

3633

pc->pc_pmap = pmap;

sys/arm64/arm64/pmap.c

3641

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3654

reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)

sys/arm64/arm64/pmap.c

3663

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3676

TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {

sys/arm64/arm64/pmap.c

3688

m = reclaim_pv_chunk(pmap, lockp);

sys/arm64/arm64/pmap.c

3697

pc->pc_pmap = pmap;

sys/arm64/arm64/pmap.c

3699

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3728

pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

3733

if (pmap == PV_PMAP(pv) && va == pv->pv_va) {

sys/arm64/arm64/pmap.c

3748

pmap_pv_demote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/arm64/arm64/pmap.c

3758

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3771

pv = pmap_pvh_remove(pvh, pmap, va);

sys/arm64/arm64/pmap.c

3780

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/arm64/arm64/pmap.c

3798

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3799

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3803

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3804

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3816

pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

3820

pv = pmap_pvh_remove(pvh, pmap, va);

sys/arm64/arm64/pmap.c

3822

free_pv_entry(pmap, pv);

sys/arm64/arm64/pmap.c

3830

pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm64/arm64/pmap.c

3835

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3837

if ((pv = get_pv_entry(pmap, NULL)) != NULL) {

sys/arm64/arm64/pmap.c

3853

pmap_pv_insert_l2(pmap_t pmap, vm_offset_t va, pd_entry_t l2e, u_int flags,

sys/arm64/arm64/pmap.c

3860

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3862

if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?

sys/arm64/arm64/pmap.c

3879

pmap_pv_insert_l3c(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm64/arm64/pmap.c

3887

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3896

pv = get_pv_entry(pmap, NULL);

sys/arm64/arm64/pmap.c

3901

pmap_pvh_free(&mt->md, pmap, tva);

sys/arm64/arm64/pmap.c

3913

pmap_remove_kernel_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va)

sys/arm64/arm64/pmap.c

3920

KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));

sys/arm64/arm64/pmap.c

3921

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3923

ml3 = pmap_remove_pt_page(pmap, va);

sys/arm64/arm64/pmap.c

3949

pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva, pd_entry_t l1e,

sys/arm64/arm64/pmap.c

3956

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3966

pmap_s1_invalidate_page(pmap, sva, true);

sys/arm64/arm64/pmap.c

3969

pmap->pm_stats.wired_count -= L2_SIZE / PAGE_SIZE;

sys/arm64/arm64/pmap.c

3970

pmap_resident_count_dec(pmap, L2_SIZE / PAGE_SIZE);

sys/arm64/arm64/pmap.c

3975

pmap_pvh_free(pvh, pmap, sva);

sys/arm64/arm64/pmap.c

3977

if (pmap_pte_dirty(pmap, old_l2))

sys/arm64/arm64/pmap.c

3986

if (pmap != kernel_pmap) {

sys/arm64/arm64/pmap.c

3987

ml3 = pmap_remove_pt_page(pmap, sva);

sys/arm64/arm64/pmap.c

3991

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

3998

pmap_remove_kernel_l2(pmap, l2, sva);

sys/arm64/arm64/pmap.c

4000

ml3 = vm_radix_lookup(&pmap->pm_root, pmap_l2_pindex(sva));

sys/arm64/arm64/pmap.c

4006

return (pmap_unuse_pt(pmap, sva, l1e, free));

sys/arm64/arm64/pmap.c

4013

pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va,

sys/arm64/arm64/pmap.c

4020

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4023

(void)pmap_demote_l3c(pmap, l3, va);

sys/arm64/arm64/pmap.c

4025

pmap_s1_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

4027

pmap->pm_stats.wired_count -= 1;

sys/arm64/arm64/pmap.c

4028

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

4031

if (pmap_pte_dirty(pmap, old_l3))

sys/arm64/arm64/pmap.c

4036

pmap_pvh_free(&m->md, pmap, va);

sys/arm64/arm64/pmap.c

4044

return (pmap_unuse_pt(pmap, va, l2e, free));

sys/arm64/arm64/pmap.c

4054

pmap_remove_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va, vm_offset_t *vap,

sys/arm64/arm64/pmap.c

4064

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4086

pmap->pm_stats.wired_count -= L3C_ENTRIES;

sys/arm64/arm64/pmap.c

4087

pmap_resident_count_dec(pmap, L3C_ENTRIES);

sys/arm64/arm64/pmap.c

4102

pmap_invalidate_range(pmap, *vap, va,

sys/arm64/arm64/pmap.c

4114

if (pmap_pte_dirty(pmap, first_l3e))

sys/arm64/arm64/pmap.c

4118

pmap_pvh_free(&mt->md, pmap, tva);

sys/arm64/arm64/pmap.c

4129

_pmap_unwire_l3(pmap, va, ml3, free);

sys/arm64/arm64/pmap.c

4141

pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva,

sys/arm64/arm64/pmap.c

4155

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4164

pmap_invalidate_range(pmap, va, sva, true);

sys/arm64/arm64/pmap.c

4177

if (pmap_remove_l3c(pmap, l3, sva, &va, eva,

sys/arm64/arm64/pmap.c

4188

(void)pmap_demote_l3c(pmap, l3, sva);

sys/arm64/arm64/pmap.c

4192

pmap->pm_stats.wired_count--;

sys/arm64/arm64/pmap.c

4193

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

4196

if (pmap_pte_dirty(pmap, old_l3))

sys/arm64/arm64/pmap.c

4212

pmap_invalidate_range(pmap, va,

sys/arm64/arm64/pmap.c

4221

pmap_pvh_free(&m->md, pmap, sva);

sys/arm64/arm64/pmap.c

4229

if (l3pg != NULL && pmap_unwire_l3(pmap, sva, l3pg, free)) {

sys/arm64/arm64/pmap.c

4243

pmap_invalidate_range(pmap, va, sva, true);

sys/arm64/arm64/pmap.c

4247

pmap_remove1(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, bool map_delete)

sys/arm64/arm64/pmap.c

4258

if (pmap->pm_stats.resident_count == 0)

sys/arm64/arm64/pmap.c

4263

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

4265

pmap_bti_on_remove(pmap, sva, eva);

sys/arm64/arm64/pmap.c

4269

if (pmap->pm_stats.resident_count == 0)

sys/arm64/arm64/pmap.c

4272

l0 = pmap_l0(pmap, sva);

sys/arm64/arm64/pmap.c

4292

MPASS(pmap != kernel_pmap);

sys/arm64/arm64/pmap.c

4295

pmap_s1_invalidate_page(pmap, sva, true);

sys/arm64/arm64/pmap.c

4296

pmap_resident_count_dec(pmap, L1_SIZE / PAGE_SIZE);

sys/arm64/arm64/pmap.c

4297

pmap_unuse_pt(pmap, sva, pmap_load(l0), &free);

sys/arm64/arm64/pmap.c

4313

pmap_remove_l2(pmap, l2, sva, pmap_load(l1),

sys/arm64/arm64/pmap.c

4316

} else if (pmap_demote_l2_locked(pmap, l2, sva,

sys/arm64/arm64/pmap.c

4336

pmap_remove_l3_range(pmap, l3_paddr, sva, va_next, &free,

sys/arm64/arm64/pmap.c

4341

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

4352

pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm64/arm64/pmap.c

4354

pmap_remove1(pmap, sva, eva, false);

sys/arm64/arm64/pmap.c

4364

pmap_map_delete(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm64/arm64/pmap.c

4366

pmap_remove1(pmap, sva, eva, true);

sys/arm64/arm64/pmap.c

4387

pmap_t pmap;

sys/arm64/arm64/pmap.c

4403

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

4404

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

4407

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

4410

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

4415

pte = pmap_pte_exists(pmap, va, 2, __func__);

sys/arm64/arm64/pmap.c

4416

pmap_demote_l2_locked(pmap, pte, va, &lock);

sys/arm64/arm64/pmap.c

4417

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

4420

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

4421

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

4425

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

4428

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

4432

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

4434

pde = pmap_pde(pmap, pv->pv_va, &lvl);

sys/arm64/arm64/pmap.c

4444

(void)pmap_demote_l3c(pmap, pte, pv->pv_va);

sys/arm64/arm64/pmap.c

4447

pmap->pm_stats.wired_count--;

sys/arm64/arm64/pmap.c

4449

pmap_invalidate_page(pmap, pv->pv_va, true);

sys/arm64/arm64/pmap.c

4456

if (pmap_pte_dirty(pmap, tpte))

sys/arm64/arm64/pmap.c

4458

pmap_unuse_pt(pmap, pv->pv_va, tpde, &free);

sys/arm64/arm64/pmap.c

4461

free_pv_entry(pmap, pv);

sys/arm64/arm64/pmap.c

4462

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

4473

pmap_protect_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva, pt_entry_t mask,

sys/arm64/arm64/pmap.c

4479

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4480

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

4504

pmap_pte_dirty(pmap, old_l2)) {

sys/arm64/arm64/pmap.c

4514

pmap_s1_invalidate_page(pmap, sva, true);

sys/arm64/arm64/pmap.c

4524

pmap_mask_set_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va,

sys/arm64/arm64/pmap.c

4531

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4570

pmap_mask_set_locked(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t mask,

sys/arm64/arm64/pmap.c

4577

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4579

l0 = pmap_l0(pmap, sva);

sys/arm64/arm64/pmap.c

4603

pmap_s1_invalidate_page(pmap, sva, true);

sys/arm64/arm64/pmap.c

4618

pmap_protect_l2(pmap, l2, sva, mask, nbits);

sys/arm64/arm64/pmap.c

4621

pmap_demote_l2(pmap, l2, sva) == NULL)

sys/arm64/arm64/pmap.c

4646

pmap_s1_invalidate_range(pmap,

sys/arm64/arm64/pmap.c

468

static void free_pv_entry(pmap_t pmap, pv_entry_t pv);

sys/arm64/arm64/pmap.c

4680

pmap_mask_set_l3c(pmap, l3p, sva, &va,

sys/arm64/arm64/pmap.c

4687

(void)pmap_demote_l3c(pmap, l3p, sva);

sys/arm64/arm64/pmap.c

469

static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp);

sys/arm64/arm64/pmap.c

4704

pmap_pte_dirty(pmap, l3))

sys/arm64/arm64/pmap.c

471

static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);

sys/arm64/arm64/pmap.c

4711

pmap_s1_invalidate_range(pmap, va, sva, true);

sys/arm64/arm64/pmap.c

4716

pmap_mask_set(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t mask,

sys/arm64/arm64/pmap.c

4719

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

472

static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,

sys/arm64/arm64/pmap.c

4720

pmap_mask_set_locked(pmap, sva, eva, mask, nbits, invalidate);

sys/arm64/arm64/pmap.c

4721

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

4729

pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)

sys/arm64/arm64/pmap.c

4733

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

4736

pmap_remove(pmap, sva, eva);

sys/arm64/arm64/pmap.c

4749

if (pmap == kernel_pmap) {

sys/arm64/arm64/pmap.c

475

static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);

sys/arm64/arm64/pmap.c

4756

pmap_mask_set(pmap, sva, eva, mask, nbits, true);

sys/arm64/arm64/pmap.c

476

static bool pmap_activate_int(struct thread *td, pmap_t pmap);

sys/arm64/arm64/pmap.c

477

static void pmap_alloc_asid(pmap_t pmap);

sys/arm64/arm64/pmap.c

4788

pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/arm64/arm64/pmap.c

4792

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4796

return (vm_radix_insert(&pmap->pm_root, mpte));

sys/arm64/arm64/pmap.c

480

static bool pmap_copy_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va,

sys/arm64/arm64/pmap.c

4806

pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

4809

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4810

return (vm_radix_remove(&pmap->pm_root, pmap_l2_pindex(va)));

sys/arm64/arm64/pmap.c

4819

pmap_update_entry(pmap_t pmap, pd_entry_t *ptep, pd_entry_t newpte,

sys/arm64/arm64/pmap.c

482

static pt_entry_t *pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va);

sys/arm64/arm64/pmap.c

4824

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

483

static pt_entry_t *pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2,

sys/arm64/arm64/pmap.c

4847

pmap_s1_invalidate_range(pmap, va, va + size, false);

sys/arm64/arm64/pmap.c

485

static pt_entry_t *pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va);

sys/arm64/arm64/pmap.c

486

static bool pmap_demote_l2c(pmap_t pmap, pt_entry_t *l2p, vm_offset_t va);

sys/arm64/arm64/pmap.c

4860

pmap_update_strided(pmap_t pmap, pd_entry_t *ptep, pd_entry_t *ptep_end,

sys/arm64/arm64/pmap.c

4866

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

487

static bool pmap_demote_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va);

sys/arm64/arm64/pmap.c

488

static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,

sys/arm64/arm64/pmap.c

4886

pmap_s1_invalidate_strided(pmap, va, va + size, stride, true);

sys/arm64/arm64/pmap.c

490

static int pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2,

sys/arm64/arm64/pmap.c

4905

pmap_pv_promote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/arm64/arm64/pmap.c

492

static int pmap_enter_l3c(pmap_t pmap, vm_offset_t va, pt_entry_t l3e, u_int flags,

sys/arm64/arm64/pmap.c

4926

pv = pmap_pvh_remove(&m->md, pmap, va);

sys/arm64/arm64/pmap.c

4936

pmap_pvh_free(&m->md, pmap, va);

sys/arm64/arm64/pmap.c

4948

pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, vm_page_t mpte,

sys/arm64/arm64/pmap.c

495

static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/arm64/arm64/pmap.c

4953

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4960

if (pmap->pm_stage != PM_STAGE1 || !pmap_ps_enabled(pmap))

sys/arm64/arm64/pmap.c

4976

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

498

static void pmap_mask_set_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va,

sys/arm64/arm64/pmap.c

500

static bool pmap_pv_insert_l3c(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm64/arm64/pmap.c

5007

" in pmap %p", va & ~L2_OFFSET, pmap);

sys/arm64/arm64/pmap.c

502

static void pmap_remove_kernel_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va);

sys/arm64/arm64/pmap.c

5024

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

503

static int pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva,

sys/arm64/arm64/pmap.c

5043

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

506

static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva,

sys/arm64/arm64/pmap.c

5070

if (pmap_insert_pt_page(pmap, mpte, true, all_l3e_AF != 0)) {

sys/arm64/arm64/pmap.c

5074

pmap);

sys/arm64/arm64/pmap.c

5079

pmap_pv_promote_l2(pmap, va, PTE_TO_PHYS(newl2), lockp);

sys/arm64/arm64/pmap.c

508

static bool pmap_remove_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va,

sys/arm64/arm64/pmap.c

5081

pmap_update_entry(pmap, l2, newl2 | L2_BLOCK, va & ~L2_OFFSET, L2_SIZE);

sys/arm64/arm64/pmap.c

5085

pmap);

sys/arm64/arm64/pmap.c

5097

pmap_promote_l3c(pmap_t pmap, pd_entry_t *l3p, vm_offset_t va)

sys/arm64/arm64/pmap.c

5101

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

5108

if (pmap->pm_stage != PM_STAGE1 || !pmap_ps_enabled(pmap))

sys/arm64/arm64/pmap.c

511

static void pmap_reset_asid_set(pmap_t pmap);

sys/arm64/arm64/pmap.c

512

static bool pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,

sys/arm64/arm64/pmap.c

5131

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

515

static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex,

sys/arm64/arm64/pmap.c

5150

" in pmap %p", va & ~L3C_OFFSET, pmap);

sys/arm64/arm64/pmap.c

5165

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

518

static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm64/arm64/pmap.c

5182

(va & ~L3C_OFFSET), pmap);

sys/arm64/arm64/pmap.c

5187

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

5205

pmap_update_strided(pmap, l3p, l3p + L3C_ENTRIES, firstl3c |

sys/arm64/arm64/pmap.c

521

static void pmap_update_entry(pmap_t pmap, pd_entry_t *pte, pd_entry_t newpte,

sys/arm64/arm64/pmap.c

5210

pmap);

sys/arm64/arm64/pmap.c

5216

pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t pte, int flags,

sys/arm64/arm64/pmap.c

5222

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

523

static __inline vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);

sys/arm64/arm64/pmap.c

5231

if (!pmap_bti_same(pmap, va, va + pagesizes[psind], &newpte))

sys/arm64/arm64/pmap.c

5238

l0p = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

5240

mp = _pmap_alloc_l3(pmap, pmap_l0_pindex(va), NULL);

sys/arm64/arm64/pmap.c

5244

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

5246

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

526

static bool pmap_bti_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/arm64/arm64/pmap.c

5270

l2p = pmap_l2(pmap, va);

sys/arm64/arm64/pmap.c

5272

mp = _pmap_alloc_l3(pmap, pmap_l1_pindex(va), NULL);

sys/arm64/arm64/pmap.c

5276

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

5278

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

528

static pt_entry_t pmap_pte_bti(pmap_t pmap, vm_offset_t va);

sys/arm64/arm64/pmap.c

5285

l1p = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

529

static void pmap_bti_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);

sys/arm64/arm64/pmap.c

5301

l2p = pmap_l2(pmap, va);

sys/arm64/arm64/pmap.c

5303

mp = _pmap_alloc_l3(pmap, pmap_l2_pindex(va), NULL);

sys/arm64/arm64/pmap.c

5307

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

5309

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

533

static void pmap_bti_deassign_all(pmap_t pmap);

sys/arm64/arm64/pmap.c

5336

pmap_resident_count_inc(pmap, pagesizes[psind] / PAGE_SIZE);

sys/arm64/arm64/pmap.c

5338

pmap->pm_stats.wired_count += pagesizes[psind] / PAGE_SIZE;

sys/arm64/arm64/pmap.c

5341

pmap->pm_stats.wired_count -= pagesizes[psind] / PAGE_SIZE;

sys/arm64/arm64/pmap.c

5359

pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/arm64/arm64/pmap.c

5381

new_l3 |= pmap_pte_memattr(pmap, m->md.pv_memattr);

sys/arm64/arm64/pmap.c

5382

new_l3 |= pmap_pte_prot(pmap, prot);

sys/arm64/arm64/pmap.c

5385

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

5390

if (pmap != kernel_pmap)

sys/arm64/arm64/pmap.c

5411

if (pmap->pm_stage == PM_STAGE1)

sys/arm64/arm64/pmap.c

5423

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

5436

rv = pmap_enter_largepage(pmap, va, new_l3, flags, psind);

sys/arm64/arm64/pmap.c

5443

rv = pmap_enter_l2(pmap, va, (new_l3 & ~L3_PAGE) | L2_BLOCK,

sys/arm64/arm64/pmap.c

5451

rv = pmap_enter_l3c(pmap, va, new_l3 | ATTR_CONTIGUOUS, flags,

sys/arm64/arm64/pmap.c

5462

pde = pmap_l2(pmap, va);

sys/arm64/arm64/pmap.c

5463

(void)pmap_promote_l2(pmap, pde, va, mpte, &lock);

sys/arm64/arm64/pmap.c

5474

pde = pmap_pde(pmap, va, &lvl);

sys/arm64/arm64/pmap.c

5485

(l3 = pmap_demote_l2_locked(pmap, l2, va, &lock)) != NULL) {

sys/arm64/arm64/pmap.c

5503

mpte = _pmap_alloc_l3(pmap, pmap_l2_pindex(va),

sys/arm64/arm64/pmap.c

5518

new_l3 |= pmap_pte_bti(pmap, va);

sys/arm64/arm64/pmap.c

5532

pmap->pm_stats.wired_count++;

sys/arm64/arm64/pmap.c

5535

pmap->pm_stats.wired_count--;

sys/arm64/arm64/pmap.c

5565

(void)pmap_demote_l3c(pmap, l3, va);

sys/arm64/arm64/pmap.c

5577

if (pmap_pte_dirty(pmap, orig_l3))

sys/arm64/arm64/pmap.c

5580

pmap_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

5584

pv = pmap_pvh_remove(&om->md, pmap, va);

sys/arm64/arm64/pmap.c

5586

free_pv_entry(pmap, pv);

sys/arm64/arm64/pmap.c

5595

pmap_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

5603

pmap->pm_stats.wired_count++;

sys/arm64/arm64/pmap.c

5604

pmap_resident_count_inc(pmap, 1);

sys/arm64/arm64/pmap.c

561

pmap_l0(pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

5611

pv = get_pv_entry(pmap, &lock);

sys/arm64/arm64/pmap.c

5622

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

5634

if ((prot & VM_PROT_EXECUTE) && pmap != kernel_pmap &&

sys/arm64/arm64/pmap.c

5637

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

564

return (&pmap->pm_l0[pmap_l0_index(va)]);

sys/arm64/arm64/pmap.c

5653

(void)pmap_demote_l3c(pmap, l3, va);

sys/arm64/arm64/pmap.c

5655

pmap_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

5657

pmap_pte_dirty(pmap, orig_l3))

sys/arm64/arm64/pmap.c

5673

__func__, pmap, va, new_l3);

sys/arm64/arm64/pmap.c

5693

pmap_promote_l3c(pmap, l3, va) &&

sys/arm64/arm64/pmap.c

5695

(void)pmap_promote_l2(pmap, pde, va, mpte, &lock);

sys/arm64/arm64/pmap.c

5702

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

5713

pmap_enter_l2_rx(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/arm64/arm64/pmap.c

5718

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

5719

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

5737

if (pmap != kernel_pmap)

sys/arm64/arm64/pmap.c

5739

return (pmap_enter_l2(pmap, va, new_l2, PMAP_ENTER_NOSLEEP |

sys/arm64/arm64/pmap.c

577

pmap_l1(pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

5774

pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags,

sys/arm64/arm64/pmap.c

5782

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

5789

if ((l2 = pmap_alloc_l2(pmap, va, &l2pg, (flags &

sys/arm64/arm64/pmap.c

5792

va, pmap);

sys/arm64/arm64/pmap.c

5801

if (!pmap_bti_same(pmap, va, va + L2_SIZE, &new_l2)) {

sys/arm64/arm64/pmap.c

5803

pmap_abort_ptp(pmap, va, l2pg);

sys/arm64/arm64/pmap.c

581

l0 = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

5819

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

5827

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

5833

(void)pmap_remove_l2(pmap, l2, va,

sys/arm64/arm64/pmap.c

5834

pmap_load(pmap_l1(pmap, va)), false, &free, lockp);

sys/arm64/arm64/pmap.c

5843

if (pmap_insert_pt_page(pmap, mt, false,

sys/arm64/arm64/pmap.c

5856

pmap_remove_l3_range(pmap, old_l2, va, va + L2_SIZE,

sys/arm64/arm64/pmap.c

5867

pmap_s1_invalidate_page(pmap, va, false);

sys/arm64/arm64/pmap.c

5884

if ((new_l2 & ATTR_SW_WIRED) != 0 && pmap != kernel_pmap) {

sys/arm64/arm64/pmap.c

5887

pmap_abort_ptp(pmap, va, l2pg);

sys/arm64/arm64/pmap.c

5891

if (pmap_insert_pt_page(pmap, uwptpg, true, false)) {

sys/arm64/arm64/pmap.c

5894

pmap_abort_ptp(pmap, va, l2pg);

sys/arm64/arm64/pmap.c

5897

pmap_resident_count_inc(pmap, 1);

sys/arm64/arm64/pmap.c

5904

if (!pmap_pv_insert_l2(pmap, va, new_l2, flags, lockp)) {

sys/arm64/arm64/pmap.c

5906

pmap_abort_ptp(pmap, va, l2pg);

sys/arm64/arm64/pmap.c

5912

mt = pmap_remove_pt_page(pmap, va);

sys/arm64/arm64/pmap.c

5917

mt = pmap_remove_pt_page(pmap, va);

sys/arm64/arm64/pmap.c

5921

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

5928

va, pmap);

sys/arm64/arm64/pmap.c

5940

pmap->pm_stats.wired_count += L2_SIZE / PAGE_SIZE;

sys/arm64/arm64/pmap.c

5941

pmap->pm_stats.resident_count += L2_SIZE / PAGE_SIZE;

sys/arm64/arm64/pmap.c

5948

pmap != kernel_pmap && m->md.pv_memattr == VM_MEMATTR_WRITE_BACK) {

sys/arm64/arm64/pmap.c

5961

va, pmap);

sys/arm64/arm64/pmap.c

5972

pmap_enter_l3c_rx(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t *ml3p,

sys/arm64/arm64/pmap.c

5977

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

5978

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

5996

if (pmap != kernel_pmap)

sys/arm64/arm64/pmap.c

5998

return (pmap_enter_l3c(pmap, va, l3e, PMAP_ENTER_NOSLEEP |

sys/arm64/arm64/pmap.c

6003

pmap_enter_l3c(pmap_t pmap, vm_offset_t va, pt_entry_t l3e, u_int flags,

sys/arm64/arm64/pmap.c

6013

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

6037

pde = pmap_pde(pmap, va, &lvl);

sys/arm64/arm64/pmap.c

6049

l3p = pmap_demote_l2_locked(pmap, l2p,

sys/arm64/arm64/pmap.c

6070

*ml3p = _pmap_alloc_l3(pmap, l2pindex, (flags &

sys/arm64/arm64/pmap.c

610

pmap_l2(pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

6100

l3p = pmap_demote_l2_locked(pmap, l2p, va, lockp);

sys/arm64/arm64/pmap.c

6116

if (!pmap_bti_same(pmap, va, va + L3C_SIZE, &l3e)) {

sys/arm64/arm64/pmap.c

6119

pmap_abort_ptp(pmap, va, *ml3p);

sys/arm64/arm64/pmap.c

614

l1 = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

6141

pmap_remove_l3_range(pmap, pmap_load(pmap_l2(pmap, va)), va,

sys/arm64/arm64/pmap.c

6149

if (!pmap_pv_insert_l3c(pmap, va, m, lockp)) {

sys/arm64/arm64/pmap.c

6152

pmap_abort_ptp(pmap, va, *ml3p);

sys/arm64/arm64/pmap.c

6166

pmap->pm_stats.wired_count += L3C_ENTRIES;

sys/arm64/arm64/pmap.c

6167

pmap_resident_count_inc(pmap, L3C_ENTRIES);

sys/arm64/arm64/pmap.c

6175

if ((l3e & ATTR_S1_XN) == 0 && pmap != kernel_pmap &&

sys/arm64/arm64/pmap.c

6190

va, pmap);

sys/arm64/arm64/pmap.c

6207

pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,

sys/arm64/arm64/pmap.c

6223

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

6227

m->psind == 2 && pmap_ps_enabled(pmap) &&

sys/arm64/arm64/pmap.c

6228

((rv = pmap_enter_l2_rx(pmap, va, m, prot, &lock)) ==

sys/arm64/arm64/pmap.c

6232

m->psind >= 1 && pmap_ps_enabled(pmap) &&

sys/arm64/arm64/pmap.c

6233

((rv = pmap_enter_l3c_rx(pmap, va, m, &mpte, prot,

sys/arm64/arm64/pmap.c

6246

mpte = pmap_enter_quick_locked(pmap, va, m, prot |

sys/arm64/arm64/pmap.c

6253

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

6266

pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/arm64/arm64/pmap.c

6271

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

6272

(void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock);

sys/arm64/arm64/pmap.c

6275

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

6279

pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm64/arm64/pmap.c

6289

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

6290

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

6295

CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va);

sys/arm64/arm64/pmap.c

6318

l1 = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

6331

mpte = _pmap_alloc_l3(pmap, l2pindex,

sys/arm64/arm64/pmap.c

6337

mpte = _pmap_alloc_l3(pmap, l2pindex, NULL);

sys/arm64/arm64/pmap.c

6368

!pmap_try_insert_pv_entry(pmap, va, m, lockp)) {

sys/arm64/arm64/pmap.c

6370

pmap_abort_ptp(pmap, va, mpte);

sys/arm64/arm64/pmap.c

6377

pmap_resident_count_inc(pmap, 1);

sys/arm64/arm64/pmap.c

6382

l3_val |= pmap_pte_bti(pmap, va);

sys/arm64/arm64/pmap.c

6390

if (pmap != kernel_pmap)

sys/arm64/arm64/pmap.c

6402

if ((prot & VM_PROT_EXECUTE) && pmap != kernel_pmap &&

sys/arm64/arm64/pmap.c

6422

pmap_promote_l3c(pmap, l3, va) &&

sys/arm64/arm64/pmap.c

6425

l2 = pmap_l2(pmap, va);

sys/arm64/arm64/pmap.c

6431

if (pmap_promote_l2(pmap, l2, va, mpte, lockp))

sys/arm64/arm64/pmap.c

6445

pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,

sys/arm64/arm64/pmap.c

6464

pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm64/arm64/pmap.c

6471

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

6473

l0 = pmap_l0(pmap, sva);

sys/arm64/arm64/pmap.c

648

pmap_pde(pmap_t pmap, vm_offset_t va, int *level)

sys/arm64/arm64/pmap.c

6494

MPASS(pmap != kernel_pmap);

sys/arm64/arm64/pmap.c

6498

pmap->pm_stats.wired_count -= L1_SIZE / PAGE_SIZE;

sys/arm64/arm64/pmap.c

652

l0 = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

6521

pmap->pm_stats.wired_count -= L2_SIZE /

sys/arm64/arm64/pmap.c

6524

} else if (pmap_demote_l2(pmap, l2, sva) == NULL)

sys/arm64/arm64/pmap.c

6550

(void)pmap_demote_l3c(pmap, l3, sva);

sys/arm64/arm64/pmap.c

6562

pmap->pm_stats.wired_count--;

sys/arm64/arm64/pmap.c

6565

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

6573

pmap_copy_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va, pt_entry_t l3e,

sys/arm64/arm64/pmap.c

6578

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

6594

if (!pmap_pv_insert_l3c(pmap, va, PTE_TO_VM_PAGE(l3e), lockp)) {

sys/arm64/arm64/pmap.c

6596

pmap_abort_ptp(pmap, va, ml3);

sys/arm64/arm64/pmap.c

6612

pmap_resident_count_inc(pmap, L3C_ENTRIES);

sys/arm64/arm64/pmap.c

6615

va, pmap);

sys/arm64/arm64/pmap.c

683

pmap_pte(pmap_t pmap, vm_offset_t va, int *level)

sys/arm64/arm64/pmap.c

688

l1 = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

6945

pmap_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/arm64/arm64/pmap.c

6959

if (PV_PMAP(pv) == pmap) {

sys/arm64/arm64/pmap.c

6970

if (PV_PMAP(pv) == pmap) {

sys/arm64/arm64/pmap.c

6994

pmap_t pmap;

sys/arm64/arm64/pmap.c

7006

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7007

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7010

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7013

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7017

pte = pmap_pte_exists(pmap, pv->pv_va, 3, __func__);

sys/arm64/arm64/pmap.c

7020

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7025

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7026

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7030

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7034

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7038

pte = pmap_pte_exists(pmap, pv->pv_va, 2, __func__);

sys/arm64/arm64/pmap.c

7041

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7086

pmap_remove_pages(pmap_t pmap)

sys/arm64/arm64/pmap.c

7108

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7109

TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {

sys/arm64/arm64/pmap.c

7121

pde = pmap_pde(pmap, pv->pv_va, &lvl);

sys/arm64/arm64/pmap.c

7191

if (pmap_pte_dirty(pmap, tpte)) {

sys/arm64/arm64/pmap.c

7207

pmap_resident_count_dec(pmap,

sys/arm64/arm64/pmap.c

7218

ml3 = pmap_remove_pt_page(pmap,

sys/arm64/arm64/pmap.c

7223

pmap_resident_count_dec(pmap,1);

sys/arm64/arm64/pmap.c

7232

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

7246

pmap_unuse_pt(pmap, pv->pv_va, pmap_load(pde),

sys/arm64/arm64/pmap.c

7255

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

7262

pmap_invalidate_all(pmap);

sys/arm64/arm64/pmap.c

7263

pmap_bti_deassign_all(pmap);

sys/arm64/arm64/pmap.c

7265

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7279

pmap_t pmap;

sys/arm64/arm64/pmap.c

7288

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7289

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

7290

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7293

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7296

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7300

pte = pmap_pte_exists(pmap, pv->pv_va, 3, __func__);

sys/arm64/arm64/pmap.c

7314

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

732

pmap_pte_exists(pmap_t pmap, vm_offset_t va, int level, const char *diag)

sys/arm64/arm64/pmap.c

7322

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7323

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

7324

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7328

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7332

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7336

pte = pmap_pte_exists(pmap, pv->pv_va, 2, __func__);

sys/arm64/arm64/pmap.c

7348

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7386

pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/arm64/arm64/pmap.c

7398

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7399

pde = pmap_pde(pmap, addr, &lvl);

sys/arm64/arm64/pmap.c

7404

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

741

l0p = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

7430

pmap_t pmap;

sys/arm64/arm64/pmap.c

7448

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7449

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

7450

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7453

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7456

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7461

pte = pmap_pte_exists(pmap, va, 2, __func__);

sys/arm64/arm64/pmap.c

7463

(void)pmap_demote_l2_locked(pmap, pte, va, &lock);

sys/arm64/arm64/pmap.c

7467

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7470

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7471

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7475

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7479

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7483

pte = pmap_pte_exists(pmap, pv->pv_va, 3, __func__);

sys/arm64/arm64/pmap.c

7487

(void)pmap_demote_l3c(pmap, pte, pv->pv_va);

sys/arm64/arm64/pmap.c

7495

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

7513

pmap_invalidate_page(pmap, pv->pv_va, true);

sys/arm64/arm64/pmap.c

7515

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7542

pmap_t pmap;

sys/arm64/arm64/pmap.c

7565

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7566

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7569

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7572

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7577

pte = pmap_pte_exists(pmap, va, 2, __func__);

sys/arm64/arm64/pmap.c

7579

if (pmap_pte_dirty(pmap, tpte)) {

sys/arm64/arm64/pmap.c

7609

(uintptr_t)pmap) & (Ln_ENTRIES - 1)) == 0 &&

sys/arm64/arm64/pmap.c

7612

pmap_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

7617

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7634

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7635

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7639

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7642

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7646

pte = pmap_pte_exists(pmap, pv->pv_va, 3, __func__);

sys/arm64/arm64/pmap.c

7648

if (pmap_pte_dirty(pmap, tpte))

sys/arm64/arm64/pmap.c

7657

pmap_invalidate_page(pmap, pv->pv_va, true);

sys/arm64/arm64/pmap.c

7670

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7691

pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)

sys/arm64/arm64/pmap.c

7699

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

7704

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7706

l0 = pmap_l0(pmap, sva);

sys/arm64/arm64/pmap.c

7736

if (!pmap_demote_l2_locked(pmap, l2, sva, &lock)) {

sys/arm64/arm64/pmap.c

775

pmap_ps_enabled(pmap_t pmap)

sys/arm64/arm64/pmap.c

7765

pmap_remove_l3(pmap, l3, va, pmap_load(l2),

sys/arm64/arm64/pmap.c

7782

else if (pmap_pte_dirty(pmap, oldl3)) {

sys/arm64/arm64/pmap.c

7798

(void)pmap_demote_l3c(pmap, l3, sva);

sys/arm64/arm64/pmap.c

7815

pmap_remove_l3(pmap, dl3, dva,

sys/arm64/arm64/pmap.c

782

if (pmap->pm_stage != PM_STAGE1)

sys/arm64/arm64/pmap.c

7851

pmap_s1_invalidate_range(pmap, va, sva, true);

sys/arm64/arm64/pmap.c

7856

pmap_s1_invalidate_range(pmap, va, sva, true);

sys/arm64/arm64/pmap.c

7858

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7869

pmap_t pmap;

sys/arm64/arm64/pmap.c

7887

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7888

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

7889

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7892

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7895

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7900

l2 = pmap_l2(pmap, va);

sys/arm64/arm64/pmap.c

7904

pmap_demote_l2_locked(pmap, l2, va, &lock) &&

sys/arm64/arm64/pmap.c

7917

pmap_s1_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

7919

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7922

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7923

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

7924

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7928

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

793

if (pmap == kernel_pmap)

sys/arm64/arm64/pmap.c

7931

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7935

l2 = pmap_l2(pmap, pv->pv_va);

sys/arm64/arm64/pmap.c

7944

(void)pmap_demote_l3c(pmap, l3, pv->pv_va);

sys/arm64/arm64/pmap.c

7946

pmap_s1_invalidate_page(pmap, pv->pv_va, true);

sys/arm64/arm64/pmap.c

7948

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

801

pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t **l0, pd_entry_t **l1,

sys/arm64/arm64/pmap.c

806

if (pmap->pm_l0 == NULL)

sys/arm64/arm64/pmap.c

809

l0p = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

8418

pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va)

sys/arm64/arm64/pmap.c

8426

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

8448

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

8478

pmap_update_entry(pmap, l1, l2phys | L1_TABLE, va, PAGE_SIZE);

sys/arm64/arm64/pmap.c

8532

pmap_demote_l2_abort(pmap_t pmap, vm_offset_t va, pt_entry_t *l2,

sys/arm64/arm64/pmap.c

8538

(void)pmap_remove_l2(pmap, l2, va, pmap_load(pmap_l1(pmap, va)), true,

sys/arm64/arm64/pmap.c

854

pmap_pte_memattr(pmap_t pmap, vm_memattr_t memattr)

sys/arm64/arm64/pmap.c

8547

pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_offset_t va,

sys/arm64/arm64/pmap.c

8555

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

8556

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

858

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

8581

pmap_demote_l2_abort(pmap, va, l2, lockp);

sys/arm64/arm64/pmap.c

8584

va, pmap);

sys/arm64/arm64/pmap.c

8587

ml3 = pmap_remove_pt_page(pmap, va);

sys/arm64/arm64/pmap.c

8590

} else if ((ml3 = pmap_remove_pt_page(pmap, va)) == NULL) {

sys/arm64/arm64/pmap.c

8624

pmap_demote_l2_abort(pmap, va, l2, lockp);

sys/arm64/arm64/pmap.c

8626

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

8633

pmap_resident_count_inc(pmap, 1);

sys/arm64/arm64/pmap.c

8682

reserve_pv_entries(pmap, Ln_ENTRIES - 1, lockp);

sys/arm64/arm64/pmap.c

8688

pmap_update_entry(pmap, l2, l3phys | L2_TABLE, va, PAGE_SIZE);

sys/arm64/arm64/pmap.c

8694

pmap_pv_demote_l2(pmap, va, PTE_TO_PHYS(oldl2), lockp);

sys/arm64/arm64/pmap.c

8698

" in pmap %p %lx", va, pmap, l3[0]);

sys/arm64/arm64/pmap.c

8711

pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va)

sys/arm64/arm64/pmap.c

8717

l3 = pmap_demote_l2_locked(pmap, l2, va, &lock);

sys/arm64/arm64/pmap.c

8727

pmap_demote_l2c(pmap_t pmap, pt_entry_t *l2p, vm_offset_t va)

sys/arm64/arm64/pmap.c

8733

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

8734

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

8788

pmap_s1_invalidate_strided(pmap, va & ~L2C_OFFSET, (va +

sys/arm64/arm64/pmap.c

8809

va, pmap);

sys/arm64/arm64/pmap.c

8817

pmap_demote_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va)

sys/arm64/arm64/pmap.c

8823

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

883

pmap_pte_prot(pmap_t pmap, vm_prot_t prot)

sys/arm64/arm64/pmap.c

8877

pmap_invalidate_range(pmap, va & ~L3C_OFFSET, (va + L3C_SIZE) &

sys/arm64/arm64/pmap.c

888

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

8898

va, pmap);

sys/arm64/arm64/pmap.c

8937

pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/arm64/arm64/pmap.c

8944

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

8945

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

8946

pte = pmap_pte(pmap, addr, &lvl);

sys/arm64/arm64/pmap.c

8969

if ((managed && pmap_pte_dirty(pmap, tpte)) || (!managed &&

sys/arm64/arm64/pmap.c

8985

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

8994

pmap_reset_asid_set(pmap_t pmap)

sys/arm64/arm64/pmap.c

9001

set = pmap->pm_asid_set;

sys/arm64/arm64/pmap.c

9002

stage = pmap->pm_stage;

sys/arm64/arm64/pmap.c

9004

set = pmap->pm_asid_set;

sys/arm64/arm64/pmap.c

9033

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

9038

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

9053

pmap_alloc_asid(pmap_t pmap)

sys/arm64/arm64/pmap.c

9058

set = pmap->pm_asid_set;

sys/arm64/arm64/pmap.c

9069

if (COOKIE_TO_EPOCH(pmap->pm_cookie) == set->asid_epoch)

sys/arm64/arm64/pmap.c

9078

pmap_reset_asid_set(pmap);

sys/arm64/arm64/pmap.c

9086

pmap->pm_cookie = COOKIE_FROM(new_asid, set->asid_epoch);

sys/arm64/arm64/pmap.c

909

pmap_pte_dirty(pmap_t pmap, pt_entry_t pte)

sys/arm64/arm64/pmap.c

9098

pmap_to_ttbr0(pmap_t pmap)

sys/arm64/arm64/pmap.c

9102

ttbr = pmap->pm_ttbr;

sys/arm64/arm64/pmap.c

9103

ttbr |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));

sys/arm64/arm64/pmap.c

914

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

9186

pmap_activate_int(struct thread *td, pmap_t pmap)

sys/arm64/arm64/pmap.c

9192

KASSERT(pmap != kernel_pmap, ("kernel pmap activation"));

sys/arm64/arm64/pmap.c

9194

if ((pmap->pm_stage == PM_STAGE1 && pmap == PCPU_GET(curpmap)) ||

sys/arm64/arm64/pmap.c

9195

(pmap->pm_stage == PM_STAGE2 && pmap == PCPU_GET(curvmpmap))) {

sys/arm64/arm64/pmap.c

9208

set = pmap->pm_asid_set;

sys/arm64/arm64/pmap.c

9215

if (pmap->pm_stage == PM_STAGE1)

sys/arm64/arm64/pmap.c

9216

PCPU_SET(curpmap, pmap);

sys/arm64/arm64/pmap.c

9218

PCPU_SET(curvmpmap, pmap);

sys/arm64/arm64/pmap.c

9220

epoch = COOKIE_TO_EPOCH(pmap->pm_cookie);

sys/arm64/arm64/pmap.c

9222

pmap_alloc_asid(pmap);

sys/arm64/arm64/pmap.c

9224

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

9234

set_ttbr0(pmap_to_ttbr0(pmap));

sys/arm64/arm64/pmap.c

9242

pmap_activate_vm(pmap_t pmap)

sys/arm64/arm64/pmap.c

9245

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

9247

(void)pmap_activate_int(NULL, pmap);

sys/arm64/arm64/pmap.c

9253

pmap_t pmap;

sys/arm64/arm64/pmap.c

9255

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/arm64/arm64/pmap.c

9256

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

9258

(void)pmap_activate_int(td, pmap);

sys/arm64/arm64/pmap.c

927

pmap_resident_count_inc(pmap_t pmap, int count)

sys/arm64/arm64/pmap.c

9299

pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz)

sys/arm64/arm64/pmap.c

930

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9302

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

931

pmap->pm_stats.resident_count += count;

sys/arm64/arm64/pmap.c

9318

pa = pmap_extract(pmap, va);

sys/arm64/arm64/pmap.c

9333

pmap_stage2_fault(pmap_t pmap, uint64_t esr, uint64_t far)

sys/arm64/arm64/pmap.c

9339

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

9349

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

935

pmap_resident_count_dec(pmap_t pmap, int count)

sys/arm64/arm64/pmap.c

9350

pdep = pmap_pde(pmap, far, &lvl);

sys/arm64/arm64/pmap.c

9352

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

9374

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

9375

ptep = pmap_pte(pmap, far, &lvl);

sys/arm64/arm64/pmap.c

938

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

939

KASSERT(pmap->pm_stats.resident_count >= count,

sys/arm64/arm64/pmap.c

9392

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

940

("pmap %p resident count underflow %ld %d", pmap,

sys/arm64/arm64/pmap.c

9400

pmap_fault(pmap_t pmap, uint64_t esr, uint64_t far)

sys/arm64/arm64/pmap.c

941

pmap->pm_stats.resident_count, count));

sys/arm64/arm64/pmap.c

942

pmap->pm_stats.resident_count -= count;

sys/arm64/arm64/pmap.c

9420

if (pmap->pm_stage == PM_STAGE2)

sys/arm64/arm64/pmap.c

9421

return (pmap_stage2_fault(pmap, esr, far));

sys/arm64/arm64/pmap.c

9428

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

9429

ptep = pmap_pte(pmap, far, &lvl);

sys/arm64/arm64/pmap.c

9438

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

9446

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

9447

ptep = pmap_pte(pmap, far, &lvl);

sys/arm64/arm64/pmap.c

9453

pmap_s1_invalidate_page(pmap, far, true);

sys/arm64/arm64/pmap.c

9457

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

9467

if (pmap == kernel_pmap) {

sys/arm64/arm64/pmap.c

9484

owned = PMAP_OWNED(pmap);

sys/arm64/arm64/pmap.c

9486

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

9492

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

9627

pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)

sys/arm64/arm64/pmap.c

9654

pmap_bti_assign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm64/arm64/pmap.c

9659

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9660

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

9661

MPASS(pmap->pm_bti != NULL);

sys/arm64/arm64/pmap.c

9665

error = rangeset_insert(pmap->pm_bti, sva, eva, rs);

sys/arm64/arm64/pmap.c

9672

pmap_bti_deassign_all(pmap_t pmap)

sys/arm64/arm64/pmap.c

9675

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9676

if (pmap->pm_bti != NULL)

sys/arm64/arm64/pmap.c

9677

rangeset_remove_all(pmap->pm_bti);

sys/arm64/arm64/pmap.c

9689

pmap_bti_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte)

sys/arm64/arm64/pmap.c

9694

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9702

if (pmap == kernel_pmap) {

sys/arm64/arm64/pmap.c

9706

if (pmap->pm_bti == NULL)

sys/arm64/arm64/pmap.c

9708

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

9709

rs = rangeset_containing(pmap->pm_bti, sva);

sys/arm64/arm64/pmap.c

9711

return (rangeset_empty(pmap->pm_bti, sva, eva));

sys/arm64/arm64/pmap.c

9713

if ((rs = rangeset_beginning(pmap->pm_bti, va)) == NULL)

sys/arm64/arm64/pmap.c

9721

pmap_pte_bti(pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

9723

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9726

if (pmap->pm_stage != PM_STAGE1)

sys/arm64/arm64/pmap.c

9728

if (pmap == kernel_pmap)

sys/arm64/arm64/pmap.c

9730

if (pmap->pm_bti != NULL &&

sys/arm64/arm64/pmap.c

9731

rangeset_containing(pmap->pm_bti, va) != NULL)

sys/arm64/arm64/pmap.c

9737

pmap_bti_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm64/arm64/pmap.c

9740

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9741

if (pmap->pm_bti != NULL)

sys/arm64/arm64/pmap.c

9742

rangeset_remove(pmap->pm_bti, sva, eva);

sys/arm64/arm64/pmap.c

9760

pmap_bti_update_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, bool set)

sys/arm64/arm64/pmap.c

9762

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9763

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

9765

pmap_mask_set_locked(pmap, sva, eva, ATTR_S1_GP, set ? ATTR_S1_GP : 0,

sys/arm64/arm64/pmap.c

9770

pmap_bti_set(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm64/arm64/pmap.c

9774

if (pmap->pm_bti == NULL)

sys/arm64/arm64/pmap.c

9778

if (pmap->pm_stage != PM_STAGE1)

sys/arm64/arm64/pmap.c

9786

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

9787

error = pmap_bti_assign(pmap, sva, eva);

sys/arm64/arm64/pmap.c

9789

pmap_bti_update_range(pmap, sva, eva, true);

sys/arm64/arm64/pmap.c

9790

PMAP_UNLOCK(pmap);

sys/arm64/arm64/trap.c

345

pmap_fault(map->pmap, esr, fault_va) == KERN_SUCCESS)

sys/arm64/include/bus_dma.h

122

bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,

sys/arm64/include/bus_dma.h

128

return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs,

sys/arm64/include/bus_dma_impl.h

68

void *buf, bus_size_t buflen, struct pmap *pmap, int flags,

sys/arm64/include/pcpu.h

49

struct pmap *pc_curpmap; \

sys/arm64/include/pcpu.h

50

struct pmap *pc_curvmpmap; \

sys/arm64/include/pmap.h

101

extern struct pmap kernel_pmap_store;

sys/arm64/include/pmap.h

107

#define PMAP_ASSERT_LOCKED(pmap) \

sys/arm64/include/pmap.h

108

mtx_assert(&(pmap)->pm_mtx, MA_OWNED)

sys/arm64/include/pmap.h

109

#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)

sys/arm64/include/pmap.h

110

#define PMAP_LOCK_ASSERT(pmap, type) \

sys/arm64/include/pmap.h

111

mtx_assert(&(pmap)->pm_mtx, (type))

sys/arm64/include/pmap.h

112

#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)

sys/arm64/include/pmap.h

113

#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \

sys/arm64/include/pmap.h

115

#define PMAP_OWNED(pmap) mtx_owned(&(pmap)->pm_mtx)

sys/arm64/include/pmap.h

116

#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)

sys/arm64/include/pmap.h

117

#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)

sys/arm64/include/pmap.h

118

#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)

sys/arm64/include/pmap.h

158

bool pmap_ps_enabled(pmap_t pmap);

sys/arm64/include/pmap.h

159

uint64_t pmap_to_ttbr0(pmap_t pmap);

sys/arm64/include/pmap.h

96

typedef struct pmap *pmap_t;

sys/arm64/include/vmm.h

149

struct pmap;

sys/arm64/include/vmm.h

153

DECLARE_VMMOPS_FUNC(void *, init, (struct vm *vm, struct pmap *pmap));

sys/arm64/include/vmm.h

156

DECLARE_VMMOPS_FUNC(int, run, (void *vcpui, register_t pc, struct pmap *pmap,

sys/arm64/iommu/iommu_pmap.c

104

smmu_pmap_l0(struct smmu_pmap *pmap, vm_offset_t va)

sys/arm64/iommu/iommu_pmap.c

107

return (&pmap->sp_l0[smmu_l0_index(va)]);

sys/arm64/iommu/iommu_pmap.c

120

smmu_pmap_l1(struct smmu_pmap *pmap, vm_offset_t va)

sys/arm64/iommu/iommu_pmap.c

124

l0 = smmu_pmap_l0(pmap, va);

sys/arm64/iommu/iommu_pmap.c

151

smmu_pmap_l2(struct smmu_pmap *pmap, vm_offset_t va)

sys/arm64/iommu/iommu_pmap.c

155

l1 = smmu_pmap_l1(pmap, va);

sys/arm64/iommu/iommu_pmap.c

187

smmu_pmap_pde(struct smmu_pmap *pmap, vm_offset_t va, int *level)

sys/arm64/iommu/iommu_pmap.c

191

l0 = smmu_pmap_l0(pmap, va);

sys/arm64/iommu/iommu_pmap.c

222

smmu_pmap_pte(struct smmu_pmap *pmap, vm_offset_t va, int *level)

sys/arm64/iommu/iommu_pmap.c

227

l1 = smmu_pmap_l1(pmap, va);

sys/arm64/iommu/iommu_pmap.c

274

smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count)

sys/arm64/iommu/iommu_pmap.c

277

SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/iommu/iommu_pmap.c

278

pmap->sp_resident_count += count;

sys/arm64/iommu/iommu_pmap.c

282

smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count)

sys/arm64/iommu/iommu_pmap.c

285

SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/iommu/iommu_pmap.c

286

KASSERT(pmap->sp_resident_count >= count,

sys/arm64/iommu/iommu_pmap.c

287

("pmap %p resident count underflow %ld %d", pmap,

sys/arm64/iommu/iommu_pmap.c

288

pmap->sp_resident_count, count));

sys/arm64/iommu/iommu_pmap.c

289

pmap->sp_resident_count -= count;

sys/arm64/iommu/iommu_pmap.c

293

smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count)

sys/arm64/iommu/iommu_pmap.c

298

smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count)

sys/arm64/iommu/iommu_pmap.c

334

smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m,

sys/arm64/iommu/iommu_pmap.c

340

_smmu_pmap_unwire_l3(pmap, va, m, free);

sys/arm64/iommu/iommu_pmap.c

347

_smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m,

sys/arm64/iommu/iommu_pmap.c

351

SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/iommu/iommu_pmap.c

359

l0 = smmu_pmap_l0(pmap, va);

sys/arm64/iommu/iommu_pmap.c

365

l1 = smmu_pmap_l1(pmap, va);

sys/arm64/iommu/iommu_pmap.c

371

l2 = smmu_pmap_l2(pmap, va);

sys/arm64/iommu/iommu_pmap.c

374

smmu_pmap_resident_count_dec(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

380

l1 = smmu_pmap_l1(pmap, va);

sys/arm64/iommu/iommu_pmap.c

383

smmu_pmap_unwire_l3(pmap, va, l2pg, free);

sys/arm64/iommu/iommu_pmap.c

389

l0 = smmu_pmap_l0(pmap, va);

sys/arm64/iommu/iommu_pmap.c

392

smmu_pmap_unwire_l3(pmap, va, l1pg, free);

sys/arm64/iommu/iommu_pmap.c

403

smmu_pmap_pinit(struct smmu_pmap *pmap)

sys/arm64/iommu/iommu_pmap.c

412

pmap->sp_l0_paddr = VM_PAGE_TO_PHYS(m);

sys/arm64/iommu/iommu_pmap.c

413

pmap->sp_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->sp_l0_paddr);

sys/arm64/iommu/iommu_pmap.c

416

pmap->sp_resident_count = 0;

sys/arm64/iommu/iommu_pmap.c

418

mtx_init(&pmap->sp_mtx, "smmu pmap", NULL, MTX_DEF);

sys/arm64/iommu/iommu_pmap.c

435

_pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex)

sys/arm64/iommu/iommu_pmap.c

439

SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/iommu/iommu_pmap.c

473

l0 = &pmap->sp_l0[l0index];

sys/arm64/iommu/iommu_pmap.c

483

l0 = &pmap->sp_l0[l0index];

sys/arm64/iommu/iommu_pmap.c

487

if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index)

sys/arm64/iommu/iommu_pmap.c

509

l0 = &pmap->sp_l0[l0index];

sys/arm64/iommu/iommu_pmap.c

513

if (_pmap_alloc_l3(pmap, NUL2E + l1index) == NULL) {

sys/arm64/iommu/iommu_pmap.c

527

if (_pmap_alloc_l3(pmap, NUL2E + l1index)

sys/arm64/iommu/iommu_pmap.c

544

smmu_pmap_resident_count_inc(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

559

smmu_pmap_release(struct smmu_pmap *pmap)

sys/arm64/iommu/iommu_pmap.c

563

KASSERT(pmap->sp_resident_count == 0,

sys/arm64/iommu/iommu_pmap.c

565

pmap->sp_resident_count));

sys/arm64/iommu/iommu_pmap.c

567

m = PHYS_TO_VM_PAGE(pmap->sp_l0_paddr);

sys/arm64/iommu/iommu_pmap.c

570

mtx_destroy(&pmap->sp_mtx);

sys/arm64/iommu/iommu_pmap.c

581

pmap_gpu_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,

sys/arm64/iommu/iommu_pmap.c

609

SMMU_PMAP_LOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

616

pde = smmu_pmap_pde(pmap, va, &lvl);

sys/arm64/iommu/iommu_pmap.c

620

mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va));

sys/arm64/iommu/iommu_pmap.c

63

#define SMMU_PMAP_LOCK(pmap) mtx_lock(&(pmap)->sp_mtx)

sys/arm64/iommu/iommu_pmap.c

633

l1p = smmu_pmap_l1(pmap, va);

sys/arm64/iommu/iommu_pmap.c

634

l2p = smmu_pmap_l2(pmap, va);

sys/arm64/iommu/iommu_pmap.c

64

#define SMMU_PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->sp_mtx)

sys/arm64/iommu/iommu_pmap.c

649

smmu_pmap_resident_count_inc(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

65

#define SMMU_PMAP_LOCK_ASSERT(pmap, type) \

sys/arm64/iommu/iommu_pmap.c

654

SMMU_PMAP_UNLOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

66

mtx_assert(&(pmap)->sp_mtx, (type))

sys/arm64/iommu/iommu_pmap.c

663

pmap_gpu_remove(struct smmu_pmap *pmap, vm_offset_t va)

sys/arm64/iommu/iommu_pmap.c

672

SMMU_PMAP_LOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

674

pde = smmu_pmap_pde(pmap, va, &lvl);

sys/arm64/iommu/iommu_pmap.c

682

smmu_pmap_resident_count_dec(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

688

SMMU_PMAP_UNLOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

697

smmu_pmap_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,

sys/arm64/iommu/iommu_pmap.c

721

SMMU_PMAP_LOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

728

pde = smmu_pmap_pde(pmap, va, &lvl);

sys/arm64/iommu/iommu_pmap.c

732

mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va));

sys/arm64/iommu/iommu_pmap.c

746

smmu_pmap_resident_count_inc(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

751

SMMU_PMAP_UNLOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

760

smmu_pmap_remove(struct smmu_pmap *pmap, vm_offset_t va)

sys/arm64/iommu/iommu_pmap.c

766

SMMU_PMAP_LOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

768

pte = smmu_pmap_pte(pmap, va, &lvl);

sys/arm64/iommu/iommu_pmap.c

773

smmu_pmap_resident_count_dec(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

779

SMMU_PMAP_UNLOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

790

smmu_pmap_remove_pages(struct smmu_pmap *pmap)

sys/arm64/iommu/iommu_pmap.c

800

SMMU_PMAP_LOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

803

l0e = pmap->sp_l0[i];

sys/arm64/iommu/iommu_pmap.c

843

smmu_pmap_resident_count_dec(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

849

smmu_pmap_resident_count_dec(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

854

smmu_pmap_resident_count_dec(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

856

smmu_pmap_clear(&pmap->sp_l0[i]);

sys/arm64/iommu/iommu_pmap.c

859

KASSERT(pmap->sp_resident_count == 0,

sys/arm64/iommu/iommu_pmap.c

86

static vm_page_t _pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex);

sys/arm64/iommu/iommu_pmap.c

860

("Invalid resident count %jd", pmap->sp_resident_count));

sys/arm64/iommu/iommu_pmap.c

862

SMMU_PMAP_UNLOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

87

static void _smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va,

sys/arm64/iommu/iommu_pmap.h

44

int smmu_pmap_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,

sys/arm64/iommu/iommu_pmap.h

46

int smmu_pmap_remove(struct smmu_pmap *pmap, vm_offset_t va);

sys/arm64/iommu/iommu_pmap.h

49

int pmap_gpu_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,

sys/arm64/iommu/iommu_pmap.h

51

int pmap_gpu_remove(struct smmu_pmap *pmap, vm_offset_t va);

sys/arm64/iommu/iommu_pmap.h

54

void smmu_pmap_remove_pages(struct smmu_pmap *pmap);

sys/arm64/iommu/iommu_pmap.h

55

void smmu_pmap_release(struct smmu_pmap *pmap);

sys/arm64/iommu/iommu_pmap.h

56

int smmu_pmap_pinit(struct smmu_pmap *pmap);

sys/arm64/vmm/vmm.c

1001

pmap_t pmap;

sys/arm64/vmm/vmm.c

1011

pmap = vmspace_pmap(vm_vmspace(vm));

sys/arm64/vmm/vmm.c

1022

error = vmmops_run(vcpu->cookie, vcpu->nextpc, pmap, &evinfo);

sys/arm64/vmm/vmm.c

353

pmap_t pmap __diagused;

sys/arm64/vmm/vmm.c

358

pmap = vmspace_pmap(vm_vmspace(vm));

sys/arm64/vmm/vmm.c

363

MPASS(cpuid_to_pcpu[i]->pc_curvmpmap != pmap);

sys/arm64/vmm/vmm.c

915

pmap_t pmap;

sys/arm64/vmm/vmm.c

920

pmap = vmspace_pmap(vm_vmspace(vcpu->vm));

sys/arm64/vmm/vmm.c

925

if (pmap_fault(pmap, esr, addr) == KERN_SUCCESS)

sys/arm64/vmm/vmm_arm64.c

1077

vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)

sys/arm64/vmm/vmm_arm64.c

1149

pmap_activate_vm(pmap);

sys/arm64/vmm/vmm_arm64.c

1150

hyp->vttbr_el2 = pmap_to_ttbr0(pmap);

sys/arm64/vmm/vmm_arm64.c

1183

pmap);

sys/arm64/vmm/vmm_arm64.c

510

vmmops_init(struct vm *vm, pmap_t pmap)

sys/arm64/vmm/vmm_arm64.c

583

arm_vmm_pinit(pmap_t pmap)

sys/arm64/vmm/vmm_arm64.c

586

pmap_pinit_stage(pmap, PM_STAGE2, vmm_pmap_levels);

sys/arm64/vmm/vmm_arm64.c

685

pmap_t pmap)

sys/arm64/vmm/vmm_arm64.c

793

struct vm_exit *vme, pmap_t pmap)

sys/arm64/vmm/vmm_arm64.c

800

handled = handle_el1_sync_excp(hypctx, vme, pmap);

sys/compat/linux/linux_mmap.c

332

pmap_remove(map->pmap, estart, eend);

sys/compat/linuxkpi/common/src/linux_page.c

277

*mp = pmap_extract_and_hold(map->pmap, va, prot);

sys/compat/x86bios/x86bios.c

151

if (x86bios_vmc.pmap[i].kva == (vm_offset_t)addr)

sys/compat/x86bios/x86bios.c

158

bzero(x86bios_vmc.pmap + i, sizeof(*x86bios_vmc.pmap) * nfree);

sys/compat/x86bios/x86bios.c

161

while (--i >= 0 && x86bios_vmc.pmap[i].kva == 0)

sys/ddb/db_ps.c

491

(p->p_vmspace != NULL) ? &p->p_vmspace->vm_map.pmap : 0);

sys/ddb/db_watch.c

231

pmap_protect(watch->map->pmap,

sys/dev/hptmv/entry.c

1372

PBUS_DMAMAP pmap = &(pAdapter->pbus_dmamap[i]);

sys/dev/hptmv/entry.c

1373

pmap->pAdapter = pAdapter;

sys/dev/hptmv/entry.c

1374

dmamap_put(pmap);

sys/dev/hptmv/entry.c

1376

if(bus_dmamap_create(pAdapter->io_dma_parent, 0, &pmap->dma_map)) {

sys/dev/hptmv/entry.c

1382

callout_init_mtx(&pmap->timeout, &pAdapter->lock, 0);

sys/dev/hptmv/entry.c

2200

PBUS_DMAMAP pmap = (PBUS_DMAMAP)ccb->ccb_adapter;

sys/dev/hptmv/entry.c

2201

IAL_ADAPTER_T * pAdapter = pmap->pAdapter;

sys/dev/hptmv/entry.c

2204

dmamap_put(pmap);

sys/dev/hptmv/entry.c

2228

PBUS_DMAMAP pmap;

sys/dev/hptmv/entry.c

2256

pmap = dmamap_get(pAdapter);

sys/dev/hptmv/entry.c

2257

HPT_ASSERT(pmap);

sys/dev/hptmv/entry.c

2258

ccb->ccb_adapter = pmap;

sys/dev/hptmv/entry.c

2259

memset((void *)pmap->psg, 0, sizeof(pmap->psg));

sys/dev/hptmv/entry.c

2598

PBUS_DMAMAP pmap = (PBUS_DMAMAP)((union ccb *)arg)->ccb_adapter;

sys/dev/hptmv/entry.c

2599

IAL_ADAPTER_T *pAdapter = pmap->pAdapter;

sys/dev/hptmv/entry.c

2612

PBUS_DMAMAP pmap = (PBUS_DMAMAP) ccb->ccb_adapter;

sys/dev/hptmv/entry.c

2613

IAL_ADAPTER_T *pAdapter = pmap->pAdapter;

sys/dev/hptmv/entry.c

2636

bus_dmamap_sync(pAdapter->io_dma_parent, pmap->dma_map,

sys/dev/hptmv/entry.c

2640

bus_dmamap_sync(pAdapter->io_dma_parent, pmap->dma_map,

sys/dev/hptmv/entry.c

2645

callout_reset(&pmap->timeout, 20 * hz, hpt_timeout, ccb);

sys/dev/hptmv/entry.c

2655

PBUS_DMAMAP pmap = (PBUS_DMAMAP)ccb->ccb_adapter;

sys/dev/hptmv/entry.c

2656

IAL_ADAPTER_T *pAdapter = pmap->pAdapter;

sys/dev/hptmv/entry.c

2777

pCmd->pSgTable = pmap->psg;

sys/dev/hptmv/entry.c

2831

pmap->dma_map,

sys/dev/hptmv/entry.c

2841

dmamap_put(pmap);

sys/dev/hptmv/entry.c

2864

PBUS_DMAMAP pmap = (PBUS_DMAMAP)ccb->ccb_adapter;

sys/dev/hptmv/entry.c

2865

IAL_ADAPTER_T *pAdapter = pmap->pAdapter;

sys/dev/hptmv/entry.c

2869

callout_stop(&pmap->timeout);

sys/dev/hptmv/entry.c

2896

bus_dmamap_sync(pAdapter->io_dma_parent, pmap->dma_map, BUS_DMASYNC_POSTREAD);

sys/dev/hptmv/entry.c

2899

bus_dmamap_sync(pAdapter->io_dma_parent, pmap->dma_map, BUS_DMASYNC_POSTWRITE);

sys/dev/hptmv/entry.c

2902

bus_dmamap_unload(pAdapter->io_dma_parent, pmap->dma_map);

sys/dev/hyperv/vmbus/hyperv_mmu.c

130

hv_vm_tlb_flush(pmap_t pmap, vm_offset_t addr1, vm_offset_t addr2,

sys/dev/hyperv/vmbus/hyperv_mmu.c

144

return smp_targeted_tlb_shootdown_native(pmap, addr1, addr2,

sys/dev/hyperv/vmbus/hyperv_mmu.c

149

return smp_targeted_tlb_shootdown_native(pmap, addr1, addr2,

sys/dev/hyperv/vmbus/hyperv_mmu.c

164

CPU_COPY(pmap_invalidate_cpu_mask(pmap), &tmp_mask);

sys/dev/hyperv/vmbus/hyperv_mmu.c

165

CPU_COPY(pmap_invalidate_cpu_mask(pmap), &mask);

sys/dev/hyperv/vmbus/hyperv_mmu.c

181

cr3 = pmap->pm_cr3;

sys/dev/hyperv/vmbus/hyperv_mmu.c

232

curcpu_cb(pmap, addr1, addr2);

sys/dev/hyperv/vmbus/hyperv_mmu.c

237

status = hv_flush_tlb_others_ex(pmap, addr1, addr2, mask, op, sc);

sys/dev/hyperv/vmbus/hyperv_mmu.c

245

return smp_targeted_tlb_shootdown_native(pmap, addr1,

sys/dev/hyperv/vmbus/hyperv_mmu.c

250

hv_flush_tlb_others_ex(pmap_t pmap, vm_offset_t addr1, vm_offset_t addr2,

sys/dev/hyperv/vmbus/hyperv_mmu.c

264

cr3 = pmap->pm_cr3;

sys/dev/hyperv/vmbus/vmbus.c

807

hyperv_vm_tlb_flush(pmap_t pmap, vm_offset_t addr1, vm_offset_t addr2,

sys/dev/hyperv/vmbus/vmbus.c

811

return hv_vm_tlb_flush(pmap, addr1, addr2, op, sc, curcpu_cb);

sys/dev/ice/ice_flex_pipe.c

2894

struct ice_prof_map *pmap;

sys/dev/ice/ice_flex_pipe.c

2899

pmap = ice_search_prof_id(hw, blk, id);

sys/dev/ice/ice_flex_pipe.c

2900

if (!pmap) {

sys/dev/ice/ice_flex_pipe.c

2906

status = ice_rem_flow_all(hw, blk, pmap->profile_cookie);

sys/dev/ice/ice_flex_pipe.c

2911

ice_prof_dec_ref(hw, blk, pmap->prof_id);

sys/dev/ice/ice_flex_pipe.c

2913

LIST_DEL(&pmap->list);

sys/dev/ice/ice_flex_pipe.c

2914

ice_free(hw, pmap);

sys/dev/iommu/busdma_iommu.c

779

bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,

sys/dev/iommu/busdma_iommu.c

800

if (pmap == kernel_pmap)

sys/dev/iommu/busdma_iommu.c

803

paddr = pmap_extract(pmap, pstart);

sys/dev/pci/pci_user.c

968

!pmap_is_valid_memattr(map->pmap, pbm->pbm_memattr))

sys/dev/proto/proto_busdma.c

282

pmap_t pmap;

sys/dev/proto/proto_busdma.c

304

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/dev/proto/proto_busdma.c

305

md->physaddr = pmap_extract(pmap, ioc->u.md.virt_addr);

sys/dev/qlnx/qlnxe/ecore_rdma.c

451

u64 *pmap;

sys/dev/qlnx/qlnxe/ecore_rdma.c

467

pmap = (u64 *)bmap->bitmap;

sys/dev/qlnx/qlnxe/ecore_rdma.c

473

if (OSAL_BITMAP_WEIGHT((unsigned long *)&pmap[item], 64*8))

sys/dev/qlnx/qlnxe/ecore_rdma.c

476

line, (unsigned long long)pmap[item],

sys/dev/qlnx/qlnxe/ecore_rdma.c

477

(unsigned long long)pmap[item+1],

sys/dev/qlnx/qlnxe/ecore_rdma.c

478

(unsigned long long)pmap[item+2],

sys/dev/qlnx/qlnxe/ecore_rdma.c

479

(unsigned long long)pmap[item+3],

sys/dev/qlnx/qlnxe/ecore_rdma.c

480

(unsigned long long)pmap[item+4],

sys/dev/qlnx/qlnxe/ecore_rdma.c

481

(unsigned long long)pmap[item+5],

sys/dev/qlnx/qlnxe/ecore_rdma.c

482

(unsigned long long)pmap[item+6],

sys/dev/qlnx/qlnxe/ecore_rdma.c

483

(unsigned long long)pmap[item+7]);

sys/dev/qlnx/qlnxe/ecore_rdma.c

488

(OSAL_BITMAP_WEIGHT((unsigned long *)&pmap[item],

sys/dev/qlnx/qlnxe/ecore_rdma.c

497

(unsigned long long)pmap[item]);

sys/dev/sound/pci/envy24.c

165

bus_dmamap_t pmap, rmap;

sys/dev/sound/pci/envy24.c

2167

printf("%p -> %lx\n", sc->pmap, sc->paddr);

sys/dev/sound/pci/envy24.c

2205

bus_dmamap_unload(sc->dmat, sc->pmap);

sys/dev/sound/pci/envy24.c

2209

bus_dmamem_free(sc->dmat, sc->pbuf, sc->pmap);

sys/dev/sound/pci/envy24.c

2212

bus_dmamap_unload(sc->dmat, sc->pmap);

sys/dev/sound/pci/envy24.c

2214

bus_dmamem_free(sc->dmat, sc->pbuf, sc->pmap);

sys/dev/sound/pci/envy24.c

2243

if (bus_dmamem_alloc(sc->dmat, (void **)&sc->pbuf, BUS_DMA_NOWAIT, &sc->pmap))

sys/dev/sound/pci/envy24.c

2253

if (bus_dmamap_load(sc->dmat, sc->pmap, sc->pbuf, sc->psize,

sys/dev/sound/pci/envy24ht.c

164

bus_dmamap_t pmap, rmap;

sys/dev/sound/pci/envy24ht.c

2121

bus_dmamap_unload(sc->dmat, sc->pmap);

sys/dev/sound/pci/envy24ht.c

2125

bus_dmamem_free(sc->dmat, sc->pbuf, sc->pmap);

sys/dev/sound/pci/envy24ht.c

2128

bus_dmamap_unload(sc->dmat, sc->pmap);

sys/dev/sound/pci/envy24ht.c

2130

bus_dmamem_free(sc->dmat, sc->pbuf, sc->pmap);

sys/dev/sound/pci/envy24ht.c

2159

if (bus_dmamem_alloc(sc->dmat, (void **)&sc->pbuf, BUS_DMA_NOWAIT, &sc->pmap))

sys/dev/sound/pci/envy24ht.c

2169

if (bus_dmamap_load(sc->dmat, sc->pmap, sc->pbuf, sc->psize, envy24ht_dmapsetmap, sc, BUS_DMA_NOWAIT))

sys/dev/sound/pci/hdsp.c

192

&sc->pmap)) {

sys/dev/sound/pci/hdsp.c

197

if (bus_dmamap_load(sc->dmat, sc->pmap, sc->pbuf, sc->bufsize,

sys/dev/sound/pci/hdsp.c

970

bus_dmamap_unload(sc->dmat, sc->pmap);

sys/dev/sound/pci/hdsp.c

972

bus_dmamem_free(sc->dmat, sc->pbuf, sc->pmap);

sys/dev/sound/pci/hdsp.h

246

bus_dmamap_t pmap;

sys/dev/sound/pci/hdspe.c

202

&sc->pmap)) {

sys/dev/sound/pci/hdspe.c

207

if (bus_dmamap_load(sc->dmat, sc->pmap, sc->pbuf, sc->bufsize,

sys/dev/sound/pci/hdspe.c

862

bus_dmamap_unload(sc->dmat, sc->pmap);

sys/dev/sound/pci/hdspe.c

864

bus_dmamem_free(sc->dmat, sc->pbuf, sc->pmap);

sys/dev/sound/pci/hdspe.h

238

bus_dmamap_t pmap;

sys/dev/xdma/xdma.h

131

struct pmap p;

sys/fs/tmpfs/tmpfs_vfsops.c

180

pmap_protect(map->pmap, entry->start, entry->end,

sys/fs/udf/udf_vfsops.c

763

uint8_t *pmap = (uint8_t *) &lvd->maps[0];

sys/fs/udf/udf_vfsops.c

766

ptype = pmap[0];

sys/fs/udf/udf_vfsops.c

767

psize = pmap[1];

sys/fs/udf/udf_vfsops.c

777

pmap += UDF_PMAP_TYPE1_SIZE;

sys/fs/udf/udf_vfsops.c

782

pmap_id = (struct regid *)&pmap[4];

sys/fs/udf/udf_vfsops.c

792

pms = (struct part_map_spare *)pmap;

sys/fs/udf/udf_vfsops.c

793

pmap += UDF_PMAP_TYPE2_SIZE;

sys/i386/i386/genassym.c

81

ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));

sys/i386/i386/mp_machdep.c

547

smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap,

sys/i386/i386/mp_machdep.c

560

curcpu_cb(pmap, addr1, addr2);

sys/i386/i386/mp_machdep.c

583

smp_tlb_pmap = pmap;

sys/i386/i386/mp_machdep.c

593

curcpu_cb(pmap, addr1, addr2);

sys/i386/i386/mp_machdep.c

604

curcpu_cb(pmap, addr1, addr2);

sys/i386/i386/mp_machdep.c

609

smp_masked_invltlb(cpuset_t mask, pmap_t pmap, smp_invl_cb_t curcpu_cb)

sys/i386/i386/mp_machdep.c

611

smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0, curcpu_cb);

sys/i386/i386/mp_machdep.c

618

smp_masked_invlpg(cpuset_t mask, vm_offset_t addr, pmap_t pmap,

sys/i386/i386/mp_machdep.c

621

smp_targeted_tlb_shootdown(mask, IPI_INVLPG, pmap, addr, 0, curcpu_cb);

sys/i386/i386/mp_machdep.c

629

pmap_t pmap, smp_invl_cb_t curcpu_cb)

sys/i386/i386/mp_machdep.c

631

smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, pmap, addr1, addr2,

sys/i386/i386/pmap.c

1077

__CONCAT(PMTYPE, is_valid_memattr)(pmap_t pmap __unused, vm_memattr_t mode)

sys/i386/i386/pmap.c

1089

__CONCAT(PMTYPE, cache_bits)(pmap_t pmap, int mode, bool is_pde)

sys/i386/i386/pmap.c

1093

if (!pmap_is_valid_memattr(pmap, mode))

sys/i386/i386/pmap.c

1114

pmap_pat_index(pmap_t pmap, pt_entry_t pte, bool is_pde)

sys/i386/i386/pmap.c

1146

__CONCAT(PMTYPE, ps_enabled)(pmap_t pmap __unused)

sys/i386/i386/pmap.c

1189

pmap_curcpu_cb_dummy(pmap_t pmap __unused, vm_offset_t addr1 __unused,

sys/i386/i386/pmap.c

1214

pmap_invalidate_page_int(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1220

if (pmap == kernel_pmap) {

sys/i386/i386/pmap.c

1223

} else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {

sys/i386/i386/pmap.c

1229

CPU_AND(&other_cpus, &other_cpus, &pmap->pm_active);

sys/i386/i386/pmap.c

1232

smp_masked_invlpg(*mask, va, pmap, pmap_curcpu_cb_dummy);

sys/i386/i386/pmap.c

1240

pmap_invalidate_range_int(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap.c

1247

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

1252

if (pmap == kernel_pmap) {

sys/i386/i386/pmap.c

1256

} else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {

sys/i386/i386/pmap.c

1262

CPU_AND(&other_cpus, &other_cpus, &pmap->pm_active);

sys/i386/i386/pmap.c

1265

smp_masked_invlpg_range(*mask, sva, eva, pmap, pmap_curcpu_cb_dummy);

sys/i386/i386/pmap.c

1270

pmap_invalidate_all_int(pmap_t pmap)

sys/i386/i386/pmap.c

1276

if (pmap == kernel_pmap) {

sys/i386/i386/pmap.c

1279

} else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {

sys/i386/i386/pmap.c

1285

CPU_AND(&other_cpus, &other_cpus, &pmap->pm_active);

sys/i386/i386/pmap.c

1288

smp_masked_invltlb(*mask, pmap, pmap_curcpu_cb_dummy);

sys/i386/i386/pmap.c

1293

pmap_invalidate_cache_curcpu_cb(pmap_t pmap __unused,

sys/i386/i386/pmap.c

1352

pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)

sys/i386/i386/pmap.c

1362

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

1365

active = pmap->pm_active;

sys/i386/i386/pmap.c

1374

smp_no_rendezvous_barrier, pmap == kernel_pmap ?

sys/i386/i386/pmap.c

1378

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

1393

pmap_invalidate_page_int(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1396

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

1401

pmap_invalidate_range_int(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap.c

1405

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

1411

pmap_invalidate_all_int(pmap_t pmap)

sys/i386/i386/pmap.c

1414

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

1426

pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)

sys/i386/i386/pmap.c

1429

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

1433

if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))

sys/i386/i386/pmap.c

1439

__CONCAT(PMTYPE, invalidate_page)(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1442

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

1446

__CONCAT(PMTYPE, invalidate_range)(pmap_t pmap, vm_offset_t sva,

sys/i386/i386/pmap.c

1450

pmap_invalidate_range_int(pmap, sva, eva);

sys/i386/i386/pmap.c

1454

__CONCAT(PMTYPE, invalidate_all)(pmap_t pmap)

sys/i386/i386/pmap.c

1457

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

1461

pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde)

sys/i386/i386/pmap.c

1476

pmap_invalidate_range_int(pmap, va, va + NBPDR - 1);

sys/i386/i386/pmap.c

1478

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

1485

pmap_is_current(pmap_t pmap)

sys/i386/i386/pmap.c

1488

return (pmap == kernel_pmap);

sys/i386/i386/pmap.c

1496

__CONCAT(PMTYPE, pte)(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1501

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

1506

if (pmap_is_current(pmap))

sys/i386/i386/pmap.c

1557

pmap_pte_quick(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1562

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

1567

if (pmap_is_current(pmap))

sys/i386/i386/pmap.c

1594

pmap_pte_quick3(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1599

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

1628

pmap_pte_ufast(pmap_t pmap, vm_offset_t va, pd_entry_t pde)

sys/i386/i386/pmap.c

1632

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

1681

__CONCAT(PMTYPE, extract)(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1688

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

1689

pde = pmap->pm_pdir[va >> PDRSHIFT];

sys/i386/i386/pmap.c

1694

pte = pmap_pte_ufast(pmap, va, pde);

sys/i386/i386/pmap.c

1698

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

1710

__CONCAT(PMTYPE, extract_and_hold)(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/i386/i386/pmap.c

1717

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

1718

pde = *pmap_pde(pmap, va);

sys/i386/i386/pmap.c

1725

pte = pmap_pte_ufast(pmap, va, pde);

sys/i386/i386/pmap.c

1733

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

1926

pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/i386/i386/pmap.c

1930

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

1934

return (vm_radix_insert(&pmap->pm_root, mpte));

sys/i386/i386/pmap.c

1944

pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1947

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

1948

return (vm_radix_remove(&pmap->pm_root, va >> PDRSHIFT));

sys/i386/i386/pmap.c

1958

pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free)

sys/i386/i386/pmap.c

1963

_pmap_unwire_ptp(pmap, m, free);

sys/i386/i386/pmap.c

1970

_pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free)

sys/i386/i386/pmap.c

1976

pmap->pm_pdir[m->pindex] = 0;

sys/i386/i386/pmap.c

1977

--pmap->pm_stats.resident_count;

sys/i386/i386/pmap.c

1986

MPASS(pmap != kernel_pmap);

sys/i386/i386/pmap.c

1995

pmap_unuse_pt(pmap_t pmap, vm_offset_t va, struct spglist *free)

sys/i386/i386/pmap.c

2000

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

2002

ptepde = *pmap_pde(pmap, va);

sys/i386/i386/pmap.c

2004

return (pmap_unwire_ptp(pmap, mpte, free));

sys/i386/i386/pmap.c

2012

pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte)

sys/i386/i386/pmap.c

2017

if (pmap_unwire_ptp(pmap, mpte, &free)) {

sys/i386/i386/pmap.c

2023

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

2032

__CONCAT(PMTYPE, pinit0)(pmap_t pmap)

sys/i386/i386/pmap.c

2035

PMAP_LOCK_INIT(pmap);

sys/i386/i386/pmap.c

2036

pmap->pm_pdir = IdlePTD;

sys/i386/i386/pmap.c

2038

pmap->pm_pdpt = IdlePDPT;

sys/i386/i386/pmap.c

2040

vm_radix_init(&pmap->pm_root);

sys/i386/i386/pmap.c

2041

CPU_ZERO(&pmap->pm_active);

sys/i386/i386/pmap.c

2042

TAILQ_INIT(&pmap->pm_pvchunk);

sys/i386/i386/pmap.c

2043

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/i386/i386/pmap.c

2044

pmap_activate_boot(pmap);

sys/i386/i386/pmap.c

2052

__CONCAT(PMTYPE, pinit)(pmap_t pmap)

sys/i386/i386/pmap.c

2060

if (pmap->pm_pdir == NULL) {

sys/i386/i386/pmap.c

2061

pmap->pm_pdir = (pd_entry_t *)kva_alloc(NBPTD);

sys/i386/i386/pmap.c

2062

if (pmap->pm_pdir == NULL)

sys/i386/i386/pmap.c

2065

pmap->pm_pdpt = uma_zalloc(pdptzone, M_WAITOK | M_ZERO);

sys/i386/i386/pmap.c

2066

KASSERT(((vm_offset_t)pmap->pm_pdpt &

sys/i386/i386/pmap.c

2069

KASSERT(pmap_kextract((vm_offset_t)pmap->pm_pdpt) < (4ULL<<30),

sys/i386/i386/pmap.c

2072

vm_radix_init(&pmap->pm_root);

sys/i386/i386/pmap.c

2074

KASSERT(vm_radix_is_empty(&pmap->pm_root),

sys/i386/i386/pmap.c

2081

pmap->pm_ptdpg[i] = vm_page_alloc_noobj(VM_ALLOC_WIRED |

sys/i386/i386/pmap.c

2084

pmap->pm_pdpt[i] = VM_PAGE_TO_PHYS(pmap->pm_ptdpg[i]) | PG_V;

sys/i386/i386/pmap.c

2088

pmap_qenter((vm_offset_t)pmap->pm_pdir, pmap->pm_ptdpg, NPGPTD);

sys/i386/i386/pmap.c

2092

trunc_page((vm_offset_t)pmap->pm_pdpt),

sys/i386/i386/pmap.c

2093

round_page((vm_offset_t)pmap->pm_pdpt +

sys/i386/i386/pmap.c

2099

pmap->pm_pdir[TRPTDI] = PTD[TRPTDI];

sys/i386/i386/pmap.c

2101

CPU_ZERO(&pmap->pm_active);

sys/i386/i386/pmap.c

2102

TAILQ_INIT(&pmap->pm_pvchunk);

sys/i386/i386/pmap.c

2103

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/i386/i386/pmap.c

2113

_pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags)

sys/i386/i386/pmap.c

2123

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

2127

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

2143

pmap->pm_stats.resident_count++;

sys/i386/i386/pmap.c

2146

KASSERT((pmap->pm_pdir[ptepindex] & PG_V) == 0,

sys/i386/i386/pmap.c

2148

__func__, (uintmax_t)pmap->pm_pdir[ptepindex]));

sys/i386/i386/pmap.c

2149

pmap->pm_pdir[ptepindex] =

sys/i386/i386/pmap.c

2156

pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags)

sys/i386/i386/pmap.c

2170

ptepa = pmap->pm_pdir[ptepindex];

sys/i386/i386/pmap.c

2177

(void)pmap_demote_pde(pmap, &pmap->pm_pdir[ptepindex], va);

sys/i386/i386/pmap.c

2178

ptepa = pmap->pm_pdir[ptepindex];

sys/i386/i386/pmap.c

2193

m = _pmap_allocpte(pmap, ptepindex, flags);

sys/i386/i386/pmap.c

2210

__CONCAT(PMTYPE, release)(pmap_t pmap)

sys/i386/i386/pmap.c

2215

KASSERT(pmap->pm_stats.resident_count == 0,

sys/i386/i386/pmap.c

2217

pmap->pm_stats.resident_count));

sys/i386/i386/pmap.c

2218

KASSERT(vm_radix_is_empty(&pmap->pm_root),

sys/i386/i386/pmap.c

2220

KASSERT(CPU_EMPTY(&pmap->pm_active),

sys/i386/i386/pmap.c

2221

("releasing active pmap %p", pmap));

sys/i386/i386/pmap.c

2223

pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD);

sys/i386/i386/pmap.c

2226

m = pmap->pm_ptdpg[i];

sys/i386/i386/pmap.c

2228

KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME),

sys/i386/i386/pmap.c

2309

pmap_t pmap;

sys/i386/i386/pmap.c

2319

pmap = NULL;

sys/i386/i386/pmap.c

2326

if (pmap != pc->pc_pmap) {

sys/i386/i386/pmap.c

2327

if (pmap != NULL) {

sys/i386/i386/pmap.c

2328

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

2329

if (pmap != locked_pmap)

sys/i386/i386/pmap.c

2330

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

2332

pmap = pc->pc_pmap;

sys/i386/i386/pmap.c

2334

if (pmap > locked_pmap)

sys/i386/i386/pmap.c

2335

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

2336

else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {

sys/i386/i386/pmap.c

2337

pmap = NULL;

sys/i386/i386/pmap.c

2353

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

2356

pte = __CONCAT(PMTYPE, pte)(pmap, va);

sys/i386/i386/pmap.c

2365

pmap, va));

sys/i386/i386/pmap.c

2367

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

2383

pmap_unuse_pt(pmap, va, &free);

sys/i386/i386/pmap.c

2392

pmap->pm_stats.resident_count -= freed;

sys/i386/i386/pmap.c

2396

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

2399

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,

sys/i386/i386/pmap.c

2407

if (pmap == locked_pmap)

sys/i386/i386/pmap.c

2424

if (pmap != NULL) {

sys/i386/i386/pmap.c

2425

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

2426

if (pmap != locked_pmap)

sys/i386/i386/pmap.c

2427

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

2443

free_pv_entry(pmap_t pmap, pv_entry_t pv)

sys/i386/i386/pmap.c

2449

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2464

if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=

sys/i386/i386/pmap.c

2466

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

2467

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,

sys/i386/i386/pmap.c

2472

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

2498

get_pv_entry(pmap_t pmap, bool try)

sys/i386/i386/pmap.c

2508

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2517

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/i386/i386/pmap.c

2534

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

2535

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

2552

m = pmap_pv_reclaim(pmap);

sys/i386/i386/pmap.c

2560

pc->pc_pmap = pmap;

sys/i386/i386/pmap.c

2566

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

2572

pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

2578

if (pmap == PV_PMAP(pv) && va == pv->pv_va) {

sys/i386/i386/pmap.c

2587

pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)

sys/i386/i386/pmap.c

2604

pv = pmap_pvh_remove(pvh, pmap, va);

sys/i386/i386/pmap.c

2615

pmap_insert_entry(pmap, va, m);

sys/i386/i386/pmap.c

2621

pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)

sys/i386/i386/pmap.c

2641

pv = pmap_pvh_remove(&m->md, pmap, va);

sys/i386/i386/pmap.c

2650

pmap_pvh_free(&m->md, pmap, va);

sys/i386/i386/pmap.c

2656

pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

2660

pv = pmap_pvh_remove(pvh, pmap, va);

sys/i386/i386/pmap.c

2662

free_pv_entry(pmap, pv);

sys/i386/i386/pmap.c

2666

pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)

sys/i386/i386/pmap.c

2671

pmap_pvh_free(&m->md, pmap, va);

sys/i386/i386/pmap.c

2684

pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/i386/i386/pmap.c

2689

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2690

pv = get_pv_entry(pmap, false);

sys/i386/i386/pmap.c

2699

pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/i386/i386/pmap.c

2704

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2706

(pv = get_pv_entry(pmap, true)) != NULL) {

sys/i386/i386/pmap.c

2718

pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, u_int flags)

sys/i386/i386/pmap.c

2727

(pv = get_pv_entry(pmap, noreclaim)) == NULL)

sys/i386/i386/pmap.c

2754

pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)

sys/i386/i386/pmap.c

2763

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2767

if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==

sys/i386/i386/pmap.c

2782

pmap_remove_pde(pmap, pde, sva, &free);

sys/i386/i386/pmap.c

2784

pmap_invalidate_pde_page(pmap, sva, oldpde);

sys/i386/i386/pmap.c

2787

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

2791

if (pmap != kernel_pmap) {

sys/i386/i386/pmap.c

2793

pmap->pm_stats.resident_count++;

sys/i386/i386/pmap.c

2804

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

2868

pmap_update_pde(pmap, va, pde, newpde);

sys/i386/i386/pmap.c

2869

else if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

287

static void free_pv_entry(pmap_t pmap, pv_entry_t pv);

sys/i386/i386/pmap.c

2879

pmap_invalidate_page_int(pmap, (vm_offset_t)vtopte(va));

sys/i386/i386/pmap.c

288

static pv_entry_t get_pv_entry(pmap_t pmap, bool try);

sys/i386/i386/pmap.c

289

static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);

sys/i386/i386/pmap.c

2891

pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME);

sys/i386/i386/pmap.c

2895

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

290

static bool pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde,

sys/i386/i386/pmap.c

2903

pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)

sys/i386/i386/pmap.c

2909

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2910

mpte = pmap_remove_pt_page(pmap, va);

sys/i386/i386/pmap.c

2928

pmap_update_pde(pmap, va, pde, newpde);

sys/i386/i386/pmap.c

293

static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);

sys/i386/i386/pmap.c

2935

pmap_invalidate_page_int(pmap, (vm_offset_t)vtopte(va));

sys/i386/i386/pmap.c

2942

pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,

sys/i386/i386/pmap.c

295

static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);

sys/i386/i386/pmap.c

2950

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2955

pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;

sys/i386/i386/pmap.c

296

static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,

sys/i386/i386/pmap.c

2964

pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;

sys/i386/i386/pmap.c

2967

pmap_pvh_free(pvh, pmap, sva);

sys/i386/i386/pmap.c

2980

if (pmap == kernel_pmap) {

sys/i386/i386/pmap.c

2981

pmap_remove_kernel_pde(pmap, pdq, sva);

sys/i386/i386/pmap.c

2983

mpte = pmap_remove_pt_page(pmap, sva);

sys/i386/i386/pmap.c

2987

pmap->pm_stats.resident_count--;

sys/i386/i386/pmap.c

300

static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);

sys/i386/i386/pmap.c

3000

pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,

sys/i386/i386/pmap.c

3007

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

301

static bool pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);

sys/i386/i386/pmap.c

3010

("pmap_remove_pte: pmap %p va %x zero pte", pmap, va));

sys/i386/i386/pmap.c

3012

pmap->pm_stats.wired_count -= 1;

sys/i386/i386/pmap.c

3019

pmap->pm_stats.resident_count -= 1;

sys/i386/i386/pmap.c

302

static int pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/i386/i386/pmap.c

3026

pmap_remove_entry(pmap, m, va);

sys/i386/i386/pmap.c

3028

return (pmap_unuse_pt(pmap, va, free));

sys/i386/i386/pmap.c

3035

pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free)

sys/i386/i386/pmap.c

304

static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,

sys/i386/i386/pmap.c

3041

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

3042

if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0)

sys/i386/i386/pmap.c

3044

pmap_remove_pte(pmap, pte, va, free);

sys/i386/i386/pmap.c

3045

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

3052

pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/i386/i386/pmap.c

306

static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,

sys/i386/i386/pmap.c

3060

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

3062

for (pte = pmap_pte_quick(pmap, sva); sva != eva; pte++,

sys/i386/i386/pmap.c

3074

if (pmap_remove_pte(pmap, pte, sva, free))

sys/i386/i386/pmap.c

308

static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/i386/i386/pmap.c

3087

__CONCAT(PMTYPE, remove)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap.c

3097

if (pmap->pm_stats.resident_count == 0)

sys/i386/i386/pmap.c

310

static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,

sys/i386/i386/pmap.c

3105

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

3113

((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) {

sys/i386/i386/pmap.c

3114

pmap_remove_page(pmap, sva, &free);

sys/i386/i386/pmap.c

3127

if (pmap->pm_stats.resident_count == 0)

sys/i386/i386/pmap.c

3131

ptpaddr = pmap->pm_pdir[pdirindex];

sys/i386/i386/pmap.c

3155

pmap_remove_pde(pmap,

sys/i386/i386/pmap.c

3156

&pmap->pm_pdir[pdirindex], sva, &free);

sys/i386/i386/pmap.c

3158

} else if (!pmap_demote_pde(pmap,

sys/i386/i386/pmap.c

3159

&pmap->pm_pdir[pdirindex], sva)) {

sys/i386/i386/pmap.c

3173

if (pmap_remove_ptes(pmap, sva, pdnxt, &free))

sys/i386/i386/pmap.c

3179

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

3181

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

319

static bool pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,

sys/i386/i386/pmap.c

3203

pmap_t pmap;

sys/i386/i386/pmap.c

3219

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

322

static bool pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,

sys/i386/i386/pmap.c

3220

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

3221

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

3222

(void)pmap_demote_pde(pmap, pde, va);

sys/i386/i386/pmap.c

3223

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

3227

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

3228

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

3229

pmap->pm_stats.resident_count--;

sys/i386/i386/pmap.c

3230

pde = pmap_pde(pmap, pv->pv_va);

sys/i386/i386/pmap.c

3233

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

3236

pmap, pv->pv_va));

sys/i386/i386/pmap.c

3238

pmap->pm_stats.wired_count--;

sys/i386/i386/pmap.c

3247

pmap_unuse_pt(pmap, pv->pv_va, &free);

sys/i386/i386/pmap.c

3248

pmap_invalidate_page_int(pmap, pv->pv_va);

sys/i386/i386/pmap.c

325

static void pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,

sys/i386/i386/pmap.c

3250

free_pv_entry(pmap, pv);

sys/i386/i386/pmap.c

3251

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

3263

pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)

sys/i386/i386/pmap.c

3269

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

327

static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,

sys/i386/i386/pmap.c

329

static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);

sys/i386/i386/pmap.c

330

static void pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free);

sys/i386/i386/pmap.c

3309

__CONCAT(PMTYPE, protect)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/i386/i386/pmap.c

331

static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/i386/i386/pmap.c

3319

pmap_remove(pmap, sva, eva);

sys/i386/i386/pmap.c

333

static void pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va);

sys/i386/i386/pmap.c

3332

if (pmap_is_current(pmap))

sys/i386/i386/pmap.c

334

static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);

sys/i386/i386/pmap.c

3342

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

335

static bool pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,

sys/i386/i386/pmap.c

3352

ptpaddr = pmap->pm_pdir[pdirindex];

sys/i386/i386/pmap.c

337

static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,

sys/i386/i386/pmap.c

3374

if (pmap_protect_pde(pmap,

sys/i386/i386/pmap.c

3375

&pmap->pm_pdir[pdirindex], sva, prot))

sys/i386/i386/pmap.c

3384

pmap);

sys/i386/i386/pmap.c

3385

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

3390

if (!pmap_demote_pde(pmap,

sys/i386/i386/pmap.c

3391

&pmap->pm_pdir[pdirindex], sva)) {

sys/i386/i386/pmap.c

3404

for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,

sys/i386/i386/pmap.c

341

static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags);

sys/i386/i386/pmap.c

343

static vm_page_t _pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags);

sys/i386/i386/pmap.c

344

static void _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free);

sys/i386/i386/pmap.c

3441

pmap_invalidate_page_int(pmap, sva);

sys/i386/i386/pmap.c

3448

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

345

static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va);

sys/i386/i386/pmap.c

3453

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

3470

pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, vm_page_t mpte)

sys/i386/i386/pmap.c

3478

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

3487

firstpte = pmap_pte_quick(pmap, trunc_4mpage(va));

sys/i386/i386/pmap.c

3493

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

3496

if ((*firstpte & PG_MANAGED) != 0 && pmap == kernel_pmap) {

sys/i386/i386/pmap.c

3499

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

3529

" in pmap %p", va & ~PDRMASK, pmap);

sys/i386/i386/pmap.c

353

static void pmap_invalidate_all_int(pmap_t pmap);

sys/i386/i386/pmap.c

3545

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

3562

" in pmap %p", oldpteva, pmap);

sys/i386/i386/pmap.c

3567

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

3595

if (pmap_insert_pt_page(pmap, mpte, true, allpte_PG_A != 0)) {

sys/i386/i386/pmap.c

3599

pmap);

sys/i386/i386/pmap.c

3607

pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME);

sys/i386/i386/pmap.c

3619

pmap_update_pde(pmap, va, pde, PG_PS | newpde);

sys/i386/i386/pmap.c

3620

else if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

3627

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

3645

__CONCAT(PMTYPE, enter)(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/i386/i386/pmap.c

3657

KASSERT((pmap == kernel_pmap && va < VM_MAX_KERNEL_ADDRESS) ||

sys/i386/i386/pmap.c

3658

(pmap != kernel_pmap && va < VM_MAXUSER_ADDRESS),

sys/i386/i386/pmap.c

3659

("pmap_enter: toobig k%d %#x", pmap == kernel_pmap, va));

sys/i386/i386/pmap.c

3663

KASSERT(pmap != kernel_pmap || (m->oflags & VPO_UNMANAGED) != 0 ||

sys/i386/i386/pmap.c

3684

if (pmap != kernel_pmap)

sys/i386/i386/pmap.c

3686

newpte |= pmap_cache_bits(pmap, m->md.pat_mode, psind > 0);

sys/i386/i386/pmap.c

3691

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

3697

rv = pmap_enter_pde(pmap, va, newpte | PG_PS, flags, m);

sys/i386/i386/pmap.c

3701

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

3702

if (pmap != kernel_pmap) {

sys/i386/i386/pmap.c

3709

mpte = pmap_allocpte(pmap, va, flags);

sys/i386/i386/pmap.c

3725

(uintmax_t)pmap->pm_pdir[PTDPTDI]));

sys/i386/i386/pmap.c

3727

pmap_demote_pde(pmap, pde, va);

sys/i386/i386/pmap.c

3729

pte = pmap_pte_quick(pmap, va);

sys/i386/i386/pmap.c

3738

(uintmax_t)pmap->pm_pdir[PTDPTDI], va);

sys/i386/i386/pmap.c

3755

pmap->pm_stats.wired_count++;

sys/i386/i386/pmap.c

3757

pmap->pm_stats.wired_count--;

sys/i386/i386/pmap.c

3809

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

3812

pv = pmap_pvh_remove(&om->md, pmap, va);

sys/i386/i386/pmap.c

3816

free_pv_entry(pmap, pv);

sys/i386/i386/pmap.c

3827

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

3835

pmap->pm_stats.wired_count++;

sys/i386/i386/pmap.c

3836

pmap->pm_stats.resident_count++;

sys/i386/i386/pmap.c

3844

pv = get_pv_entry(pmap, false);

sys/i386/i386/pmap.c

3880

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

3894

(void)pmap_promote_pde(pmap, pde, va, mpte);

sys/i386/i386/pmap.c

3901

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

3912

pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/i386/i386/pmap.c

3916

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

3918

pmap_cache_bits(pmap, m->md.pat_mode, true) | PG_PS | PG_V;

sys/i386/i386/pmap.c

3925

if (pmap != kernel_pmap)

sys/i386/i386/pmap.c

3927

return (pmap_enter_pde(pmap, va, newpde, PMAP_ENTER_NOSLEEP |

sys/i386/i386/pmap.c

3963

pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags,

sys/i386/i386/pmap.c

3974

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

3975

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

3982

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

3984

} else if (pmap != kernel_pmap ||

sys/i386/i386/pmap.c

3988

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

3999

(void)pmap_remove_pde(pmap, pde, va, &free);

sys/i386/i386/pmap.c

4001

pmap_invalidate_pde_page(pmap, va, oldpde);

sys/i386/i386/pmap.c

4003

if (pmap_remove_ptes(pmap, va, va + NBPDR, &free))

sys/i386/i386/pmap.c

4004

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

4006

if (pmap != kernel_pmap) {

sys/i386/i386/pmap.c

4019

if (pmap_insert_pt_page(pmap, mt, false, false))

sys/i386/i386/pmap.c

4028

if ((newpde & PG_W) != 0 && pmap != kernel_pmap) {

sys/i386/i386/pmap.c

4034

if (pmap_insert_pt_page(pmap, uwptpg, true, false)) {

sys/i386/i386/pmap.c

4039

pmap->pm_stats.resident_count++;

sys/i386/i386/pmap.c

4046

if (!pmap_pv_insert_pde(pmap, va, newpde, flags)) {

sys/i386/i386/pmap.c

4048

mt = pmap_remove_pt_page(pmap, va);

sys/i386/i386/pmap.c

4052

pmap->pm_stats.resident_count--;

sys/i386/i386/pmap.c

4058

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

4071

pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE;

sys/i386/i386/pmap.c

4072

pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE;

sys/i386/i386/pmap.c

4082

va, pmap);

sys/i386/i386/pmap.c

4099

__CONCAT(PMTYPE, enter_object)(pmap_t pmap, vm_offset_t start, vm_offset_t end,

sys/i386/i386/pmap.c

4114

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

4119

((rv = pmap_enter_4mpage(pmap, va, m, prot)) ==

sys/i386/i386/pmap.c

4123

mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte);

sys/i386/i386/pmap.c

4128

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

4141

__CONCAT(PMTYPE, enter_quick)(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/i386/i386/pmap.c

4146

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

4147

(void)pmap_enter_quick_locked(pmap, va, m, prot, NULL);

sys/i386/i386/pmap.c

4149

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

4153

pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/i386/i386/pmap.c

4159

KASSERT(pmap != kernel_pmap || !VA_IS_CLEANMAP(va) ||

sys/i386/i386/pmap.c

4163

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

4170

if (pmap != kernel_pmap) {

sys/i386/i386/pmap.c

4184

pde = &pmap->pm_pdir[ptepindex];

sys/i386/i386/pmap.c

4197

mpte = _pmap_allocpte(pmap, ptepindex,

sys/i386/i386/pmap.c

4208

pte = pmap_pte_quick(pmap, va);

sys/i386/i386/pmap.c

4220

!pmap_try_insert_pv_entry(pmap, va, m)) {

sys/i386/i386/pmap.c

4222

pmap_abort_ptp(pmap, va, mpte);

sys/i386/i386/pmap.c

4230

pmap->pm_stats.resident_count++;

sys/i386/i386/pmap.c

4233

pmap_cache_bits(pmap, m->md.pat_mode, false);

sys/i386/i386/pmap.c

4240

if (pmap != kernel_pmap)

sys/i386/i386/pmap.c

4254

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

4260

if (pmap_promote_pde(pmap, pde, va, mpte))

sys/i386/i386/pmap.c

4290

__CONCAT(PMTYPE, object_init_pt)(pmap_t pmap, vm_offset_t addr,

sys/i386/i386/pmap.c

4340

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

4341

for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, true);

sys/i386/i386/pmap.c

4343

pde = pmap_pde(pmap, addr);

sys/i386/i386/pmap.c

4347

pmap->pm_stats.resident_count += NBPDR /

sys/i386/i386/pmap.c

4354

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

4368

__CONCAT(PMTYPE, unwire)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap.c

4375

if (pmap_is_current(pmap))

sys/i386/i386/pmap.c

4383

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

4388

pde = pmap_pde(pmap, sva);

sys/i386/i386/pmap.c

4407

pmap->pm_stats.wired_count -= NBPDR /

sys/i386/i386/pmap.c

4414

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

4420

if (!pmap_demote_pde(pmap, pde, sva))

sys/i386/i386/pmap.c

4426

for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,

sys/i386/i386/pmap.c

4442

pmap->pm_stats.wired_count--;

sys/i386/i386/pmap.c

4449

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

4726

__CONCAT(PMTYPE, page_exists_quick)(pmap_t pmap, vm_page_t m)

sys/i386/i386/pmap.c

4738

if (PV_PMAP(pv) == pmap) {

sys/i386/i386/pmap.c

4749

if (PV_PMAP(pv) == pmap) {

sys/i386/i386/pmap.c

4794

pmap_t pmap;

sys/i386/i386/pmap.c

4801

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

4802

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

4803

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

4806

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

4840

__CONCAT(PMTYPE, remove_pages)(pmap_t pmap)

sys/i386/i386/pmap.c

4853

if (pmap != PCPU_GET(curpmap)) {

sys/i386/i386/pmap.c

4859

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

4861

TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {

sys/i386/i386/pmap.c

4862

KASSERT(pc->pc_pmap == pmap, ("Wrong pmap %p %p", pmap,

sys/i386/i386/pmap.c

4874

pte = pmap_pde(pmap, pv->pv_va);

sys/i386/i386/pmap.c

4877

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

4926

pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;

sys/i386/i386/pmap.c

4934

mpte = pmap_remove_pt_page(pmap, pv->pv_va);

sys/i386/i386/pmap.c

4938

pmap->pm_stats.resident_count--;

sys/i386/i386/pmap.c

4945

pmap->pm_stats.resident_count--;

sys/i386/i386/pmap.c

4953

pmap_unuse_pt(pmap, pv->pv_va, &free);

sys/i386/i386/pmap.c

4958

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

4963

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

4965

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5006

pmap_t pmap;

sys/i386/i386/pmap.c

5013

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5014

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5015

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5017

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5032

__CONCAT(PMTYPE, is_prefaultable)(pmap_t pmap, vm_offset_t addr)

sys/i386/i386/pmap.c

5038

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5039

pde = *pmap_pde(pmap, addr);

sys/i386/i386/pmap.c

5041

rv = pmap_pte_ufast(pmap, addr, pde) == 0;

sys/i386/i386/pmap.c

5042

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5076

pmap_t pmap;

sys/i386/i386/pmap.c

5083

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5084

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5085

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5087

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5103

pmap_t pmap;

sys/i386/i386/pmap.c

5121

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5122

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5123

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

5125

(void)pmap_demote_pde(pmap, pde, va);

sys/i386/i386/pmap.c

5126

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5130

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5131

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5132

pde = pmap_pde(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5135

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5149

pmap_invalidate_page_int(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5151

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5179

pmap_t pmap;

sys/i386/i386/pmap.c

5196

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5197

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5198

pde = pmap_pde(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5227

(uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&

sys/i386/i386/pmap.c

5230

pmap_invalidate_page_int(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5234

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5248

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5249

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5250

pde = pmap_pde(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5254

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5259

pmap_invalidate_page_int(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5262

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5282

__CONCAT(PMTYPE, advise)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/i386/i386/pmap.c

5293

if (pmap_is_current(pmap))

sys/i386/i386/pmap.c

5302

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5307

pde = pmap_pde(pmap, sva);

sys/i386/i386/pmap.c

5318

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

5319

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5324

if (!pmap_demote_pde(pmap, pde, sva)) {

sys/i386/i386/pmap.c

5347

pte = pmap_pte_quick(pmap, va);

sys/i386/i386/pmap.c

5350

pmap_remove_pte(pmap, pte, va, NULL);

sys/i386/i386/pmap.c

5357

for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,

sys/i386/i386/pmap.c

5384

pmap_invalidate_range_int(pmap, va, sva);

sys/i386/i386/pmap.c

5389

pmap_invalidate_range_int(pmap, va, sva);

sys/i386/i386/pmap.c

5392

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

5397

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5408

pmap_t pmap;

sys/i386/i386/pmap.c

5426

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5427

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5428

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

5432

pmap_demote_pde(pmap, pde, va) &&

sys/i386/i386/pmap.c

5439

pte = pmap_pte_quick(pmap, va);

sys/i386/i386/pmap.c

5447

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

5449

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5453

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5454

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5455

pde = pmap_pde(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5458

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5466

pmap_invalidate_page_int(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5468

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5820

__CONCAT(PMTYPE, mincore)(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/i386/i386/pmap.c

5827

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5828

pde = *pmap_pde(pmap, addr);

sys/i386/i386/pmap.c

5837

pte = pmap_pte_ufast(pmap, addr, pde);

sys/i386/i386/pmap.c

5858

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5865

pmap_t pmap, oldpmap;

sys/i386/i386/pmap.c

5870

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/i386/i386/pmap.c

5875

CPU_SET_ATOMIC(cpuid, &pmap->pm_active);

sys/i386/i386/pmap.c

5878

CPU_SET(cpuid, &pmap->pm_active);

sys/i386/i386/pmap.c

5881

cr3 = vtophys(pmap->pm_pdpt);

sys/i386/i386/pmap.c

5883

cr3 = vtophys(pmap->pm_pdir);

sys/i386/i386/pmap.c

5889

PCPU_SET(curpmap, pmap);

sys/i386/i386/pmap.c

5894

__CONCAT(PMTYPE, activate_boot)(pmap_t pmap)

sys/i386/i386/pmap.c

5900

CPU_SET_ATOMIC(cpuid, &pmap->pm_active);

sys/i386/i386/pmap.c

5902

CPU_SET(cpuid, &pmap->pm_active);

sys/i386/i386/pmap.c

5904

PCPU_SET(curpmap, pmap);

sys/i386/i386/pmap.c

6133

__CONCAT(PMTYPE, get_cr3)(pmap_t pmap)

sys/i386/i386/pmap.c

6137

return ((u_int)vtophys(pmap->pm_pdpt));

sys/i386/i386/pmap.c

6139

return ((u_int)vtophys(pmap->pm_pdir));

sys/i386/i386/pmap_base.c

106

static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,

sys/i386/i386/pmap_base.c

261

struct pmap kernel_pmap_store;

sys/i386/i386/pmap_base.c

518

pmap_get_cr3(pmap_t pmap)

sys/i386/i386/pmap_base.c

521

return (pmap_methods_ptr->pm_get_cr3(pmap));

sys/i386/i386/pmap_base.c

567

pmap_is_valid_memattr(pmap_t pmap, vm_memattr_t mode)

sys/i386/i386/pmap_base.c

570

return (pmap_methods_ptr->pm_is_valid_memattr(pmap, mode));

sys/i386/i386/pmap_base.c

574

pmap_cache_bits(pmap_t pmap, int mode, bool is_pde)

sys/i386/i386/pmap_base.c

577

return (pmap_methods_ptr->pm_cache_bits(pmap, mode, is_pde));

sys/i386/i386/pmap_base.c

581

pmap_ps_enabled(pmap_t pmap)

sys/i386/i386/pmap_base.c

584

return (pmap_methods_ptr->pm_ps_enabled(pmap));

sys/i386/i386/pmap_base.c

588

pmap_pinit0(pmap_t pmap)

sys/i386/i386/pmap_base.c

591

pmap_methods_ptr->pm_pinit0(pmap);

sys/i386/i386/pmap_base.c

595

pmap_pinit(pmap_t pmap)

sys/i386/i386/pmap_base.c

598

return (pmap_methods_ptr->pm_pinit(pmap));

sys/i386/i386/pmap_base.c

609

pmap_activate_boot(pmap_t pmap)

sys/i386/i386/pmap_base.c

612

pmap_methods_ptr->pm_activate_boot(pmap);

sys/i386/i386/pmap_base.c

616

pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)

sys/i386/i386/pmap_base.c

619

pmap_methods_ptr->pm_advise(pmap, sva, eva, advice);

sys/i386/i386/pmap_base.c

637

pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/i386/i386/pmap_base.c

640

return (pmap_methods_ptr->pm_mincore(pmap, addr, pap));

sys/i386/i386/pmap_base.c

681

pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/i386/i386/pmap_base.c

685

return (pmap_methods_ptr->pm_enter(pmap, va, m, prot, flags, psind));

sys/i386/i386/pmap_base.c

689

pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,

sys/i386/i386/pmap_base.c

693

pmap_methods_ptr->pm_enter_object(pmap, start, end, m_start, prot);

sys/i386/i386/pmap_base.c

697

pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/i386/i386/pmap_base.c

700

pmap_methods_ptr->pm_enter_quick(pmap, va, m, prot);

sys/i386/i386/pmap_base.c

711

pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,

sys/i386/i386/pmap_base.c

715

pmap_methods_ptr->pm_object_init_pt(pmap, addr, object, pindex, size);

sys/i386/i386/pmap_base.c

719

pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap_base.c

722

pmap_methods_ptr->pm_unwire(pmap, sva, eva);

sys/i386/i386/pmap_base.c

726

pmap_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/i386/i386/pmap_base.c

729

return (pmap_methods_ptr->pm_page_exists_quick(pmap, m));

sys/i386/i386/pmap_base.c

747

pmap_remove_pages(pmap_t pmap)

sys/i386/i386/pmap_base.c

750

pmap_methods_ptr->pm_remove_pages(pmap);

sys/i386/i386/pmap_base.c

761

pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/i386/i386/pmap_base.c

764

return (pmap_methods_ptr->pm_is_prefaultable(pmap, addr));

sys/i386/i386/pmap_base.c

826

pmap_extract(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap_base.c

829

return (pmap_methods_ptr->pm_extract(pmap, va));

sys/i386/i386/pmap_base.c

833

pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/i386/i386/pmap_base.c

836

return (pmap_methods_ptr->pm_extract_and_hold(pmap, va, prot));

sys/i386/i386/pmap_base.c

861

pmap_release(pmap_t pmap)

sys/i386/i386/pmap_base.c

864

pmap_methods_ptr->pm_release(pmap);

sys/i386/i386/pmap_base.c

868

pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap_base.c

871

pmap_methods_ptr->pm_remove(pmap, sva, eva);

sys/i386/i386/pmap_base.c

875

pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)

sys/i386/i386/pmap_base.c

878

pmap_methods_ptr->pm_protect(pmap, sva, eva, prot);

sys/i386/i386/pmap_base.c

914

pmap_invalidate_page(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap_base.c

917

pmap_methods_ptr->pm_invalidate_page(pmap, va);

sys/i386/i386/pmap_base.c

921

pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap_base.c

924

pmap_methods_ptr->pm_invalidate_range(pmap, sva, eva);

sys/i386/i386/pmap_base.c

928

pmap_invalidate_all(pmap_t pmap)

sys/i386/i386/pmap_base.c

931

pmap_methods_ptr->pm_invalidate_all(pmap);

sys/i386/i386/pmap_base.c

956

pmap_active_cpus(pmap_t pmap, cpuset_t *res)

sys/i386/i386/pmap_base.c

958

*res = pmap->pm_active;

sys/i386/i386/vm86.c

605

if (vmc->pmap[i].pte_num == pagenum)

sys/i386/i386/vm86.c

606

return (vmc->pmap[i].kva);

sys/i386/i386/vm86.c

616

if (vmc->pmap[i].pte_num == pagenum)

sys/i386/i386/vm86.c

628

vmc->pmap[i].flags = flags;

sys/i386/i386/vm86.c

629

vmc->pmap[i].kva = kva;

sys/i386/i386/vm86.c

630

vmc->pmap[i].pte_num = pagenum;

sys/i386/i386/vm86.c

736

page = vtophys(vmc->pmap[i].kva & PG_FRAME_PAE);

sys/i386/i386/vm86.c

737

entry = vmc->pmap[i].pte_num;

sys/i386/i386/vm86.c

738

vmc->pmap[i].old_pte = pte_pae[entry];

sys/i386/i386/vm86.c

740

pmap_invalidate_page(kernel_pmap, vmc->pmap[i].kva);

sys/i386/i386/vm86.c

745

page = vtophys(vmc->pmap[i].kva & PG_FRAME_NOPAE);

sys/i386/i386/vm86.c

746

entry = vmc->pmap[i].pte_num;

sys/i386/i386/vm86.c

747

vmc->pmap[i].old_pte = pte_nopae[entry];

sys/i386/i386/vm86.c

749

pmap_invalidate_page(kernel_pmap, vmc->pmap[i].kva);

sys/i386/i386/vm86.c

762

entry = vmc->pmap[i].pte_num;

sys/i386/i386/vm86.c

763

pte_pae[entry] = vmc->pmap[i].old_pte;

sys/i386/i386/vm86.c

764

pmap_invalidate_page(kernel_pmap, vmc->pmap[i].kva);

sys/i386/i386/vm86.c

768

entry = vmc->pmap[i].pte_num;

sys/i386/i386/vm86.c

769

pte_nopae[entry] = vmc->pmap[i].old_pte;

sys/i386/i386/vm86.c

770

pmap_invalidate_page(kernel_pmap, vmc->pmap[i].kva);

sys/i386/i386/vm86.c

787

if (page == vmc->pmap[i].pte_num)

sys/i386/i386/vm86.c

788

return (vmc->pmap[i].kva + (addr & PAGE_MASK));

sys/i386/i386/vm86.c

799

if (kva >= vmc->pmap[i].kva &&

sys/i386/i386/vm86.c

800

kva < vmc->pmap[i].kva + PAGE_SIZE) {

sys/i386/i386/vm86.c

801

*off = kva - vmc->pmap[i].kva;

sys/i386/i386/vm86.c

802

*sel = vmc->pmap[i].pte_num << 8;

sys/i386/i386/vm_machdep.c

574

sf_buf_shootdown_curcpu_cb(pmap_t pmap __unused,

sys/i386/include/pcpu.h

56

struct pmap *pc_curpmap; \

sys/i386/include/pmap.h

169

LIST_ENTRY(pmap) pm_list; /* List of all pmaps */

sys/i386/include/pmap.h

175

typedef struct pmap *pmap_t;

sys/i386/include/pmap.h

178

extern struct pmap kernel_pmap_store;

sys/i386/include/pmap.h

181

#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)

sys/i386/include/pmap.h

182

#define PMAP_LOCK_ASSERT(pmap, type) \

sys/i386/include/pmap.h

183

mtx_assert(&(pmap)->pm_mtx, (type))

sys/i386/include/pmap.h

184

#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)

sys/i386/include/pmap.h

185

#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \

sys/i386/include/pmap.h

187

#define PMAP_LOCKED(pmap) mtx_owned(&(pmap)->pm_mtx)

sys/i386/include/pmap.h

188

#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)

sys/i386/include/pmap.h

189

#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)

sys/i386/include/pmap.h

190

#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)

sys/i386/include/pmap.h

216

void pmap_activate_boot(pmap_t pmap);

sys/i386/include/pmap.h

242

bool pmap_ps_enabled(pmap_t pmap);

sys/i386/include/pmap.h

257

#define pmap_map_delete(pmap, sva, eva) pmap_remove(pmap, sva, eva)

sys/i386/include/pmap_nopae.h

94

struct pmap;

sys/i386/include/pmap_nopae.h

95

pt_entry_t *__CONCAT(PMTYPE, pmap_pte)(struct pmap *, vm_offset_t) __pure2;

sys/i386/include/vm86.h

113

} pmap[VM86_PMAPSIZE];

sys/kern/kern_proc.c

2446

if (pmap_extract(map->pmap, addr))

sys/kern/kern_proc.c

2586

pmap_mincore(map->pmap, addr, &pa) & MINCORE_SUPER) != 0) {

sys/kern/subr_bus_dma.c

103

bus_dma_segment_t *list, int sglist_cnt, struct pmap *pmap, int *nsegs,

sys/kern/subr_bus_dma.c

122

error = _bus_dmamap_load_buffer(dmat, map, addr, ds_len, pmap,

sys/kern/subr_bus_dma.c

289

pmap_t pmap;

sys/kern/subr_bus_dma.c

296

pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);

sys/kern/subr_bus_dma.c

298

pmap = kernel_pmap;

sys/kern/subr_bus_dma.c

313

minlen, pmap, flags, NULL, nsegs);

sys/kern/subr_sglist.c

108

_sglist_append_buf(struct sglist *sg, void *buf, size_t len, pmap_t pmap,

sys/kern/subr_sglist.c

125

if (pmap != NULL)

sys/kern/subr_sglist.c

126

paddr = pmap_extract(pmap, vaddr);

sys/kern/subr_sglist.c

148

if (pmap != NULL)

sys/kern/subr_sglist.c

149

paddr = pmap_extract(pmap, vaddr);

sys/kern/subr_sglist.c

601

pmap_t pmap;

sys/kern/subr_sglist.c

613

pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);

sys/kern/subr_sglist.c

615

pmap = NULL;

sys/kern/subr_sglist.c

627

pmap, NULL);

sys/kern/subr_sglist.c

648

pmap_t pmap;

sys/kern/subr_sglist.c

657

pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);

sys/kern/subr_sglist.c

659

pmap = NULL;

sys/kern/subr_sglist.c

677

error = _sglist_append_buf(sg, iov->iov_base, len, pmap, &done);

sys/powerpc/aim/mmu_oea.c

1126

moea_ps_enabled(pmap_t pmap __unused)

sys/powerpc/aim/mmu_oea.c

1137

moea_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/powerpc/aim/mmu_oea.c

1144

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1145

error = moea_enter_locked(pmap, va, m, prot, flags, psind);

sys/powerpc/aim/mmu_oea.c

1147

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1165

moea_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/powerpc/aim/mmu_oea.c

1175

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_oea.c

1206

error = moea_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m),

sys/powerpc/aim/mmu_oea.c

1215

if (pmap != kernel_pmap && error == ENOENT &&

sys/powerpc/aim/mmu_oea.c

1295

moea_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/powerpc/aim/mmu_oea.c

1301

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1302

pvo = moea_pvo_find_va(pmap, va & ~ADDR_POFF, NULL);

sys/powerpc/aim/mmu_oea.c

1310

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1361

moea_is_prefaultable(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_oea.c

1366

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1367

pvo = moea_pvo_find_va(pmap, va & ~ADDR_POFF, NULL);

sys/powerpc/aim/mmu_oea.c

1369

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1396

pmap_t pmap;

sys/powerpc/aim/mmu_oea.c

1409

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea.c

1410

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1424

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1468

pmap_t pmap;

sys/powerpc/aim/mmu_oea.c

1484

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea.c

1485

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1496

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1667

moea_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/powerpc/aim/mmu_oea.c

1679

if (pvo->pvo_pmap == pmap) {

sys/powerpc/aim/mmu_oea.c

1723

moea_pinit(pmap_t pmap)

sys/powerpc/aim/mmu_oea.c

1728

RB_INIT(&pmap->pmap_pvo);

sys/powerpc/aim/mmu_oea.c

1733

if ((pmap->pmap_phys = (pmap_t)moea_kextract((vm_offset_t)pmap))

sys/powerpc/aim/mmu_oea.c

1735

pmap->pmap_phys = pmap;

sys/powerpc/aim/mmu_oea.c

1774

pmap->pm_sr[i] = VSID_MAKE(i, hash);

sys/powerpc/aim/mmu_oea.c

1878

moea_release(pmap_t pmap)

sys/powerpc/aim/mmu_oea.c

1885

if (pmap->pm_sr[0] == 0)

sys/powerpc/aim/mmu_oea.c

1889

idx = VSID_TO_HASH(pmap->pm_sr[0]) & (NPMAPS-1);

sys/powerpc/aim/mmu_oea.c

1925

pmap_t pmap;

sys/powerpc/aim/mmu_oea.c

1932

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea.c

1933

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1935

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea.c

329

bool moea_ps_enabled(pmap_t pmap);

sys/powerpc/aim/mmu_oea64.c

1359

moea64_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/powerpc/aim/mmu_oea64.c

1367

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1369

pvo = moea64_pvo_find_va(pmap, addr);

sys/powerpc/aim/mmu_oea64.c

1379

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1383

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1614

moea64_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_oea64.c

1631

return (moea64_sp_enter(pmap, va, m, prot, flags, psind));

sys/powerpc/aim/mmu_oea64.c

1654

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1656

init_pvo_entry(pvo, pmap, va);

sys/powerpc/aim/mmu_oea64.c

1658

if (moea64_ps_enabled(pmap) &&

sys/powerpc/aim/mmu_oea64.c

1659

(tpvo = moea64_pvo_find_va(pmap, va & ~HPT_SP_MASK)) != NULL &&

sys/powerpc/aim/mmu_oea64.c

1687

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1694

KASSERT(oldpvo->pvo_pmap == pmap, ("pmap of old "

sys/powerpc/aim/mmu_oea64.c

1700

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1714

if (pmap != kernel_pmap && (m->a.flags & PGA_EXECUTABLE) == 0 &&

sys/powerpc/aim/mmu_oea64.c

1717

moea64_syncicache(pmap, va, pa, PAGE_SIZE);

sys/powerpc/aim/mmu_oea64.c

1730

if (moea64_ps_enabled(pmap) && pmap != kernel_pmap && pvo != NULL &&

sys/powerpc/aim/mmu_oea64.c

1737

moea64_sp_promote(pmap, va, m);

sys/powerpc/aim/mmu_oea64.c

1744

moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/powerpc/aim/mmu_oea64.c

1763

} else if (pmap == kernel_pmap) {

sys/powerpc/aim/mmu_oea64.c

1856

moea64_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/powerpc/aim/mmu_oea64.c

1862

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1863

pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);

sys/powerpc/aim/mmu_oea64.c

1869

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1988

moea64_is_prefaultable(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_oea64.c

1993

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1994

pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);

sys/powerpc/aim/mmu_oea64.c

1997

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

2022

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

2035

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

2036

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

2052

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

2089

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

2107

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

2108

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

2133

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

2354

moea64_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/powerpc/aim/mmu_oea64.c

2366

if (!(pvo->pvo_vaddr & PVO_DEAD) && pvo->pvo_pmap == pmap) {

sys/powerpc/aim/mmu_oea64.c

2463

moea64_pinit(pmap_t pmap)

sys/powerpc/aim/mmu_oea64.c

2466

RB_INIT(&pmap->pmap_pvo);

sys/powerpc/aim/mmu_oea64.c

2468

pmap->pm_slb_tree_root = slb_alloc_tree();

sys/powerpc/aim/mmu_oea64.c

2469

pmap->pm_slb = slb_alloc_user_cache();

sys/powerpc/aim/mmu_oea64.c

2470

pmap->pm_slb_len = 0;

sys/powerpc/aim/mmu_oea64.c

2476

moea64_pinit(pmap_t pmap)

sys/powerpc/aim/mmu_oea64.c

2481

RB_INIT(&pmap->pmap_pvo);

sys/powerpc/aim/mmu_oea64.c

2484

pmap->pmap_phys = (pmap_t)moea64_kextract((vm_offset_t)pmap);

sys/powerpc/aim/mmu_oea64.c

2486

pmap->pmap_phys = pmap;

sys/powerpc/aim/mmu_oea64.c

2494

pmap->pm_sr[i] = VSID_MAKE(i, hash);

sys/powerpc/aim/mmu_oea64.c

2496

KASSERT(pmap->pm_sr[0] != 0, ("moea64_pinit: pm_sr[0] = 0"));

sys/powerpc/aim/mmu_oea64.c

263

static void moea64_syncicache(pmap_t pmap, vm_offset_t va,

sys/powerpc/aim/mmu_oea64.c

2644

moea64_release(pmap_t pmap)

sys/powerpc/aim/mmu_oea64.c

2651

slb_free_tree(pmap);

sys/powerpc/aim/mmu_oea64.c

2652

slb_free_user_cache(pmap->pm_slb);

sys/powerpc/aim/mmu_oea64.c

2654

KASSERT(pmap->pm_sr[0] != 0, ("moea64_release: pm_sr[0] = 0"));

sys/powerpc/aim/mmu_oea64.c

2656

moea64_release_vsid(VSID_TO_HASH(pmap->pm_sr[0]));

sys/powerpc/aim/mmu_oea64.c

2765

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

2771

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

2772

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

2785

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

331

static int moea64_sp_enter(pmap_t pmap, vm_offset_t va,

sys/powerpc/aim/mmu_oea64.c

337

static void moea64_sp_promote(pmap_t pmap, vm_offset_t va, vm_page_t m);

sys/powerpc/aim/mmu_oea64.c

3540

moea64_ps_enabled(pmap_t pmap)

sys/powerpc/aim/mmu_oea64.c

3649

moea64_sp_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_oea64.c

3667

KASSERT(pmap != kernel_pmap,

sys/powerpc/aim/mmu_oea64.c

3700

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

3703

moea64_remove_locked(pmap, va, va + HPT_SP_SIZE, &tofree);

sys/powerpc/aim/mmu_oea64.c

3725

init_pvo_entry(pvo, pmap, va);

sys/powerpc/aim/mmu_oea64.c

3737

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

3756

moea64_syncicache(pmap, sva, spa, HPT_SP_SIZE);

sys/powerpc/aim/mmu_oea64.c

3760

__func__, (uintmax_t)sva, pmap);

sys/powerpc/aim/mmu_oea64.c

3768

moea64_sp_promote(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/powerpc/aim/mmu_oea64.c

3784

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

3802

first = pvo = moea64_pvo_find_va(pmap, sva);

sys/powerpc/aim/mmu_oea64.c

3808

__func__, pmap, (uintmax_t)va);

sys/powerpc/aim/mmu_oea64.c

3814

__func__, pmap, (uintmax_t)va,

sys/powerpc/aim/mmu_oea64.c

3823

__func__, pmap, (uintmax_t)va,

sys/powerpc/aim/mmu_oea64.c

3832

__func__, pmap, (uintmax_t)va,

sys/powerpc/aim/mmu_oea64.c

3841

__func__, pmap, (uintmax_t)va,

sys/powerpc/aim/mmu_oea64.c

3848

pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo);

sys/powerpc/aim/mmu_oea64.c

3868

pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {

sys/powerpc/aim/mmu_oea64.c

3881

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

3886

__func__, (uintmax_t)sva, pmap);

sys/powerpc/aim/mmu_oea64.c

3891

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

3902

pmap_t pmap __diagused;

sys/powerpc/aim/mmu_oea64.c

3907

pmap = sp->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

3908

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_oea64.c

3920

pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo),

sys/powerpc/aim/mmu_oea64.c

3950

__func__, (uintmax_t)PVO_VADDR(sp), pmap);

sys/powerpc/aim/mmu_oea64.c

4100

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

4105

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

4106

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_oea64.c

4110

sp = moea64_pvo_find_va(pmap, PVO_VADDR(pvo) & ~HPT_SP_MASK);

sys/powerpc/aim/mmu_oea64.c

4119

pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {

sys/powerpc/aim/mmu_oea64.c

4145

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

4149

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

4150

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4158

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4163

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4176

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

4181

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

4182

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4190

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4196

sp = moea64_pvo_find_va(pmap, PVO_VADDR(pvo) & ~HPT_SP_MASK);

sys/powerpc/aim/mmu_oea64.c

4205

pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {

sys/powerpc/aim/mmu_oea64.c

4213

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4226

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

4229

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

4246

(PVO_VADDR(pvo) >> HPT_SP_SHIFT) ^ (uintptr_t)pmap) &

sys/powerpc/aim/mmu_oea64.c

4271

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4280

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4288

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4300

moea64_pvo_protect(pmap, pvo,

sys/powerpc/aim/mmu_oea64.c

4303

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

537

init_pvo_entry(struct pvo_entry *pvo, pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_oea64.c

543

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_oea64.c

545

pvo->pvo_pmap = pmap;

sys/powerpc/aim/mmu_oea64.c

548

vsid = va_to_vsid(pmap, va);

sys/powerpc/aim/mmu_oea64.c

552

if (pmap == kernel_pmap && (pvo->pvo_vaddr & PVO_LARGE) != 0)

sys/powerpc/aim/mmu_radix.c

1020

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

1025

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

1073

pmap_invalidate_page(pmap_t pmap, vm_offset_t start)

sys/powerpc/aim/mmu_radix.c

1076

if (pmap == kernel_pmap)

sys/powerpc/aim/mmu_radix.c

1079

radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);

sys/powerpc/aim/mmu_radix.c

1084

pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start)

sys/powerpc/aim/mmu_radix.c

1087

if (pmap == kernel_pmap)

sys/powerpc/aim/mmu_radix.c

1090

radix_tlbie_invlpg_user_2m(pmap->pm_pid, start);

sys/powerpc/aim/mmu_radix.c

1095

pmap_invalidate_pwc(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

1098

if (pmap == kernel_pmap)

sys/powerpc/aim/mmu_radix.c

1101

radix_tlbie_invlpwc_user(pmap->pm_pid);

sys/powerpc/aim/mmu_radix.c

1106

pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)

sys/powerpc/aim/mmu_radix.c

1109

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

1113

if (pmap == kernel_pmap) {

sys/powerpc/aim/mmu_radix.c

1120

radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);

sys/powerpc/aim/mmu_radix.c

1128

pmap_invalidate_all(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

1131

if (pmap == kernel_pmap)

sys/powerpc/aim/mmu_radix.c

1134

radix_tlbie_flush_user(pmap->pm_pid);

sys/powerpc/aim/mmu_radix.c

1139

pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e)

sys/powerpc/aim/mmu_radix.c

1155

pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1);

sys/powerpc/aim/mmu_radix.c

1157

pmap_invalidate_page_2m(pmap, va);

sys/powerpc/aim/mmu_radix.c

1159

pmap_invalidate_pwc(pmap);

sys/powerpc/aim/mmu_radix.c

1183

reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)

sys/powerpc/aim/mmu_radix.c

1191

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

1203

TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {

sys/powerpc/aim/mmu_radix.c

1219

m = reclaim_pv_chunk(pmap, lockp);

sys/powerpc/aim/mmu_radix.c

1228

pc->pc_pmap = pmap;

sys/powerpc/aim/mmu_radix.c

1231

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1257

pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

1270

if (pmap == PV_PMAP(pv) && va == pv->pv_va) {

sys/powerpc/aim/mmu_radix.c

1285

pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/powerpc/aim/mmu_radix.c

1295

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

1307

pv = pmap_pvh_remove(pvh, pmap, va);

sys/powerpc/aim/mmu_radix.c

1317

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/powerpc/aim/mmu_radix.c

1337

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1338

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1342

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1343

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1350

reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap)

sys/powerpc/aim/mmu_radix.c

1353

if (pmap == NULL)

sys/powerpc/aim/mmu_radix.c

1355

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

1356

if (pmap != locked_pmap)

sys/powerpc/aim/mmu_radix.c

1357

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

1379

pmap_t next_pmap, pmap;

sys/powerpc/aim/mmu_radix.c

1390

pmap = NULL;

sys/powerpc/aim/mmu_radix.c

1421

if (pmap != next_pmap) {

sys/powerpc/aim/mmu_radix.c

1422

reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);

sys/powerpc/aim/mmu_radix.c

1423

pmap = next_pmap;

sys/powerpc/aim/mmu_radix.c

1425

if (pmap > locked_pmap) {

sys/powerpc/aim/mmu_radix.c

1427

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

1430

} else if (pmap != locked_pmap) {

sys/powerpc/aim/mmu_radix.c

1431

if (PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

1435

pmap = NULL; /* pmap is not locked */

sys/powerpc/aim/mmu_radix.c

1456

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

1481

pmap_unuse_pt(pmap, va, be64toh(*l3e), &free);

sys/powerpc/aim/mmu_radix.c

1490

pmap_resident_count_dec(pmap, freed);

sys/powerpc/aim/mmu_radix.c

1494

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1506

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1509

if (pmap == locked_pmap)

sys/powerpc/aim/mmu_radix.c

1514

if (active_reclaims == 1 && pmap != NULL) {

sys/powerpc/aim/mmu_radix.c

1533

reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);

sys/powerpc/aim/mmu_radix.c

1548

free_pv_entry(pmap_t pmap, pv_entry_t pv)

sys/powerpc/aim/mmu_radix.c

1554

if (pmap != kernel_pmap)

sys/powerpc/aim/mmu_radix.c

1555

printf("%s(%p, %p)\n", __func__, pmap, pv);

sys/powerpc/aim/mmu_radix.c

1557

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

1568

if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {

sys/powerpc/aim/mmu_radix.c

1569

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1570

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1574

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1605

get_pv_entry(pmap_t pmap, struct rwlock **lockp)

sys/powerpc/aim/mmu_radix.c

1612

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

1615

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/powerpc/aim/mmu_radix.c

1628

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1629

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,

sys/powerpc/aim/mmu_radix.c

1645

m = reclaim_pv_chunk(pmap, lockp);

sys/powerpc/aim/mmu_radix.c

1653

pc->pc_pmap = pmap;

sys/powerpc/aim/mmu_radix.c

1660

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1674

pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/powerpc/aim/mmu_radix.c

1695

pv = pmap_pvh_remove(&m->md, pmap, va);

sys/powerpc/aim/mmu_radix.c

1705

pmap_pvh_free(&m->md, pmap, va);

sys/powerpc/aim/mmu_radix.c

1716

pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

1720

pv = pmap_pvh_remove(pvh, pmap, va);

sys/powerpc/aim/mmu_radix.c

1722

free_pv_entry(pmap, pv);

sys/powerpc/aim/mmu_radix.c

1730

pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

1735

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

1737

if ((pv = get_pv_entry(pmap, NULL)) != NULL) {

sys/powerpc/aim/mmu_radix.c

1844

bzero(kernel_pmap, sizeof(struct pmap));

sys/powerpc/aim/mmu_radix.c

2217

mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/powerpc/aim/mmu_radix.c

2232

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2234

l1e = pmap_pml1e(pmap, sva);

sys/powerpc/aim/mmu_radix.c

2259

if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) {

sys/powerpc/aim/mmu_radix.c

2288

pmap_remove_pte(pmap, pte, va, be64toh(*l3e), NULL,

sys/powerpc/aim/mmu_radix.c

2300

MPASS(pte == pmap_pte(pmap, sva));

sys/powerpc/aim/mmu_radix.c

2331

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

2332

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2427

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

2453

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

2454

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

2457

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2460

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2465

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

2468

pmap_demote_l3e_locked(pmap, l3e, va, &lock) &&

sys/powerpc/aim/mmu_radix.c

2484

pmap_invalidate_page(pmap, va);

sys/powerpc/aim/mmu_radix.c

2486

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2489

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

2490

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

2494

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2497

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2501

l3e = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

2507

pmap_invalidate_page(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

2509

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2722

pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va,

sys/powerpc/aim/mmu_radix.c

2729

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

2741

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

2765

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

2778

(va & ~L3_PAGE_MASK), pmap);

sys/powerpc/aim/mmu_radix.c

2782

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

2799

if (pmap_insert_pt_page(pmap, mpte)) {

sys/powerpc/aim/mmu_radix.c

2802

pmap);

sys/powerpc/aim/mmu_radix.c

2810

pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp);

sys/powerpc/aim/mmu_radix.c

2816

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

2825

mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

2841

CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va,

sys/powerpc/aim/mmu_radix.c

2883

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2888

rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock);

sys/powerpc/aim/mmu_radix.c

2898

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

2900

pmap_demote_l3e_locked(pmap, l3e, va, &lock))) {

sys/powerpc/aim/mmu_radix.c

2912

mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va),

sys/powerpc/aim/mmu_radix.c

2936

pmap, va, m, prot, flags, psind, pmap->pm_pid,

sys/powerpc/aim/mmu_radix.c

2939

pmap_pte_walk(pmap->pm_pml1, va);

sys/powerpc/aim/mmu_radix.c

2950

pmap->pm_stats.wired_count++;

sys/powerpc/aim/mmu_radix.c

2952

pmap->pm_stats.wired_count--;

sys/powerpc/aim/mmu_radix.c

3018

pv = pmap_pvh_remove(&om->md, pmap, va);

sys/powerpc/aim/mmu_radix.c

3020

free_pv_entry(pmap, pv);

sys/powerpc/aim/mmu_radix.c

3041

if (pmap != kernel_pmap) {

sys/powerpc/aim/mmu_radix.c

3045

pmap, va, m, prot, flags, psind,

sys/powerpc/aim/mmu_radix.c

3046

pmap->pm_pid, curproc->p_pid,

sys/powerpc/aim/mmu_radix.c

3055

pmap->pm_stats.wired_count++;

sys/powerpc/aim/mmu_radix.c

3056

pmap_resident_count_inc(pmap, 1);

sys/powerpc/aim/mmu_radix.c

3064

pv = get_pv_entry(pmap, &lock);

sys/powerpc/aim/mmu_radix.c

3070

pv, pmap);

sys/powerpc/aim/mmu_radix.c

3119

mmu_radix_ps_enabled(pmap) &&

sys/powerpc/aim/mmu_radix.c

3122

pmap_promote_l3e(pmap, l3e, va, &lock) == 0)

sys/powerpc/aim/mmu_radix.c

3126

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

3128

pmap_invalidate_page(pmap, va);

sys/powerpc/aim/mmu_radix.c

3134

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3144

pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t pdpg)

sys/powerpc/aim/mmu_radix.c

3149

if (pmap_unwire_ptp(pmap, va, pdpg, &free)) {

sys/powerpc/aim/mmu_radix.c

3156

pmap_invalidate_page(pmap, va);

sys/powerpc/aim/mmu_radix.c

3169

pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/powerpc/aim/mmu_radix.c

3174

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

3185

return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP |

sys/powerpc/aim/mmu_radix.c

3202

pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags,

sys/powerpc/aim/mmu_radix.c

3212

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

3214

if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?

sys/powerpc/aim/mmu_radix.c

3217

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

3229

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

3241

(void)pmap_remove_l3e(pmap, l3e, va, &free, lockp);

sys/powerpc/aim/mmu_radix.c

3242

pmap_invalidate_l3e_page(pmap, va, oldl3e);

sys/powerpc/aim/mmu_radix.c

3244

if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e,

sys/powerpc/aim/mmu_radix.c

3246

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

3251

if (pmap_insert_pt_page(pmap, mt)) {

sys/powerpc/aim/mmu_radix.c

3268

if ((newpde & PG_W) != 0 && pmap != kernel_pmap) {

sys/powerpc/aim/mmu_radix.c

3271

pmap_abort_ptp(pmap, va, pdpg);

sys/powerpc/aim/mmu_radix.c

3275

if (pmap_insert_pt_page(pmap, uwptpg)) {

sys/powerpc/aim/mmu_radix.c

3278

pmap_abort_ptp(pmap, va, pdpg);

sys/powerpc/aim/mmu_radix.c

3281

pmap_resident_count_inc(pmap, 1);

sys/powerpc/aim/mmu_radix.c

3290

if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) {

sys/powerpc/aim/mmu_radix.c

3291

pmap_abort_ptp(pmap, va, pdpg);

sys/powerpc/aim/mmu_radix.c

3293

mt = pmap_remove_pt_page(pmap, va);

sys/powerpc/aim/mmu_radix.c

3297

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

3303

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

3316

pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE;

sys/powerpc/aim/mmu_radix.c

3317

pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);

sys/powerpc/aim/mmu_radix.c

3328

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

3333

mmu_radix_enter_object(pmap_t pmap, vm_offset_t start,

sys/powerpc/aim/mmu_radix.c

3344

CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start,

sys/powerpc/aim/mmu_radix.c

3352

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3356

m->psind == 1 && mmu_radix_ps_enabled(pmap) &&

sys/powerpc/aim/mmu_radix.c

3357

pmap_enter_2mpage(pmap, va, m, prot, &lock)) {

sys/powerpc/aim/mmu_radix.c

3360

mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot,

sys/powerpc/aim/mmu_radix.c

3369

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

3370

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3374

mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

3384

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

3404

ptepa = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

3422

mpte = _pmap_allocpte(pmap, ptepindex, NULL);

sys/powerpc/aim/mmu_radix.c

3431

pte = pmap_pte(pmap, va);

sys/powerpc/aim/mmu_radix.c

3445

!pmap_try_insert_pv_entry(pmap, va, m, lockp)) {

sys/powerpc/aim/mmu_radix.c

3448

if (pmap_unwire_ptp(pmap, va, mpte, &free)) {

sys/powerpc/aim/mmu_radix.c

3466

pmap_resident_count_inc(pmap, 1);

sys/powerpc/aim/mmu_radix.c

3481

mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

3489

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3490

mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock,

sys/powerpc/aim/mmu_radix.c

3496

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

3497

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3501

mmu_radix_extract(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

3507

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

3533

mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/powerpc/aim/mmu_radix.c

3540

CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot);

sys/powerpc/aim/mmu_radix.c

3541

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3542

l3ep = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

3558

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

364

pmap_pml1e(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

367

return (&pmap->pm_pml1[pmap_pml1e_index(va)]);

sys/powerpc/aim/mmu_radix.c

371

pmap_pml2e(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

375

l1e = pmap_pml1e(pmap, va);

sys/powerpc/aim/mmu_radix.c

3761

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

3770

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

3771

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

3774

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3777

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3781

pte = pmap_pte(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

3788

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3795

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

3796

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

3800

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3804

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3808

pte = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

3815

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

382

pmap_pml3e(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

3848

mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/powerpc/aim/mmu_radix.c

3854

CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);

sys/powerpc/aim/mmu_radix.c

3856

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3857

l3e = pmap_pml3e(pmap, addr);

sys/powerpc/aim/mmu_radix.c

386

l2e = pmap_pml2e(pmap, va);

sys/powerpc/aim/mmu_radix.c

3862

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3900

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

3925

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

3926

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

3929

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

393

pmap_pte(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

3932

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3936

l3e = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

3966

(uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&

sys/powerpc/aim/mmu_radix.c

3969

pmap_invalidate_page(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

397

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

3977

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3994

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

3995

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

3999

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4002

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4006

l3e = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

4015

pmap_invalidate_page(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

4018

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4044

mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr,

sys/powerpc/aim/mmu_radix.c

4053

CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr,

sys/powerpc/aim/mmu_radix.c

4060

if (!mmu_radix_ps_enabled(pmap))

sys/powerpc/aim/mmu_radix.c

4094

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4097

pdpg = pmap_allocl3e(pmap, addr, NULL);

sys/powerpc/aim/mmu_radix.c

4114

pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);

sys/powerpc/aim/mmu_radix.c

4126

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4131

mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/powerpc/aim/mmu_radix.c

4141

CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m);

sys/powerpc/aim/mmu_radix.c

4146

if (PV_PMAP(pv) == pmap) {

sys/powerpc/aim/mmu_radix.c

4157

if (PV_PMAP(pv) == pmap) {

sys/powerpc/aim/mmu_radix.c

4184

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

4197

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

4198

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

4201

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4204

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4208

pte = pmap_pte(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

4211

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4216

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

4217

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

4221

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4225

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4229

pte = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

4232

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4246

mmu_radix_pinit(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

4251

CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);

sys/powerpc/aim/mmu_radix.c

4256

pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK);

sys/powerpc/aim/mmu_radix.c

4259

pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE);

sys/powerpc/aim/mmu_radix.c

4260

vm_radix_init(&pmap->pm_radix);

sys/powerpc/aim/mmu_radix.c

4261

TAILQ_INIT(&pmap->pm_pvchunk);

sys/powerpc/aim/mmu_radix.c

4262

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/powerpc/aim/mmu_radix.c

4263

pmap->pm_flags = PMAP_PDE_SUPERPAGE;

sys/powerpc/aim/mmu_radix.c

4266

pmap->pm_pid = pid;

sys/powerpc/aim/mmu_radix.c

4267

l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1);

sys/powerpc/aim/mmu_radix.c

4286

_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)

sys/powerpc/aim/mmu_radix.c

4290

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4298

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4300

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4321

l1e = &pmap->pm_pml1[pml1index];

sys/powerpc/aim/mmu_radix.c

4335

l1e = &pmap->pm_pml1[pml1index];

sys/powerpc/aim/mmu_radix.c

4338

if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index,

sys/powerpc/aim/mmu_radix.c

4368

l1e = &pmap->pm_pml1[pml1index];

sys/powerpc/aim/mmu_radix.c

437

vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va);

sys/powerpc/aim/mmu_radix.c

4371

if (_pmap_allocpte(pmap, NUPDE + pdpindex,

sys/powerpc/aim/mmu_radix.c

4384

if (_pmap_allocpte(pmap, NUPDE + pdpindex,

sys/powerpc/aim/mmu_radix.c

4405

pmap_resident_count_inc(pmap, 1);

sys/powerpc/aim/mmu_radix.c

4409

pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)

sys/powerpc/aim/mmu_radix.c

4416

pdpe = pmap_pml2e(pmap, va);

sys/powerpc/aim/mmu_radix.c

4425

pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp);

sys/powerpc/aim/mmu_radix.c

4433

pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)

sys/powerpc/aim/mmu_radix.c

4447

pd = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

4454

if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) {

sys/powerpc/aim/mmu_radix.c

4475

m = _pmap_allocpte(pmap, ptepindex, lockp);

sys/powerpc/aim/mmu_radix.c

4483

mmu_radix_pinit0(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

4486

CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);

sys/powerpc/aim/mmu_radix.c

4487

PMAP_LOCK_INIT(pmap);

sys/powerpc/aim/mmu_radix.c

4488

pmap->pm_pml1 = kernel_pmap->pm_pml1;

sys/powerpc/aim/mmu_radix.c

4489

pmap->pm_pid = kernel_pmap->pm_pid;

sys/powerpc/aim/mmu_radix.c

4491

vm_radix_init(&pmap->pm_radix);

sys/powerpc/aim/mmu_radix.c

4492

TAILQ_INIT(&pmap->pm_pvchunk);

sys/powerpc/aim/mmu_radix.c

4493

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/powerpc/aim/mmu_radix.c

4495

pmap->pm_flags = PMAP_PDE_SUPERPAGE;

sys/powerpc/aim/mmu_radix.c

4501

pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot)

sys/powerpc/aim/mmu_radix.c

4508

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4541

mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/powerpc/aim/mmu_radix.c

4551

CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva,

sys/powerpc/aim/mmu_radix.c

4556

mmu_radix_remove(pmap, sva, eva);

sys/powerpc/aim/mmu_radix.c

4567

pmap, sva, eva, prot, pmap->pm_pid);

sys/powerpc/aim/mmu_radix.c

4571

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4573

l1e = pmap_pml1e(pmap, sva);

sys/powerpc/aim/mmu_radix.c

4611

if (pmap_protect_l3e(pmap, l3e, sva, prot))

sys/powerpc/aim/mmu_radix.c

4614

} else if (!pmap_demote_l3e(pmap, l3e, sva)) {

sys/powerpc/aim/mmu_radix.c

4631

MPASS(pte == pmap_pte(pmap, sva));

sys/powerpc/aim/mmu_radix.c

4663

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

4664

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4756

pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)

sys/powerpc/aim/mmu_radix.c

4759

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4760

return (vm_radix_insert(&pmap->pm_radix, mpte));

sys/powerpc/aim/mmu_radix.c

4770

pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

4773

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4774

return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va)));

sys/powerpc/aim/mmu_radix.c

4784

pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/powerpc/aim/mmu_radix.c

4789

_pmap_unwire_ptp(pmap, va, m, free);

sys/powerpc/aim/mmu_radix.c

4796

_pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/powerpc/aim/mmu_radix.c

4799

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4806

pml1 = pmap_pml1e(pmap, va);

sys/powerpc/aim/mmu_radix.c

4811

l2e = pmap_pml2e(pmap, va);

sys/powerpc/aim/mmu_radix.c

4816

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

4819

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

4824

pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME);

sys/powerpc/aim/mmu_radix.c

4825

pmap_unwire_ptp(pmap, va, pdpg, free);

sys/powerpc/aim/mmu_radix.c

4831

pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME);

sys/powerpc/aim/mmu_radix.c

4832

pmap_unwire_ptp(pmap, va, pdppg, free);

sys/powerpc/aim/mmu_radix.c

4847

pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde,

sys/powerpc/aim/mmu_radix.c

4856

return (pmap_unwire_ptp(pmap, va, mpte, free));

sys/powerpc/aim/mmu_radix.c

4860

mmu_radix_release(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

4863

CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);

sys/powerpc/aim/mmu_radix.c

4864

KASSERT(pmap->pm_stats.resident_count == 0,

sys/powerpc/aim/mmu_radix.c

4866

pmap->pm_stats.resident_count));

sys/powerpc/aim/mmu_radix.c

4867

KASSERT(vm_radix_is_empty(&pmap->pm_radix),

sys/powerpc/aim/mmu_radix.c

4870

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

4871

isa3_proctab[pmap->pm_pid].proctab0 = 0;

sys/powerpc/aim/mmu_radix.c

4872

uma_zfree(zone_radix_pgd, pmap->pm_pml1);

sys/powerpc/aim/mmu_radix.c

4873

vmem_free(asid_arena, pmap->pm_pid, 1);

sys/powerpc/aim/mmu_radix.c

4882

pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags,

sys/powerpc/aim/mmu_radix.c

4889

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4891

if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?

sys/powerpc/aim/mmu_radix.c

4918

pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

4924

rv = pmap_demote_l3e_locked(pmap, pde, va, &lock);

sys/powerpc/aim/mmu_radix.c

4931

pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,

sys/powerpc/aim/mmu_radix.c

4941

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4946

if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==

sys/powerpc/aim/mmu_radix.c

4969

pmap_remove_l3e(pmap, l3e, sva, &free, lockp);

sys/powerpc/aim/mmu_radix.c

4970

pmap_invalidate_l3e_page(pmap, sva, oldpde);

sys/powerpc/aim/mmu_radix.c

4973

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

4978

pmap_resident_count_inc(pmap, 1);

sys/powerpc/aim/mmu_radix.c

5015

reserve_pv_entries(pmap, NPTEPG - 1, lockp);

sys/powerpc/aim/mmu_radix.c

5025

pmap_invalidate_l3e_page(pmap, trunc_2mpage(va), oldpde);

sys/powerpc/aim/mmu_radix.c

5030

pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp);

sys/powerpc/aim/mmu_radix.c

5034

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

5042

pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

5047

KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));

sys/powerpc/aim/mmu_radix.c

5048

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

5049

mpte = pmap_remove_pt_page(pmap, va);

sys/powerpc/aim/mmu_radix.c

5071

pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,

sys/powerpc/aim/mmu_radix.c

5079

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

5084

pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE);

sys/powerpc/aim/mmu_radix.c

5085

pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);

sys/powerpc/aim/mmu_radix.c

5089

pmap_pvh_free(pvh, pmap, sva);

sys/powerpc/aim/mmu_radix.c

5102

if (pmap == kernel_pmap) {

sys/powerpc/aim/mmu_radix.c

5103

pmap_remove_kernel_l3e(pmap, pdq, sva);

sys/powerpc/aim/mmu_radix.c

5105

mpte = pmap_remove_pt_page(pmap, sva);

sys/powerpc/aim/mmu_radix.c

5107

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

5114

return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free));

sys/powerpc/aim/mmu_radix.c

5121

pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,

sys/powerpc/aim/mmu_radix.c

5128

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

5131

pmap->pm_stats.wired_count -= 1;

sys/powerpc/aim/mmu_radix.c

5132

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

5140

pmap_pvh_free(&m->md, pmap, va);

sys/powerpc/aim/mmu_radix.c

5148

return (pmap_unuse_pt(pmap, va, ptepde, free));

sys/powerpc/aim/mmu_radix.c

5155

pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e,

sys/powerpc/aim/mmu_radix.c

5162

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

5172

invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock);

sys/powerpc/aim/mmu_radix.c

5176

pmap_invalidate_page(pmap, va);

sys/powerpc/aim/mmu_radix.c

5184

pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/powerpc/aim/mmu_radix.c

5191

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

5196

MPASS(pte == pmap_pte(pmap, sva));

sys/powerpc/aim/mmu_radix.c

5206

if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) {

sys/powerpc/aim/mmu_radix.c

5213

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

5215

pmap_invalidate_range(pmap, va, sva);

sys/powerpc/aim/mmu_radix.c

5220

mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/powerpc/aim/mmu_radix.c

5230

CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);

sys/powerpc/aim/mmu_radix.c

5235

if (pmap->pm_stats.resident_count == 0)

sys/powerpc/aim/mmu_radix.c

5245

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5253

l3e = pmap_pml3e(pmap, sva);

sys/powerpc/aim/mmu_radix.c

5255

anyvalid = pmap_remove_page(pmap, sva, l3e, &free);

sys/powerpc/aim/mmu_radix.c

5262

if (pmap->pm_stats.resident_count == 0)

sys/powerpc/aim/mmu_radix.c

5264

l1e = pmap_pml1e(pmap, sva);

sys/powerpc/aim/mmu_radix.c

5305

pmap_remove_l3e(pmap, l3e, sva, &free, &lock);

sys/powerpc/aim/mmu_radix.c

5308

} else if (!pmap_demote_l3e_locked(pmap, l3e, sva,

sys/powerpc/aim/mmu_radix.c

5324

if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock))

sys/powerpc/aim/mmu_radix.c

5331

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

5332

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5341

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

5359

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

5360

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

5363

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5367

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5372

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

5373

(void)pmap_demote_l3e_locked(pmap, l3e, va, &lock);

sys/powerpc/aim/mmu_radix.c

5374

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5377

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

5378

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

5382

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5386

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5390

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

5391

l3e = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

5397

pmap->pm_stats.wired_count--;

sys/powerpc/aim/mmu_radix.c

5406

pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free);

sys/powerpc/aim/mmu_radix.c

5407

pmap_invalidate_page(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

5410

free_pv_entry(pmap, pv);

sys/powerpc/aim/mmu_radix.c

5411

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5445

mmu_radix_remove_pages(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

5448

CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);

sys/powerpc/aim/mmu_radix.c

5471

KASSERT(pmap->pm_pid == mfspr(SPR_PID),

sys/powerpc/aim/mmu_radix.c

5472

("non-current asid %lu - expected %lu", pmap->pm_pid,

sys/powerpc/aim/mmu_radix.c

5478

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5479

TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {

sys/powerpc/aim/mmu_radix.c

5493

pte = pmap_pml2e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

5565

pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);

sys/powerpc/aim/mmu_radix.c

5575

mpte = pmap_remove_pt_page(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

5577

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

5584

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

5587

pmap, pv);

sys/powerpc/aim/mmu_radix.c

5599

pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free);

sys/powerpc/aim/mmu_radix.c

5609

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

5615

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

5616

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5624

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

5644

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

5645

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

5648

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5651

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5656

l3e = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

5658

(void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock);

sys/powerpc/aim/mmu_radix.c

566

static bool pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,

sys/powerpc/aim/mmu_radix.c

5662

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5665

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

5666

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

5670

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5674

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5679

l3e = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

568

static bool pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va);

sys/powerpc/aim/mmu_radix.c

5692

pmap_invalidate_page(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

5694

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

570

static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,

sys/powerpc/aim/mmu_radix.c

5713

mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/powerpc/aim/mmu_radix.c

572

static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,

sys/powerpc/aim/mmu_radix.c

5721

CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);

sys/powerpc/aim/mmu_radix.c

5722

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5724

l1e = pmap_pml1e(pmap, sva);

sys/powerpc/aim/mmu_radix.c

574

static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);

sys/powerpc/aim/mmu_radix.c

575

static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde,

sys/powerpc/aim/mmu_radix.c

5755

pmap->pm_stats.wired_count -= L3_PAGE_SIZE /

sys/powerpc/aim/mmu_radix.c

5758

} else if (!pmap_demote_l3e(pmap, l3e, sva))

sys/powerpc/aim/mmu_radix.c

5765

MPASS(pte == pmap_pte(pmap, sva));

sys/powerpc/aim/mmu_radix.c

577

static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/powerpc/aim/mmu_radix.c

5778

pmap->pm_stats.wired_count--;

sys/powerpc/aim/mmu_radix.c

5781

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

580

static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e,

sys/powerpc/aim/mmu_radix.c

5806

mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)

sys/powerpc/aim/mmu_radix.c

5813

CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);

sys/powerpc/aim/mmu_radix.c

5814

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5816

l3ep = pmap_pml3e(pmap, addr);

sys/powerpc/aim/mmu_radix.c

583

static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/powerpc/aim/mmu_radix.c

5847

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5854

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

5859

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/powerpc/aim/mmu_radix.c

586

static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);

sys/powerpc/aim/mmu_radix.c

5861

if (pmap->pm_pid > isa3_base_pid &&

sys/powerpc/aim/mmu_radix.c

5862

curpid != pmap->pm_pid) {

sys/powerpc/aim/mmu_radix.c

5863

mmu_radix_pid_set(pmap);

sys/powerpc/aim/mmu_radix.c

587

static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);

sys/powerpc/aim/mmu_radix.c

588

static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

591

static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

593

static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde,

sys/powerpc/aim/mmu_radix.c

598

static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp);

sys/powerpc/aim/mmu_radix.c

599

static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va,

sys/powerpc/aim/mmu_radix.c

601

static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,

sys/powerpc/aim/mmu_radix.c

6020

pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

6027

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

603

static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

6034

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

605

static bool pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free);

sys/powerpc/aim/mmu_radix.c

6062

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

6066

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

607

static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start);

sys/powerpc/aim/mmu_radix.c

608

static void pmap_invalidate_all(pmap_t pmap);

sys/powerpc/aim/mmu_radix.c

6513

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

6521

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

6522

db_printf("pmap %p ", pmap);

sys/powerpc/aim/mmu_radix.c

6523

if (pmap != NULL) {

sys/powerpc/aim/mmu_radix.c

6524

db_printf("asid: %lu\n", pmap->pm_pid);

sys/powerpc/aim/mmu_radix.c

6525

pmap_pte_walk(pmap->pm_pml1, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

6533

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

6542

pmap = kernel_pmap;

sys/powerpc/aim/mmu_radix.c

6544

pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace);

sys/powerpc/aim/mmu_radix.c

6546

pmap = vmspace_pmap(curthread->td_proc->p_vmspace);

sys/powerpc/aim/mmu_radix.c

6548

pmap_pte_walk(pmap->pm_pml1, va);

sys/powerpc/aim/mmu_radix.c

819

mmu_radix_pid_set(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

822

mtspr(SPR_PID, pmap->pm_pid);

sys/powerpc/aim/mmu_radix.c

867

pmap_resident_count_inc(pmap_t pmap, int count)

sys/powerpc/aim/mmu_radix.c

870

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

871

pmap->pm_stats.resident_count += count;

sys/powerpc/aim/mmu_radix.c

875

pmap_resident_count_dec(pmap_t pmap, int count)

sys/powerpc/aim/mmu_radix.c

878

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

879

KASSERT(pmap->pm_stats.resident_count >= count,

sys/powerpc/aim/mmu_radix.c

880

("pmap %p resident count underflow %ld %d", pmap,

sys/powerpc/aim/mmu_radix.c

881

pmap->pm_stats.resident_count, count));

sys/powerpc/aim/mmu_radix.c

882

pmap->pm_stats.resident_count -= count;

sys/powerpc/aim/mmu_radix.c

927

mmu_radix_ps_enabled(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

929

return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0);

sys/powerpc/aim/mmu_radix.c

933

pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e)

sys/powerpc/aim/mmu_radix.c

939

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

956

pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags)

sys/powerpc/aim/mmu_radix.c

965

if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL)

sys/powerpc/aim/mmu_radix.c

972

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

976

__func__, pmap, va, flags, origpte);

sys/powerpc/aim/mmu_radix.c

983

printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va,

sys/powerpc/aim/mmu_radix.c

986

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

987

if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL ||

sys/powerpc/aim/mmu_radix.c

989

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1012

mmu_booke_extract(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap.c

1016

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1017

pa = pte_vatopa(pmap, va);

sys/powerpc/booke/pmap.c

1018

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1238

mmu_booke_ps_enabled(pmap_t pmap __unused)

sys/powerpc/booke/pmap.c

1247

mmu_booke_pinit0(pmap_t pmap)

sys/powerpc/booke/pmap.c

1250

PMAP_LOCK_INIT(pmap);

sys/powerpc/booke/pmap.c

1251

mmu_booke_pinit(pmap);

sys/powerpc/booke/pmap.c

1252

PCPU_SET(curpmap, pmap);

sys/powerpc/booke/pmap.c

1261

mmu_booke_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/booke/pmap.c

1267

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1268

error = mmu_booke_enter_locked(pmap, va, m, prot, flags, psind);

sys/powerpc/booke/pmap.c

1269

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1275

mmu_booke_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/booke/pmap.c

1284

su = (pmap == kernel_pmap);

sys/powerpc/booke/pmap.c

1307

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/booke/pmap.c

1313

if (((pte = pte_find(pmap, va)) != NULL) &&

sys/powerpc/booke/pmap.c

1327

pmap->pm_stats.wired_count++;

sys/powerpc/booke/pmap.c

1332

pmap->pm_stats.wired_count--;

sys/powerpc/booke/pmap.c

1424

error = pte_enter(pmap, m, va, flags,

sys/powerpc/booke/pmap.c

1430

pmap->pm_stats.wired_count++;

sys/powerpc/booke/pmap.c

1437

if (sync && (su || pmap == PCPU_GET(curpmap))) {

sys/powerpc/booke/pmap.c

1458

mmu_booke_enter_object(pmap_t pmap, vm_offset_t start,

sys/powerpc/booke/pmap.c

1471

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1474

mmu_booke_enter_locked(pmap, va, m,

sys/powerpc/booke/pmap.c

1479

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1484

mmu_booke_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/booke/pmap.c

1489

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1490

mmu_booke_enter_locked(pmap, va, m,

sys/powerpc/booke/pmap.c

1493

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1503

mmu_booke_remove(pmap_t pmap, vm_offset_t va, vm_offset_t endva)

sys/powerpc/booke/pmap.c

1508

int su = (pmap == kernel_pmap);

sys/powerpc/booke/pmap.c

1522

if (PMAP_REMOVE_DONE(pmap)) {

sys/powerpc/booke/pmap.c

1527

hold_flag = PTBL_HOLD_FLAG(pmap);

sys/powerpc/booke/pmap.c

1531

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1533

pte = pte_find_next(pmap, &va);

sys/powerpc/booke/pmap.c

1538

pte_remove(pmap, va, hold_flag);

sys/powerpc/booke/pmap.c

1540

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1599

pmap_t pmap;

sys/powerpc/booke/pmap.c

1602

pmap = &td->td_proc->p_vmspace->vm_pmap;

sys/powerpc/booke/pmap.c

1605

__func__, td, td->td_proc->p_comm, td->td_proc->p_pid, pmap);

sys/powerpc/booke/pmap.c

1607

KASSERT((pmap != kernel_pmap), ("mmu_booke_activate: kernel_pmap!"));

sys/powerpc/booke/pmap.c

1612

CPU_SET_ATOMIC(cpuid, &pmap->pm_active);

sys/powerpc/booke/pmap.c

1613

PCPU_SET(curpmap, pmap);

sys/powerpc/booke/pmap.c

1615

if (pmap->pm_tid[cpuid] == TID_NONE)

sys/powerpc/booke/pmap.c

1616

tid_alloc(pmap);

sys/powerpc/booke/pmap.c

1619

mtspr(SPR_PID0, pmap->pm_tid[cpuid]);

sys/powerpc/booke/pmap.c

1627

pmap->pm_tid[PCPU_GET(cpuid)], td->td_proc->p_comm);

sys/powerpc/booke/pmap.c

1636

pmap_t pmap;

sys/powerpc/booke/pmap.c

1638

pmap = &td->td_proc->p_vmspace->vm_pmap;

sys/powerpc/booke/pmap.c

1641

__func__, td, td->td_proc->p_comm, td->td_proc->p_pid, pmap);

sys/powerpc/booke/pmap.c

1645

CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmap->pm_active);

sys/powerpc/booke/pmap.c

1667

mmu_booke_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/powerpc/booke/pmap.c

1675

mmu_booke_remove(pmap, sva, eva);

sys/powerpc/booke/pmap.c

1682

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1684

if ((pte = pte_find(pmap, va)) != NULL) {

sys/powerpc/booke/pmap.c

1703

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1754

mmu_booke_extract_and_hold(pmap_t pmap, vm_offset_t va,

sys/powerpc/booke/pmap.c

1762

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1763

pte = pte_find(pmap, va);

sys/powerpc/booke/pmap.c

1765

if (pmap == kernel_pmap)

sys/powerpc/booke/pmap.c

1776

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1833

mmu_booke_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/powerpc/booke/pmap.c

195

#define PMAP_REMOVE_DONE(pmap) \

sys/powerpc/booke/pmap.c

196

((pmap) != kernel_pmap && (pmap)->pm_stats.resident_count == 0)

sys/powerpc/booke/pmap.c

1970

mmu_booke_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/powerpc/booke/pmap.c

1975

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1977

if ((pte = pte_find(pmap, va)) != NULL &&

sys/powerpc/booke/pmap.c

1983

pmap->pm_stats.wired_count--;

sys/powerpc/booke/pmap.c

1986

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1997

mmu_booke_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/powerpc/booke/pmap.c

2009

if (pv->pv_pmap == pmap) {

sys/powerpc/booke/pmap.c

224

static tlbtid_t tid_alloc(struct pmap *);

sys/powerpc/booke/pmap.c

2350

mmu_booke_object_init_pt(pmap_t pmap, vm_offset_t addr,

sys/powerpc/booke/pmap.c

2363

mmu_booke_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/powerpc/booke/pmap.c

2462

tid_alloc(pmap_t pmap)

sys/powerpc/booke/pmap.c

2467

KASSERT((pmap != kernel_pmap), ("tid_alloc: kernel pmap"));

sys/powerpc/booke/pmap.c

2469

CTR2(KTR_PMAP, "%s: s (pmap = %p)", __func__, pmap);

sys/powerpc/booke/pmap.c

2488

tidbusy[thiscpu][tid] = pmap;

sys/powerpc/booke/pmap.c

2489

pmap->pm_tid[thiscpu] = tid;

sys/powerpc/booke/pmap.c

2932

pmap_track_page(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap.c

2943

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

2946

if ((pmap == pve->pv_pmap) && (va == pve->pv_va)) {

sys/powerpc/booke/pmap.c

2951

pv_insert(pmap, va, page);

sys/powerpc/booke/pmap.c

2953

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

357

static bool mmu_booke_ps_enabled(pmap_t pmap);

sys/powerpc/booke/pmap.c

567

pv_insert(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/powerpc/booke/pmap.c

579

pve->pv_pmap = pmap;

sys/powerpc/booke/pmap.c

583

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/booke/pmap.c

593

pv_remove(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/powerpc/booke/pmap.c

600

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/booke/pmap.c

605

if ((pmap == pve->pv_pmap) && (va == pve->pv_va)) {

sys/powerpc/booke/pmap_32.c

216

ptbl_free_pmap_ptbl(pmap_t pmap, pte_t *ptbl)

sys/powerpc/booke/pmap_32.c

222

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/booke/pmap_32.c

224

TAILQ_FOREACH(pbuf, &pmap->pm_ptbl_list, link)

sys/powerpc/booke/pmap_32.c

227

TAILQ_REMOVE(&pmap->pm_ptbl_list, pbuf, link);

sys/powerpc/booke/pmap_32.c

237

ptbl_alloc(pmap_t pmap, unsigned int pdir_idx, bool nosleep)

sys/powerpc/booke/pmap_32.c

246

CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,

sys/powerpc/booke/pmap_32.c

247

(pmap == kernel_pmap), pdir_idx);

sys/powerpc/booke/pmap_32.c

251

KASSERT((pmap->pm_pdir[pdir_idx] == NULL),

sys/powerpc/booke/pmap_32.c

266

ptbl_free_pmap_ptbl(pmap, ptbl);

sys/powerpc/booke/pmap_32.c

272

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap_32.c

276

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap_32.c

289

TAILQ_INSERT_TAIL(&pmap->pm_ptbl_list, pbuf, link);

sys/powerpc/booke/pmap_32.c

296

ptbl_free(pmap_t pmap, unsigned int pdir_idx)

sys/powerpc/booke/pmap_32.c

304

CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,

sys/powerpc/booke/pmap_32.c

305

(pmap == kernel_pmap), pdir_idx);

sys/powerpc/booke/pmap_32.c

310

ptbl = pmap->pm_pdir[pdir_idx];

sys/powerpc/booke/pmap_32.c

323

pmap->pm_pdir[pdir_idx] = NULL;

sys/powerpc/booke/pmap_32.c

337

ptbl_free_pmap_ptbl(pmap, ptbl);

sys/powerpc/booke/pmap_32.c

347

ptbl_unhold(pmap_t pmap, unsigned int pdir_idx)

sys/powerpc/booke/pmap_32.c

354

CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,

sys/powerpc/booke/pmap_32.c

355

(pmap == kernel_pmap), pdir_idx);

sys/powerpc/booke/pmap_32.c

359

KASSERT((pmap != kernel_pmap),

sys/powerpc/booke/pmap_32.c

362

ptbl = pmap->pm_pdir[pdir_idx];

sys/powerpc/booke/pmap_32.c

382

ptbl_free(pmap, pdir_idx);

sys/powerpc/booke/pmap_32.c

396

ptbl_hold(pmap_t pmap, unsigned int pdir_idx)

sys/powerpc/booke/pmap_32.c

403

CTR3(KTR_PMAP, "%s: pmap = %p pdir_idx = %d", __func__, pmap,

sys/powerpc/booke/pmap_32.c

408

KASSERT((pmap != kernel_pmap),

sys/powerpc/booke/pmap_32.c

411

ptbl = pmap->pm_pdir[pdir_idx];

sys/powerpc/booke/pmap_32.c

429

pte_remove(pmap_t pmap, vm_offset_t va, uint8_t flags)

sys/powerpc/booke/pmap_32.c

441

ptbl = pmap->pm_pdir[pdir_idx];

sys/powerpc/booke/pmap_32.c

450

pmap->pm_stats.wired_count--;

sys/powerpc/booke/pmap_32.c

463

pv_remove(pmap, va, m);

sys/powerpc/booke/pmap_32.c

464

} else if (pmap == kernel_pmap && m && m->md.pv_tracked) {

sys/powerpc/booke/pmap_32.c

470

pv_remove(pmap, va, m);

sys/powerpc/booke/pmap_32.c

484

pmap->pm_stats.resident_count--;

sys/powerpc/booke/pmap_32.c

488

return (ptbl_unhold(pmap, pdir_idx));

sys/powerpc/booke/pmap_32.c

499

pte_enter(pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags,

sys/powerpc/booke/pmap_32.c

507

pmap == kernel_pmap, pmap, va);

sys/powerpc/booke/pmap_32.c

510

ptbl = pmap->pm_pdir[pdir_idx];

sys/powerpc/booke/pmap_32.c

514

ptbl = ptbl_alloc(pmap, pdir_idx, nosleep);

sys/powerpc/booke/pmap_32.c

519

pmap->pm_pdir[pdir_idx] = ptbl;

sys/powerpc/booke/pmap_32.c

526

pte = &pmap->pm_pdir[pdir_idx][ptbl_idx];

sys/powerpc/booke/pmap_32.c

528

pte_remove(pmap, va, PTBL_HOLD);

sys/powerpc/booke/pmap_32.c

534

if (pmap != kernel_pmap)

sys/powerpc/booke/pmap_32.c

535

ptbl_hold(pmap, pdir_idx);

sys/powerpc/booke/pmap_32.c

547

pv_insert(pmap, va, m);

sys/powerpc/booke/pmap_32.c

550

pmap->pm_stats.resident_count++;

sys/powerpc/booke/pmap_32.c

568

pte_vatopa(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap_32.c

573

pte = pte_find(pmap, va);

sys/powerpc/booke/pmap_32.c

581

pte_find(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap_32.c

586

KASSERT((pmap != NULL), ("pte_find: invalid pmap"));

sys/powerpc/booke/pmap_32.c

588

if (pmap->pm_pdir[pdir_idx])

sys/powerpc/booke/pmap_32.c

589

return (&(pmap->pm_pdir[pdir_idx][ptbl_idx]));

sys/powerpc/booke/pmap_32.c

596

pte_find_next(pmap_t pmap, vm_offset_t *pva)

sys/powerpc/booke/pmap_32.c

603

KASSERT((pmap != NULL), ("pte_find: invalid pmap"));

sys/powerpc/booke/pmap_32.c

608

pdir = pmap->pm_pdir;

sys/powerpc/booke/pmap_32.c

688

mmu_booke_pinit(pmap_t pmap)

sys/powerpc/booke/pmap_32.c

692

CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap,

sys/powerpc/booke/pmap_32.c

695

KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap"));

sys/powerpc/booke/pmap_32.c

698

pmap->pm_tid[i] = TID_NONE;

sys/powerpc/booke/pmap_32.c

700

bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));

sys/powerpc/booke/pmap_32.c

701

pmap->pm_pdir = uma_zalloc(ptbl_root_zone, M_WAITOK);

sys/powerpc/booke/pmap_32.c

702

bzero(pmap->pm_pdir, sizeof(pte_t *) * PDIR_NENTRIES);

sys/powerpc/booke/pmap_32.c

703

TAILQ_INIT(&pmap->pm_ptbl_list);

sys/powerpc/booke/pmap_32.c

714

mmu_booke_release(pmap_t pmap)

sys/powerpc/booke/pmap_32.c

717

KASSERT(pmap->pm_stats.resident_count == 0,

sys/powerpc/booke/pmap_32.c

719

pmap->pm_stats.resident_count));

sys/powerpc/booke/pmap_32.c

720

uma_zfree(ptbl_root_zone, pmap->pm_pdir);

sys/powerpc/booke/pmap_32.c

729

pmap_t pmap;

sys/powerpc/booke/pmap_32.c

735

pmap = PCPU_GET(curpmap);

sys/powerpc/booke/pmap_32.c

736

active = (pm == kernel_pmap || pm == pmap) ? 1 : 0;

sys/powerpc/booke/pmap_32.c

760

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap_32.c

761

pte_enter(pmap, m, addr,

sys/powerpc/booke/pmap_32.c

765

pte_remove(pmap, addr, PTBL_UNHOLD);

sys/powerpc/booke/pmap_32.c

766

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap_64.c

134

static pte_t *ptbl_alloc(pmap_t pmap, vm_offset_t va,

sys/powerpc/booke/pmap_64.c

152

mmu_booke_alloc_page(pmap_t pmap, unsigned int idx, bool nosleep)

sys/powerpc/booke/pmap_64.c

162

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap_64.c

166

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap_64.c

181

pte_find(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap_64.c

187

KASSERT((pmap != NULL), ("pte_find: invalid pmap"));

sys/powerpc/booke/pmap_64.c

189

pdir_l1 = pmap->pm_root[PG_ROOT_IDX(va)];

sys/powerpc/booke/pmap_64.c

202

pte_find_next(pmap_t pmap, vm_offset_t *pva)

sys/powerpc/booke/pmap_64.c

209

KASSERT((pmap != NULL), ("pte_find: invalid pmap"));

sys/powerpc/booke/pmap_64.c

216

pm_root = pmap->pm_root;

sys/powerpc/booke/pmap_64.c

246

unhold_free_page(pmap_t pmap, vm_page_t m)

sys/powerpc/booke/pmap_64.c

258

get_pgtbl_page(pmap_t pmap, vm_offset_t *ptr_tbl, uint32_t index,

sys/powerpc/booke/pmap_64.c

265

KASSERT(page != 0 || pmap != kernel_pmap,

sys/powerpc/booke/pmap_64.c

268

page = mmu_booke_alloc_page(pmap, index, nosleep);

sys/powerpc/booke/pmap_64.c

291

ptbl_alloc(pmap_t pmap, vm_offset_t va, bool nosleep, bool *is_new)

sys/powerpc/booke/pmap_64.c

299

pdir_l1 = get_pgtbl_page(pmap, (vm_offset_t *)pmap->pm_root,

sys/powerpc/booke/pmap_64.c

303

pdir = get_pgtbl_page(pmap, (vm_offset_t *)pdir_l1, pdir_l1_idx,

sys/powerpc/booke/pmap_64.c

307

ptbl = get_pgtbl_page(pmap, (vm_offset_t *)pdir, pdir_idx,

sys/powerpc/booke/pmap_64.c

320

ptbl_unhold(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap_64.c

334

KASSERT((pmap != kernel_pmap),

sys/powerpc/booke/pmap_64.c

337

pdir_l1 = pmap->pm_root[pg_root_idx];

sys/powerpc/booke/pmap_64.c

344

if (!unhold_free_page(pmap, m))

sys/powerpc/booke/pmap_64.c

350

if (!unhold_free_page(pmap, m))

sys/powerpc/booke/pmap_64.c

356

if (!unhold_free_page(pmap, m))

sys/powerpc/booke/pmap_64.c

358

pmap->pm_root[pg_root_idx] = NULL;

sys/powerpc/booke/pmap_64.c

368

ptbl_hold(pmap_t pmap, pte_t *ptbl)

sys/powerpc/booke/pmap_64.c

372

KASSERT((pmap != kernel_pmap),

sys/powerpc/booke/pmap_64.c

385

pte_remove(pmap_t pmap, vm_offset_t va, u_int8_t flags)

sys/powerpc/booke/pmap_64.c

390

pte = pte_find(pmap, va);

sys/powerpc/booke/pmap_64.c

392

__func__, (uintmax_t)va, pmap));

sys/powerpc/booke/pmap_64.c

401

pmap->pm_stats.wired_count--;

sys/powerpc/booke/pmap_64.c

414

pv_remove(pmap, va, m);

sys/powerpc/booke/pmap_64.c

415

} else if (pmap == kernel_pmap && m && m->md.pv_tracked) {

sys/powerpc/booke/pmap_64.c

416

pv_remove(pmap, va, m);

sys/powerpc/booke/pmap_64.c

429

pmap->pm_stats.resident_count--;

sys/powerpc/booke/pmap_64.c

432

return (ptbl_unhold(pmap, va));

sys/powerpc/booke/pmap_64.c

441

pte_enter(pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags,

sys/powerpc/booke/pmap_64.c

449

ptbl = ptbl_alloc(pmap, va, nosleep, &is_new);

sys/powerpc/booke/pmap_64.c

463

pte_remove(pmap, va, PTBL_HOLD);

sys/powerpc/booke/pmap_64.c

469

if (pmap != kernel_pmap)

sys/powerpc/booke/pmap_64.c

470

ptbl_hold(pmap, ptbl);

sys/powerpc/booke/pmap_64.c

482

pv_insert(pmap, va, m);

sys/powerpc/booke/pmap_64.c

485

pmap->pm_stats.resident_count++;

sys/powerpc/booke/pmap_64.c

504

pte_vatopa(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap_64.c

509

pte = pte_find(pmap, va);

sys/powerpc/booke/pmap_64.c

590

mmu_booke_pinit(pmap_t pmap)

sys/powerpc/booke/pmap_64.c

594

CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap,

sys/powerpc/booke/pmap_64.c

597

KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap"));

sys/powerpc/booke/pmap_64.c

600

pmap->pm_tid[i] = TID_NONE;

sys/powerpc/booke/pmap_64.c

602

bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));

sys/powerpc/booke/pmap_64.c

603

pmap->pm_root = uma_zalloc(ptbl_root_zone, M_WAITOK);

sys/powerpc/booke/pmap_64.c

604

bzero(pmap->pm_root, sizeof(pte_t **) * PG_ROOT_NENTRIES);

sys/powerpc/booke/pmap_64.c

615

mmu_booke_release(pmap_t pmap)

sys/powerpc/booke/pmap_64.c

618

KASSERT(pmap->pm_stats.resident_count == 0,

sys/powerpc/booke/pmap_64.c

620

pmap->pm_stats.resident_count));

sys/powerpc/booke/pmap_64.c

627

KASSERT(pmap->pm_root[i] == 0,

sys/powerpc/booke/pmap_64.c

628

("Index %d on root page %p is non-zero!\n", i, pmap->pm_root));

sys/powerpc/booke/pmap_64.c

630

uma_zfree(ptbl_root_zone, pmap->pm_root);

sys/powerpc/include/pcb.h

122

extern struct pmap *curpm;

sys/powerpc/include/pcb.h

53

struct pmap *pcb_pm; /* pmap of our vmspace */

sys/powerpc/include/pcpu.h

37

struct pmap;

sys/powerpc/include/pcpu.h

43

struct pmap *pc_curpmap; /* current pmap */ \

sys/powerpc/include/pmap.h

100

typedef struct pmap *pmap_t;

sys/powerpc/include/pmap.h

174

struct pmap *pmap_phys;

sys/powerpc/include/pmap.h

284

extern struct pmap kernel_pmap_store;

sys/powerpc/include/pmap.h

289

#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)

sys/powerpc/include/pmap.h

290

#define PMAP_LOCK_ASSERT(pmap, type) \

sys/powerpc/include/pmap.h

291

mtx_assert(&(pmap)->pm_mtx, (type))

sys/powerpc/include/pmap.h

292

#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)

sys/powerpc/include/pmap.h

293

#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, \

sys/powerpc/include/pmap.h

294

(pmap == kernel_pmap) ? "kernelpmap" : \

sys/powerpc/include/pmap.h

296

#define PMAP_LOCKED(pmap) mtx_owned(&(pmap)->pm_mtx)

sys/powerpc/include/pmap.h

297

#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)

sys/powerpc/include/pmap.h

298

#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)

sys/powerpc/include/pmap.h

299

#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)

sys/powerpc/include/pmap.h

324

bool pmap_ps_enabled(pmap_t pmap);

sys/powerpc/include/pmap.h

325

int pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags);

sys/powerpc/include/pmap.h

327

#define pmap_map_delete(pmap, sva, eva) pmap_remove(pmap, sva, eva)

sys/powerpc/include/pmap.h

348

void pmap_track_page(pmap_t pmap, vm_offset_t va);

sys/powerpc/include/pmap.h

99

struct pmap;

sys/powerpc/include/pte.h

230

#define PTBL_HOLD_FLAG(pmap) (((pmap) == kernel_pmap) ? PTBL_HOLD : PTBL_UNHOLD)

sys/powerpc/include/slb.h

88

struct pmap;

sys/powerpc/include/slb.h

90

int handle_user_slb_spill(struct pmap *pm, vm_offset_t addr);

sys/powerpc/include/tlb.h

172

struct pmap;

sys/powerpc/powerpc/busdma_machdep.c

504

_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,

sys/powerpc/powerpc/busdma_machdep.c

528

if (pmap == kernel_pmap)

sys/powerpc/powerpc/busdma_machdep.c

531

paddr = pmap_extract(pmap, vaddr);

sys/powerpc/powerpc/busdma_machdep.c

609

pmap_t pmap,

sys/powerpc/powerpc/busdma_machdep.c

623

_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);

sys/powerpc/powerpc/busdma_machdep.c

637

if (pmap == kernel_pmap) {

sys/powerpc/powerpc/busdma_machdep.c

641

curaddr = pmap_extract(pmap, vaddr);

sys/powerpc/powerpc/genassym.c

119

ASSYM(PM_SR, offsetof(struct pmap, pm_sr));

sys/powerpc/powerpc/genassym.c

124

ASSYM(PM_ROOT, offsetof(struct pmap, pm_root));

sys/powerpc/powerpc/genassym.c

126

ASSYM(PM_PDIR, offsetof(struct pmap, pm_pdir));

sys/powerpc/powerpc/pmap_dispatch.c

246

pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)

sys/powerpc/powerpc/pmap_dispatch.c

264

pmap_active_cpus(pmap_t pmap, cpuset_t *res)

sys/powerpc/powerpc/pmap_dispatch.c

266

*res = pmap->pm_active;

sys/powerpc/powerpc/pmap_dispatch.c

70

struct pmap kernel_pmap_store;

sys/powerpc/powerpc/pmap_dispatch.c

83

SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");

sys/riscv/include/bus_dma.h

110

bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,

sys/riscv/include/bus_dma.h

116

return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs,

sys/riscv/include/bus_dma_impl.h

65

void *buf, bus_size_t buflen, struct pmap *pmap, int flags,

sys/riscv/include/pcpu.h

46

struct pmap *pc_curpmap; /* Currently active pmap */ \

sys/riscv/include/pmap.h

100

#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)

sys/riscv/include/pmap.h

101

#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \

sys/riscv/include/pmap.h

103

#define PMAP_OWNED(pmap) mtx_owned(&(pmap)->pm_mtx)

sys/riscv/include/pmap.h

104

#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)

sys/riscv/include/pmap.h

105

#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)

sys/riscv/include/pmap.h

106

#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)

sys/riscv/include/pmap.h

58

#define pmap_map_delete(pmap, sva, eva) pmap_remove(pmap, sva, eva)

sys/riscv/include/pmap.h

83

LIST_ENTRY(pmap) pm_list; /* List of all pmaps */

sys/riscv/include/pmap.h

88

typedef struct pmap *pmap_t;

sys/riscv/include/pmap.h

91

extern struct pmap kernel_pmap_store;

sys/riscv/include/pmap.h

95

#define PMAP_ASSERT_LOCKED(pmap) \

sys/riscv/include/pmap.h

96

mtx_assert(&(pmap)->pm_mtx, MA_OWNED)

sys/riscv/include/pmap.h

97

#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)

sys/riscv/include/pmap.h

98

#define PMAP_LOCK_ASSERT(pmap, type) \

sys/riscv/include/pmap.h

99

mtx_assert(&(pmap)->pm_mtx, (type))

sys/riscv/include/vmm.h

125

struct pmap;

sys/riscv/include/vmm.h

140

DECLARE_VMMOPS_FUNC(void *, init, (struct vm *vm, struct pmap *pmap));

sys/riscv/include/vmm.h

143

DECLARE_VMMOPS_FUNC(int, run, (void *vcpui, register_t pc, struct pmap *pmap,

sys/riscv/riscv/busdma_bounce.c

107

pmap_t pmap, void *buf, bus_size_t buflen, int flags);

sys/riscv/riscv/busdma_bounce.c

514

_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,

sys/riscv/riscv/busdma_bounce.c

539

if (pmap == kernel_pmap)

sys/riscv/riscv/busdma_bounce.c

542

paddr = pmap_extract(pmap, vaddr);

sys/riscv/riscv/busdma_bounce.c

629

bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,

sys/riscv/riscv/busdma_bounce.c

642

_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);

sys/riscv/riscv/busdma_bounce.c

659

if (pmap == kernel_pmap) {

sys/riscv/riscv/busdma_bounce.c

663

curaddr = pmap_extract(pmap, vaddr);

sys/riscv/riscv/genassym.c

63

ASSYM(PM_SATP, offsetof(struct pmap, pm_satp));

sys/riscv/riscv/pmap.c

1075

pmap_invalidate_page(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

1080

mask = pmap->pm_active;

sys/riscv/riscv/pmap.c

1090

pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/riscv/riscv/pmap.c

1095

mask = pmap->pm_active;

sys/riscv/riscv/pmap.c

1110

pmap_invalidate_all(pmap_t pmap)

sys/riscv/riscv/pmap.c

1115

mask = pmap->pm_active;

sys/riscv/riscv/pmap.c

1135

pmap_invalidate_page(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

1142

pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/riscv/riscv/pmap.c

1153

pmap_invalidate_all(pmap_t pmap)

sys/riscv/riscv/pmap.c

1167

pmap_extract(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

1178

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

1179

l2p = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

1191

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

1203

pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/riscv/riscv/pmap.c

1210

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

1211

l2p = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

1228

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

1437

pmap_ps_enabled(pmap_t pmap __unused)

sys/riscv/riscv/pmap.c

1480

pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/riscv/riscv/pmap.c

1484

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

1488

return (vm_radix_insert(&pmap->pm_root, mpte));

sys/riscv/riscv/pmap.c

1498

pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

1501

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

1502

return (vm_radix_remove(&pmap->pm_root, pmap_l2_pindex(va)));

sys/riscv/riscv/pmap.c

1512

pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/riscv/riscv/pmap.c

1519

_pmap_unwire_ptp(pmap, va, m, free);

sys/riscv/riscv/pmap.c

1527

_pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/riscv/riscv/pmap.c

1530

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

1533

l0 = pmap_l0(pmap, va);

sys/riscv/riscv/pmap.c

1537

l1 = pmap_l1(pmap, va);

sys/riscv/riscv/pmap.c

1539

pmap_distribute_l1(pmap, pmap_l1_index(va), 0);

sys/riscv/riscv/pmap.c

1542

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

1545

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

1550

l1 = pmap_l1(pmap, va);

sys/riscv/riscv/pmap.c

1552

pmap_unwire_ptp(pmap, va, pdpg, free);

sys/riscv/riscv/pmap.c

1557

l0 = pmap_l0(pmap, va);

sys/riscv/riscv/pmap.c

1559

pmap_unwire_ptp(pmap, va, pdpg, free);

sys/riscv/riscv/pmap.c

1561

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

1577

pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde,

sys/riscv/riscv/pmap.c

1586

return (pmap_unwire_ptp(pmap, va, mpte, free));

sys/riscv/riscv/pmap.c

1596

pmap_pinit0(pmap_t pmap)

sys/riscv/riscv/pmap.c

1598

PMAP_LOCK_INIT(pmap);

sys/riscv/riscv/pmap.c

1599

bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));

sys/riscv/riscv/pmap.c

1600

pmap->pm_stage = PM_STAGE1;

sys/riscv/riscv/pmap.c

1601

pmap->pm_top = kernel_pmap->pm_top;

sys/riscv/riscv/pmap.c

1602

pmap->pm_satp = pmap_satp_mode() |

sys/riscv/riscv/pmap.c

1603

(vtophys(pmap->pm_top) >> PAGE_SHIFT);

sys/riscv/riscv/pmap.c

1604

CPU_ZERO(&pmap->pm_active);

sys/riscv/riscv/pmap.c

1605

TAILQ_INIT(&pmap->pm_pvchunk);

sys/riscv/riscv/pmap.c

1606

vm_radix_init(&pmap->pm_root);

sys/riscv/riscv/pmap.c

1607

pmap_activate_boot(pmap);

sys/riscv/riscv/pmap.c

1611

pmap_pinit_stage(pmap_t pmap, enum pmap_stage stage)

sys/riscv/riscv/pmap.c

1629

pmap->pm_top = (pd_entry_t *)PHYS_TO_DMAP(topphys);

sys/riscv/riscv/pmap.c

1630

pmap->pm_satp = pmap_satp_mode() | (topphys >> PAGE_SHIFT);

sys/riscv/riscv/pmap.c

1631

pmap->pm_stage = stage;

sys/riscv/riscv/pmap.c

1633

bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));

sys/riscv/riscv/pmap.c

1635

CPU_ZERO(&pmap->pm_active);

sys/riscv/riscv/pmap.c

1647

LIST_INSERT_HEAD(&allpmaps, pmap, pm_list);

sys/riscv/riscv/pmap.c

1650

pmap->pm_top[i] = kernel_pmap->pm_top[i];

sys/riscv/riscv/pmap.c

1653

pmap->pm_top[i] = kernel_pmap->pm_top[i];

sys/riscv/riscv/pmap.c

1657

pmap->pm_top[i] = kernel_pmap->pm_top[i];

sys/riscv/riscv/pmap.c

1661

TAILQ_INIT(&pmap->pm_pvchunk);

sys/riscv/riscv/pmap.c

1662

vm_radix_init(&pmap->pm_root);

sys/riscv/riscv/pmap.c

1668

pmap_pinit(pmap_t pmap)

sys/riscv/riscv/pmap.c

1671

return (pmap_pinit_stage(pmap, PM_STAGE1));

sys/riscv/riscv/pmap.c

1686

_pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)

sys/riscv/riscv/pmap.c

1693

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

1702

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

1706

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

1732

l0 = &pmap->pm_top[l0index];

sys/riscv/riscv/pmap.c

1744

l1 = &pmap->pm_top[l1index];

sys/riscv/riscv/pmap.c

1747

l0 = &pmap->pm_top[l0index];

sys/riscv/riscv/pmap.c

1750

if (_pmap_alloc_l3(pmap,

sys/riscv/riscv/pmap.c

1767

pmap_distribute_l1(pmap, l1index, entry);

sys/riscv/riscv/pmap.c

1774

l1 = &pmap->pm_top[l1index];

sys/riscv/riscv/pmap.c

1777

if (_pmap_alloc_l3(pmap, NUL2E + l1index,

sys/riscv/riscv/pmap.c

1786

l0 = &pmap->pm_top[l0index];

sys/riscv/riscv/pmap.c

1789

if (_pmap_alloc_l3(pmap, NUL2E + l1index,

sys/riscv/riscv/pmap.c

1801

if (_pmap_alloc_l3(pmap,

sys/riscv/riscv/pmap.c

1821

pmap_resident_count_inc(pmap, 1);

sys/riscv/riscv/pmap.c

1832

pmap_alloc_l2(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)

sys/riscv/riscv/pmap.c

1839

l1 = pmap_l1(pmap, va);

sys/riscv/riscv/pmap.c

1850

l2pg = _pmap_alloc_l3(pmap, pindex, lockp);

sys/riscv/riscv/pmap.c

1858

pmap_alloc_l3(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)

sys/riscv/riscv/pmap.c

1872

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

1886

m = _pmap_alloc_l3(pmap, ptepindex, lockp);

sys/riscv/riscv/pmap.c

1903

pmap_release(pmap_t pmap)

sys/riscv/riscv/pmap.c

1909

KASSERT(pmap->pm_stats.resident_count == 0,

sys/riscv/riscv/pmap.c

1911

pmap->pm_stats.resident_count));

sys/riscv/riscv/pmap.c

1912

KASSERT(CPU_EMPTY(&pmap->pm_active),

sys/riscv/riscv/pmap.c

1913

("releasing active pmap %p", pmap));

sys/riscv/riscv/pmap.c

1915

if (pmap->pm_stage == PM_STAGE2)

sys/riscv/riscv/pmap.c

1920

LIST_REMOVE(pmap, pm_list);

sys/riscv/riscv/pmap.c

1925

npages = pmap->pm_stage == PM_STAGE2 ? 4 : 1;

sys/riscv/riscv/pmap.c

1926

m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_top));

sys/riscv/riscv/pmap.c

2095

free_pv_entry(pmap_t pmap, pv_entry_t pv)

sys/riscv/riscv/pmap.c

2101

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2112

if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {

sys/riscv/riscv/pmap.c

2113

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2114

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2118

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2149

get_pv_entry(pmap_t pmap, struct rwlock **lockp)

sys/riscv/riscv/pmap.c

2157

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2160

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/riscv/riscv/pmap.c

2173

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2174

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,

sys/riscv/riscv/pmap.c

2189

m = reclaim_pv_chunk(pmap, lockp);

sys/riscv/riscv/pmap.c

2197

pc->pc_pmap = pmap;

sys/riscv/riscv/pmap.c

2205

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2218

reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)

sys/riscv/riscv/pmap.c

222

static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,

sys/riscv/riscv/pmap.c

2227

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2239

TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {

sys/riscv/riscv/pmap.c

2251

m = reclaim_pv_chunk(pmap, lockp);

sys/riscv/riscv/pmap.c

226

LIST_HEAD(pmaplist, pmap);

sys/riscv/riscv/pmap.c

2260

pc->pc_pmap = pmap;

sys/riscv/riscv/pmap.c

2264

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2290

pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

2296

if (pmap == PV_PMAP(pv) && va == pv->pv_va) {

sys/riscv/riscv/pmap.c

2311

pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

2315

pv = pmap_pvh_remove(pvh, pmap, va);

sys/riscv/riscv/pmap.c

2318

free_pv_entry(pmap, pv);

sys/riscv/riscv/pmap.c

2326

pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/riscv/riscv/pmap.c

2332

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2334

if ((pv = get_pv_entry(pmap, NULL)) != NULL) {

sys/riscv/riscv/pmap.c

234

struct pmap kernel_pmap_store;

sys/riscv/riscv/pmap.c

2350

pmap_pv_demote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/riscv/riscv/pmap.c

2361

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2371

pv = pmap_pvh_remove(pvh, pmap, va);

sys/riscv/riscv/pmap.c

2380

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/riscv/riscv/pmap.c

2398

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2399

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2403

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2404

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2412

pmap_pv_promote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/riscv/riscv/pmap.c

2428

pv = pmap_pvh_remove(&m->md, pmap, va);

sys/riscv/riscv/pmap.c

2438

pmap_pvh_free(&m->md, pmap, va);

sys/riscv/riscv/pmap.c

2449

pmap_pv_insert_l2(pmap_t pmap, vm_offset_t va, pd_entry_t l2e, u_int flags,

sys/riscv/riscv/pmap.c

2456

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2458

if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?

sys/riscv/riscv/pmap.c

2471

pmap_remove_kernel_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va)

sys/riscv/riscv/pmap.c

2478

KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));

sys/riscv/riscv/pmap.c

2479

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2481

ml3 = pmap_remove_pt_page(pmap, va);

sys/riscv/riscv/pmap.c

2507

pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva,

sys/riscv/riscv/pmap.c

2515

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2528

pmap_invalidate_range(pmap, sva, sva + L2_SIZE);

sys/riscv/riscv/pmap.c

2530

pmap->pm_stats.wired_count -= L2_SIZE / PAGE_SIZE;

sys/riscv/riscv/pmap.c

2531

pmap_resident_count_dec(pmap, L2_SIZE / PAGE_SIZE);

sys/riscv/riscv/pmap.c

2535

pmap_pvh_free(pvh, pmap, sva);

sys/riscv/riscv/pmap.c

2548

if (pmap == kernel_pmap) {

sys/riscv/riscv/pmap.c

2549

pmap_remove_kernel_l2(pmap, l2, sva);

sys/riscv/riscv/pmap.c

2551

ml3 = pmap_remove_pt_page(pmap, sva);

sys/riscv/riscv/pmap.c

2555

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

2563

return (pmap_unuse_pt(pmap, sva, l1e, free));

sys/riscv/riscv/pmap.c

2570

pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va,

sys/riscv/riscv/pmap.c

2577

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2579

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

2581

pmap->pm_stats.wired_count -= 1;

sys/riscv/riscv/pmap.c

2582

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

2590

pmap_pvh_free(&m->md, pmap, va);

sys/riscv/riscv/pmap.c

2599

return (pmap_unuse_pt(pmap, va, l2e, free));

sys/riscv/riscv/pmap.c

2609

pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/riscv/riscv/pmap.c

2620

if (pmap->pm_stats.resident_count == 0)

sys/riscv/riscv/pmap.c

2626

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

2630

if (pmap->pm_stats.resident_count == 0)

sys/riscv/riscv/pmap.c

2634

l0 = pmap_l0(pmap, sva);

sys/riscv/riscv/pmap.c

2643

l1 = pmap_l1(pmap, sva);

sys/riscv/riscv/pmap.c

2665

(void)pmap_remove_l2(pmap, l2, sva,

sys/riscv/riscv/pmap.c

2668

} else if (!pmap_demote_l2_locked(pmap, l2, sva,

sys/riscv/riscv/pmap.c

2691

pmap_invalidate_range(pmap, va, sva);

sys/riscv/riscv/pmap.c

2698

if (pmap_remove_l3(pmap, l3, sva, l2e, &free, &lock)) {

sys/riscv/riscv/pmap.c

2704

pmap_invalidate_range(pmap, va, sva);

sys/riscv/riscv/pmap.c

2709

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

2731

pmap_t pmap;

sys/riscv/riscv/pmap.c

2745

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

2746

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

2748

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

2749

(void)pmap_demote_l2(pmap, l2, va);

sys/riscv/riscv/pmap.c

2750

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

2753

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

2754

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

2755

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

2756

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

2765

pmap_invalidate_page(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

2767

pmap->pm_stats.wired_count--;

sys/riscv/riscv/pmap.c

2776

pmap_unuse_pt(pmap, pv->pv_va, pmap_load(l2), &free);

sys/riscv/riscv/pmap.c

2779

free_pv_entry(pmap, pv);

sys/riscv/riscv/pmap.c

2780

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

2792

pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)

sys/riscv/riscv/pmap.c

2801

pmap_remove(pmap, sva, eva);

sys/riscv/riscv/pmap.c

2817

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

2820

l0 = pmap_l0(pmap, sva);

sys/riscv/riscv/pmap.c

2829

l1 = pmap_l1(pmap, sva);

sys/riscv/riscv/pmap.c

2866

pmap);

sys/riscv/riscv/pmap.c

2867

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

2872

if (!pmap_demote_l2(pmap, l2, sva)) {

sys/riscv/riscv/pmap.c

2902

pmap_invalidate_all(pmap);

sys/riscv/riscv/pmap.c

2905

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

2909

pmap_fault(pmap_t pmap, vm_offset_t va, vm_prot_t ftype)

sys/riscv/riscv/pmap.c

2918

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

2919

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

2931

if ((pmap != kernel_pmap && (oldpte & PTE_U) == 0) ||

sys/riscv/riscv/pmap.c

2951

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

2960

pmap_demote_l1(pmap_t pmap, pd_entry_t *l1, vm_offset_t va)

sys/riscv/riscv/pmap.c

2967

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2985

va, pmap);

sys/riscv/riscv/pmap.c

3012

va, pmap);

sys/riscv/riscv/pmap.c

3017

pmap_demote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va)

sys/riscv/riscv/pmap.c

3023

rv = pmap_demote_l2_locked(pmap, l2, va, &lock);

sys/riscv/riscv/pmap.c

3034

pmap_demote_l2_locked(pmap_t pmap, pd_entry_t *l2, vm_offset_t va,

sys/riscv/riscv/pmap.c

3044

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

3049

if ((oldl2 & PTE_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==

sys/riscv/riscv/pmap.c

3057

(void)pmap_remove_l2(pmap, l2, va & ~L2_OFFSET,

sys/riscv/riscv/pmap.c

3058

pmap_load(pmap_l1(pmap, va)), &free, lockp);

sys/riscv/riscv/pmap.c

3061

"failure for va %#lx in pmap %p", va, pmap);

sys/riscv/riscv/pmap.c

3067

pmap_resident_count_inc(pmap, 1);

sys/riscv/riscv/pmap.c

3107

reserve_pv_entries(pmap, Ln_ENTRIES - 1, lockp);

sys/riscv/riscv/pmap.c

3118

pmap_pv_demote_l2(pmap, va, PTE_TO_PHYS(oldl2), lockp);

sys/riscv/riscv/pmap.c

3122

va, pmap);

sys/riscv/riscv/pmap.c

3128

pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, vm_page_t ml3,

sys/riscv/riscv/pmap.c

3134

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

3135

if (!pmap_ps_enabled(pmap))

sys/riscv/riscv/pmap.c

314

static void free_pv_entry(pmap_t pmap, pv_entry_t pv);

sys/riscv/riscv/pmap.c

315

static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp);

sys/riscv/riscv/pmap.c

3151

va, pmap);

sys/riscv/riscv/pmap.c

317

static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);

sys/riscv/riscv/pmap.c

318

static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,

sys/riscv/riscv/pmap.c

3185

va, pmap);

sys/riscv/riscv/pmap.c

3198

va, pmap);

sys/riscv/riscv/pmap.c

320

static bool pmap_demote_l1(pmap_t pmap, pd_entry_t *l1, vm_offset_t va);

sys/riscv/riscv/pmap.c

321

static bool pmap_demote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va);

sys/riscv/riscv/pmap.c

322

static bool pmap_demote_l2_locked(pmap_t pmap, pd_entry_t *l2,

sys/riscv/riscv/pmap.c

3223

if (pmap_insert_pt_page(pmap, ml3, true, all_l3e_PTE_A != 0)) {

sys/riscv/riscv/pmap.c

3225

va, pmap);

sys/riscv/riscv/pmap.c

3231

pmap_pv_promote_l2(pmap, va, PTE_TO_PHYS(firstl3e), lockp);

sys/riscv/riscv/pmap.c

3237

pmap);

sys/riscv/riscv/pmap.c

324

static int pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2,

sys/riscv/riscv/pmap.c

3255

pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/riscv/riscv/pmap.c

326

static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,

sys/riscv/riscv/pmap.c

328

static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva,

sys/riscv/riscv/pmap.c

330

static bool pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,

sys/riscv/riscv/pmap.c

3306

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

3312

rv = pmap_enter_l2(pmap, va, new_l3, flags, m, &lock);

sys/riscv/riscv/pmap.c

3316

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

3318

((l2e & PTE_RWX) == 0 || pmap_demote_l2_locked(pmap, l2,

sys/riscv/riscv/pmap.c

3327

mpte = pmap_alloc_l3(pmap, va, nosleep ? NULL : &lock);

sys/riscv/riscv/pmap.c

333

static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex,

sys/riscv/riscv/pmap.c

3333

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

3336

l3 = pmap_l3(pmap, va);

sys/riscv/riscv/pmap.c

3357

pmap->pm_stats.wired_count++;

sys/riscv/riscv/pmap.c

336

static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/riscv/riscv/pmap.c

3360

pmap->pm_stats.wired_count--;

sys/riscv/riscv/pmap.c

3411

pv = pmap_pvh_remove(&om->md, pmap, va);

sys/riscv/riscv/pmap.c

3415

free_pv_entry(pmap, pv);

sys/riscv/riscv/pmap.c

3422

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

3429

pmap->pm_stats.wired_count++;

sys/riscv/riscv/pmap.c

3430

pmap_resident_count_inc(pmap, 1);

sys/riscv/riscv/pmap.c

3437

pv = get_pv_entry(pmap, &lock);

sys/riscv/riscv/pmap.c

3453

pmap_sync_icache(pmap, va, PAGE_SIZE);

sys/riscv/riscv/pmap.c

3460

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

3474

(void)pmap_promote_l2(pmap, l2, va, mpte, &lock);

sys/riscv/riscv/pmap.c

3482

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

3491

pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t l2pg)

sys/riscv/riscv/pmap.c

3496

if (pmap_unwire_ptp(pmap, va, l2pg, &free)) {

sys/riscv/riscv/pmap.c

3503

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

3515

pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/riscv/riscv/pmap.c

3521

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

3532

return (pmap_enter_l2(pmap, va, new_l2, PMAP_ENTER_NOSLEEP |

sys/riscv/riscv/pmap.c

3569

pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags,

sys/riscv/riscv/pmap.c

3578

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

3580

if ((l2pg = pmap_alloc_l2(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?

sys/riscv/riscv/pmap.c

3583

" for va %#lx in pmap %p", va, pmap);

sys/riscv/riscv/pmap.c

3597

" in pmap %p", va, pmap);

sys/riscv/riscv/pmap.c

3604

" for va %#lx in pmap %p", va, pmap);

sys/riscv/riscv/pmap.c

3610

(void)pmap_remove_l2(pmap, l2, va,

sys/riscv/riscv/pmap.c

3611

pmap_load(pmap_l1(pmap, va)), &free, lockp);

sys/riscv/riscv/pmap.c

3616

pmap_remove_l3(pmap, l3, sva, oldl2, &free,

sys/riscv/riscv/pmap.c

3627

if (pmap_insert_pt_page(pmap, mt, false, false))

sys/riscv/riscv/pmap.c

3638

if ((new_l2 & PTE_SW_WIRED) != 0 && pmap != kernel_pmap) {

sys/riscv/riscv/pmap.c

3641

pmap_abort_ptp(pmap, va, l2pg);

sys/riscv/riscv/pmap.c

3645

if (pmap_insert_pt_page(pmap, uwptpg, true, false)) {

sys/riscv/riscv/pmap.c

3648

pmap_abort_ptp(pmap, va, l2pg);

sys/riscv/riscv/pmap.c

3651

pmap_resident_count_inc(pmap, 1);

sys/riscv/riscv/pmap.c

3658

if (!pmap_pv_insert_l2(pmap, va, new_l2, flags, lockp)) {

sys/riscv/riscv/pmap.c

3659

pmap_abort_ptp(pmap, va, l2pg);

sys/riscv/riscv/pmap.c

3661

mt = pmap_remove_pt_page(pmap, va);

sys/riscv/riscv/pmap.c

3665

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

3672

" for va %#lx in pmap %p", va, pmap);

sys/riscv/riscv/pmap.c

3684

pmap->pm_stats.wired_count += L2_SIZE / PAGE_SIZE;

sys/riscv/riscv/pmap.c

3685

pmap->pm_stats.resident_count += L2_SIZE / PAGE_SIZE;

sys/riscv/riscv/pmap.c

3694

va, pmap);

sys/riscv/riscv/pmap.c

3712

pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,

sys/riscv/riscv/pmap.c

3729

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

3733

m->psind == 1 && pmap_ps_enabled(pmap) &&

sys/riscv/riscv/pmap.c

3734

((rv = pmap_enter_2mpage(pmap, va, m, prot, &lock)) ==

sys/riscv/riscv/pmap.c

3738

mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte,

sys/riscv/riscv/pmap.c

3746

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

3759

pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/riscv/riscv/pmap.c

3765

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

3766

(void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock);

sys/riscv/riscv/pmap.c

3770

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

3774

pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/riscv/riscv/pmap.c

3785

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

3788

CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va);

sys/riscv/riscv/pmap.c

3806

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

3824

mpte = _pmap_alloc_l3(pmap, l2pindex, NULL);

sys/riscv/riscv/pmap.c

3847

!pmap_try_insert_pv_entry(pmap, va, m, lockp)) {

sys/riscv/riscv/pmap.c

3850

if (pmap_unwire_ptp(pmap, va, mpte, &free))

sys/riscv/riscv/pmap.c

3859

pmap_resident_count_inc(pmap, 1);

sys/riscv/riscv/pmap.c

3875

pmap_sync_icache(pmap, va, PAGE_SIZE);

sys/riscv/riscv/pmap.c

3889

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

3895

if (pmap_promote_l2(pmap, l2, va, mpte, lockp))

sys/riscv/riscv/pmap.c

3909

pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,

sys/riscv/riscv/pmap.c

3928

pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/riscv/riscv/pmap.c

3937

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

3940

l0 = pmap_l0(pmap, sva);

sys/riscv/riscv/pmap.c

3949

l1 = pmap_l1(pmap, sva);

sys/riscv/riscv/pmap.c

3977

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

3983

if (!pmap_demote_l2(pmap, l2, sva))

sys/riscv/riscv/pmap.c

4004

pmap->pm_stats.wired_count--;

sys/riscv/riscv/pmap.c

4009

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

406

pmap_l0(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

411

return (&pmap->pm_top[pmap_l0_index(va)]);

sys/riscv/riscv/pmap.c

4129

pmap_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/riscv/riscv/pmap.c

4144

if (PV_PMAP(pv) == pmap) {

sys/riscv/riscv/pmap.c

4155

if (PV_PMAP(pv) == pmap) {

sys/riscv/riscv/pmap.c

4180

pmap_t pmap;

sys/riscv/riscv/pmap.c

4194

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4195

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4198

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4201

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4205

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4211

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4216

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4217

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4221

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4225

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4229

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4232

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4262

pmap_remove_pages_pv(pmap_t pmap, vm_page_t m, pv_entry_t pv,

sys/riscv/riscv/pmap.c

4269

pmap_resident_count_dec(pmap, Ln_ENTRIES);

sys/riscv/riscv/pmap.c

4279

mpte = pmap_remove_pt_page(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

428

pmap_l1(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

4283

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

4290

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

4319

pmap_remove_pages(pmap_t pmap)

sys/riscv/riscv/pmap.c

4337

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4338

TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {

sys/riscv/riscv/pmap.c

435

return (&pmap->pm_top[pmap_l1_index(va)]);

sys/riscv/riscv/pmap.c

4350

pte = pmap_l1(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

437

l0 = pmap_l0(pmap, va);

sys/riscv/riscv/pmap.c

4402

pmap_remove_pages_pv(pmap, m, pv, &free,

sys/riscv/riscv/pmap.c

4404

pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);

sys/riscv/riscv/pmap.c

4412

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

4418

pmap_invalidate_all(pmap);

sys/riscv/riscv/pmap.c

4420

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4432

pmap_t pmap;

sys/riscv/riscv/pmap.c

4448

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4449

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4452

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4455

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4459

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4464

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4471

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4472

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4476

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4480

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4484

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4486

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4525

pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/riscv/riscv/pmap.c

4535

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4536

l3 = pmap_l3(pmap, addr);

sys/riscv/riscv/pmap.c

4540

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4567

pmap_t pmap;

sys/riscv/riscv/pmap.c

4587

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4588

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

459

pmap_l2(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

4591

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4594

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4600

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

4602

(void)pmap_demote_l2_locked(pmap, l2, va, &lock);

sys/riscv/riscv/pmap.c

4606

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4609

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4610

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4614

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4617

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4622

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

463

l1 = pmap_l1(pmap, va);

sys/riscv/riscv/pmap.c

4634

pmap_invalidate_page(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4636

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4666

pmap_t pmap;

sys/riscv/riscv/pmap.c

4689

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4690

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4693

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4696

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4701

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

4731

(uintptr_t)pmap) & (Ln_ENTRIES - 1)) == 0 &&

sys/riscv/riscv/pmap.c

4734

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

4739

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4754

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4755

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4759

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4762

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4766

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4784

pmap_invalidate_page(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4789

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4811

pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)

sys/riscv/riscv/pmap.c

4823

pmap_t pmap;

sys/riscv/riscv/pmap.c

4844

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4845

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4848

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4851

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4856

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

4860

pmap_demote_l2_locked(pmap, l2, va, &lock) &&

sys/riscv/riscv/pmap.c

487

pmap_l3(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

4870

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

4872

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4875

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4876

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4880

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4883

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4887

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4893

pmap_invalidate_page(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4895

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

491

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

503

pmap_resident_count_inc(pmap_t pmap, int count)

sys/riscv/riscv/pmap.c

506

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

507

pmap->pm_stats.resident_count += count;

sys/riscv/riscv/pmap.c

511

pmap_resident_count_dec(pmap_t pmap, int count)

sys/riscv/riscv/pmap.c

5136

pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/riscv/riscv/pmap.c

514

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

5143

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

5144

l2 = pmap_l2(pmap, addr);

sys/riscv/riscv/pmap.c

515

KASSERT(pmap->pm_stats.resident_count >= count,

sys/riscv/riscv/pmap.c

5153

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

516

("pmap %p resident count underflow %ld %d", pmap,

sys/riscv/riscv/pmap.c

517

pmap->pm_stats.resident_count, count));

sys/riscv/riscv/pmap.c

5173

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

518

pmap->pm_stats.resident_count -= count;

sys/riscv/riscv/pmap.c

5180

pmap_t oldpmap, pmap;

sys/riscv/riscv/pmap.c

5184

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/riscv/riscv/pmap.c

5185

if (pmap == oldpmap)

sys/riscv/riscv/pmap.c

5187

csr_write(satp, pmap->pm_satp);

sys/riscv/riscv/pmap.c

5191

CPU_SET_ATOMIC(hart, &pmap->pm_active);

sys/riscv/riscv/pmap.c

5194

CPU_SET(hart, &pmap->pm_active);

sys/riscv/riscv/pmap.c

5197

PCPU_SET(curpmap, pmap);

sys/riscv/riscv/pmap.c

5212

pmap_activate_boot(pmap_t pmap)

sys/riscv/riscv/pmap.c

5218

CPU_SET_ATOMIC(hart, &pmap->pm_active);

sys/riscv/riscv/pmap.c

522

pmap_distribute_l1(struct pmap *pmap, vm_pindex_t l1index,

sys/riscv/riscv/pmap.c

5220

CPU_SET(hart, &pmap->pm_active);

sys/riscv/riscv/pmap.c

5222

PCPU_SET(curpmap, pmap);

sys/riscv/riscv/pmap.c

5226

pmap_active_cpus(pmap_t pmap, cpuset_t *res)

sys/riscv/riscv/pmap.c

5228

*res = pmap->pm_active;

sys/riscv/riscv/pmap.c

5232

pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz)

sys/riscv/riscv/pmap.c

525

struct pmap *user_pmap;

sys/riscv/riscv/pmap.c

534

if (pmap != kernel_pmap || pmap_mode != PMAP_MODE_SV39)

sys/riscv/riscv/pmap.c

5361

pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)

sys/riscv/riscv/pmap.c

5368

pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t **l1, pd_entry_t **l2,

sys/riscv/riscv/pmap.c

5374

l1p = pmap_l1(pmap, va);

sys/riscv/riscv/trap.c

264

if (VIRT_IS_VALID(va) && pmap_fault(map->pmap, va, ftype))

sys/riscv/vmm/vmm.c

582

pmap_t pmap;

sys/riscv/vmm/vmm.c

588

pmap = vmspace_pmap(vm_vmspace(vm));

sys/riscv/vmm/vmm.c

608

if (pmap_fault(pmap, addr, ftype))

sys/riscv/vmm/vmm.c

676

pmap_t pmap;

sys/riscv/vmm/vmm.c

693

pmap = vmspace_pmap(vm_vmspace(vm));

sys/riscv/vmm/vmm.c

704

error = vmmops_run(vcpu->cookie, vcpu->nextpc, pmap, &evinfo);

sys/riscv/vmm/vmm_riscv.c

124

vmmops_init(struct vm *vm, pmap_t pmap)

sys/riscv/vmm/vmm_riscv.c

241

riscv_vmm_pinit(pmap_t pmap)

sys/riscv/vmm/vmm_riscv.c

244

dprintf("%s: pmap %p\n", __func__, pmap);

sys/riscv/vmm/vmm_riscv.c

246

pmap_pinit_stage(pmap, PM_STAGE2);

sys/riscv/vmm/vmm_riscv.c

449

pmap_t pmap)

sys/riscv/vmm/vmm_riscv.c

603

vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)

sys/riscv/vmm/vmm_riscv.c

630

csr_write(hgatp, pmap->pm_satp);

sys/riscv/vmm/vmm_riscv.c

711

handled = riscv_handle_world_switch(hypctx, vme, pmap);

sys/security/mac/mac_process.c

364

pmap_protect(map->pmap, vme->start, vme->end,

sys/sys/_pv_entry.h

41

struct pmap;

sys/sys/_pv_entry.h

92

struct pmap *pc_pmap; \

sys/sys/bus_dma.h

117

struct pmap;

sys/sys/bus_dma_internal.h

43

void *buf, bus_size_t buflen, struct pmap *pmap,

sys/vm/pmap.h

120

void pmap_active_cpus(pmap_t pmap, cpuset_t *res);

sys/vm/pmap.h

121

void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/vm/pmap.h

130

int pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/vm/pmap.h

132

void pmap_enter_object(pmap_t pmap, vm_offset_t start,

sys/vm/pmap.h

134

void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/vm/pmap.h

136

vm_paddr_t pmap_extract(pmap_t pmap, vm_offset_t va);

sys/vm/pmap.h

137

vm_page_t pmap_extract_and_hold(pmap_t pmap, vm_offset_t va,

sys/vm/pmap.h

142

bool pmap_is_prefaultable(pmap_t pmap, vm_offset_t va);

sys/vm/pmap.h

146

int pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap);

sys/vm/pmap.h

147

void pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,

sys/vm/pmap.h

149

bool pmap_page_exists_quick(pmap_t pmap, vm_page_t m);

sys/vm/pmap.h

166

void pmap_unwire(pmap_t pmap, vm_offset_t start, vm_offset_t end);

sys/vm/vm_extern.h

114

typedef int (*pmap_pinit_t)(struct pmap *pmap);

sys/vm/vm_extern.h

35

struct pmap;

sys/vm/vm_fault.c

1933

pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot,

sys/vm/vm_fault.c

2027

pmap_advise(fs->map->pmap, start, end, MADV_DONTNEED);

sys/vm/vm_fault.c

2069

pmap_t pmap;

sys/vm/vm_fault.c

2078

pmap = fs->map->pmap;

sys/vm/vm_fault.c

2079

if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace))

sys/vm/vm_fault.c

2117

if (!pmap_is_prefaultable(pmap, addr))

sys/vm/vm_fault.c

2142

pmap_enter_quick(pmap, addr, m, prot);

sys/vm/vm_fault.c

2194

*mp = pmap_extract_and_hold(map->pmap, va, prot);

sys/vm/vm_fault.c

2469

pmap_enter(dst_map->pmap, vaddr, dst_m, prot,

sys/vm/vm_fault.c

421

!pmap_ps_enabled(fs->map->pmap)) {

sys/vm/vm_fault.c

440

if (pmap_enter(fs->map->pmap, vaddr, m_map, fs->prot, fs->fault_type |

sys/vm/vm_fault.c

606

rv = pmap_enter(fs->map->pmap, vaddr, m, fs->prot,

sys/vm/vm_fault.c

655

!pmap_ps_enabled(fs->map->pmap)))

sys/vm/vm_fault.c

677

rv = pmap_enter(fs->map->pmap, vaddr, m,

sys/vm/vm_fault.c

694

rv = pmap_enter(fs->map->pmap, vaddr + ptoa(i),

sys/vm/vm_glue.c

273

pmap_sync_icache(map->pmap, va, sz);

sys/vm/vm_kern.c

975

pmap_active_cpus(pmap_t pmap, cpuset_t *res)

sys/vm/vm_kern.c

991

if (pmap == vmspace_pmap(vm))

sys/vm/vm_map.c

129

static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min,

sys/vm/vm_map.c

2715

pmap_object_init_pt(map->pmap, addr, object, pindex,

sys/vm/vm_map.c

2773

pmap_enter_object(map->pmap, start, addr +

sys/vm/vm_map.c

2779

pmap_enter_object(map->pmap, start, addr + ptoa(psize),

sys/vm/vm_map.c

3024

pmap_protect(map->pmap, entry->start,

sys/vm/vm_map.c

3206

pmap_advise(map->pmap, useStart, useEnd,

sys/vm/vm_map.c

326

KASSERT(vm->vm_map.pmap == NULL, ("vm_map.pmap must be NULL"));

sys/vm/vm_map.c

3530

pmap_unwire(map->pmap, entry->start, failed_addr);

sys/vm/vm_map.c

381

vm->vm_map.pmap = NULL;

sys/vm/vm_map.c

3861

pmap_remove(map->pmap, start, end);

sys/vm/vm_map.c

3925

pmap_unwire(map->pmap, entry->start, entry->end);

sys/vm/vm_map.c

4091

pmap_map_delete(map->pmap, entry->start, entry->end);

sys/vm/vm_map.c

4248

pmap_protect(src_map->pmap,

sys/vm/vm_map.c

4296

pmap_copy(dst_map->pmap, src_map->pmap,

sys/vm/vm_map.c

4392

error = pmap_vmspace_copy(new_map->pmap, old_map->pmap);

sys/vm/vm_map.c

4508

pmap_copy(new_map->pmap, old_map->pmap,

sys/vm/vm_map.c

4797

if (ptoa(pmap_wired_count(map->pmap)) + grow_amount > lmemlim) {

sys/vm/vm_map.c

4805

ptoa(pmap_wired_count(map->pmap)) + grow_amount)) {

sys/vm/vm_map.c

4894

ptoa(pmap_wired_count(map->pmap)));

sys/vm/vm_map.c

5279

return (map->pmap);

sys/vm/vm_map.c

5368

(void *)map->pmap, map->nentries, map->timestamp);

sys/vm/vm_map.c

886

_vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max)

sys/vm/vm_map.c

890

map->pmap = pmap;

sys/vm/vm_map.c

905

vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max)

sys/vm/vm_map.c

907

_vm_map_init(map, pmap, min, max);

sys/vm/vm_map.c

912

vm_map_init_system(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max)

sys/vm/vm_map.c

914

_vm_map_init(map, pmap, min, max);

sys/vm/vm_map.h

214

pmap_t pmap; /* (c) Physical map */

sys/vm/vm_map.h

264

return (map->pmap);

sys/vm/vm_map.h

316

struct pmap vm_pmap; /* private physical map */

sys/vm/vm_mmap.c

1104

nsize = ptoa(npages + pmap_wired_count(map->pmap));

sys/vm/vm_mmap.c

1125

ptoa(pmap_wired_count(map->pmap)));

sys/vm/vm_mmap.c

1204

ptoa(pmap_wired_count(map->pmap)));

sys/vm/vm_mmap.c

1286

ptoa(pmap_wired_count(map->pmap)));

sys/vm/vm_mmap.c

1546

if (ptoa(pmap_wired_count(map->pmap)) + size >

sys/vm/vm_mmap.c

1553

ptoa(pmap_wired_count(map->pmap)) + size);

sys/vm/vm_mmap.c

833

pmap_t pmap;

sys/vm/vm_mmap.c

854

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/vm/vm_mmap.c

910

mincoreinfo = pmap_mincore(pmap, addr, &pa);

sys/vm/vm_mmap.c

936

if (pa != pmap_extract(pmap, addr))

sys/vm/vm_swapout.c

139

vm_swapout_object_deactivate_page(pmap_t pmap, vm_page_t m, bool unmap)

sys/vm/vm_swapout.c

149

if (vm_page_wired(m) || !pmap_page_exists_quick(pmap, m)) {

sys/vm/vm_swapout.c

171

vm_swapout_object_deactivate(pmap_t pmap, vm_object_t first_object,

sys/vm/vm_swapout.c

183

if (pmap_resident_count(pmap) <= desired)

sys/vm/vm_swapout.c

199

if (pmap_resident_count(pmap) <= desired)

sys/vm/vm_swapout.c

203

vm_swapout_object_deactivate_page(pmap, m, unmap);

sys/vm/vm_swapout.c

257

vm_swapout_object_deactivate(map->pmap, bigobj, desired);

sys/vm/vm_swapout.c

271

vm_swapout_object_deactivate(map->pmap, obj,

sys/vm/vm_unix.c

130

if (ptoa(pmap_wired_count(map->pmap)) +

sys/vm/vm_unix.c

160

ptoa(pmap_wired_count(map->pmap)) +

sys/vm/vm_unix.c

199

ptoa(pmap_wired_count(map->pmap)));

sys/vm/vm_unix.c

222

ptoa(pmap_wired_count(map->pmap)));

sys/x86/include/bus_dma.h

160

bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,

sys/x86/include/bus_dma.h

166

return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs,

sys/x86/include/busdma_impl.h

70

void *buf, bus_size_t buflen, struct pmap *pmap, int flags,

sys/x86/include/x86_smp.h

114

void smp_masked_invlpg(cpuset_t mask, vm_offset_t addr, struct pmap *pmap,

sys/x86/include/x86_smp.h

117

vm_offset_t endva, struct pmap *pmap, smp_invl_cb_t curcpu_cb);

sys/x86/include/x86_smp.h

118

void smp_masked_invltlb(cpuset_t mask, struct pmap *pmap,

sys/x86/include/x86_smp.h

121

void smp_masked_invlpg(vm_offset_t addr, struct pmap *pmap,

sys/x86/include/x86_smp.h

124

struct pmap *pmap, smp_invl_cb_t curcpu_cb);

sys/x86/include/x86_smp.h

125

void smp_masked_invltlb(struct pmap *pmap, smp_invl_cb_t curcpu_cb);

sys/x86/include/x86_smp.h

22

struct pmap;

sys/x86/include/x86_smp.h

41

extern struct pmap *smp_tlb_pmap;

sys/x86/include/x86_smp.h

84

typedef void (*smp_invl_cb_t)(struct pmap *, vm_offset_t addr1,

sys/x86/x86/busdma_bounce.c

104

pmap_t pmap, void *buf, bus_size_t buflen, int flags);

sys/x86/x86/busdma_bounce.c

543

_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,

sys/x86/x86/busdma_bounce.c

568

if (pmap == kernel_pmap)

sys/x86/x86/busdma_bounce.c

571

paddr = pmap_extract(pmap, vaddr);

sys/x86/x86/busdma_bounce.c

685

bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,

sys/x86/x86/busdma_bounce.c

700

_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);

sys/x86/x86/busdma_bounce.c

713

if (pmap == kernel_pmap) {

sys/x86/x86/busdma_bounce.c

717

curaddr = pmap_extract(pmap, vaddr);

usr.sbin/rpcbind/pmap_svc.c

164

struct pmap reg;

usr.sbin/rpcbind/pmap_svc.c

258

struct pmap reg;

usr.sbin/rpcbind/rpcb_svc_com.c

1403

struct pmap pmap;

usr.sbin/rpcbind/rpcb_svc_com.c

1409

pmap.pm_prot = IPPROTO_UDP;

usr.sbin/rpcbind/rpcb_svc_com.c

1412

pmap.pm_prot = IPPROTO_TCP;

usr.sbin/rpcbind/rpcb_svc_com.c

1421

pmap.pm_port = ((p1 & 0xff) << 8) + (p2 & 0xff);

usr.sbin/rpcbind/rpcb_svc_com.c

1422

pmap.pm_prog = arg->r_prog;

usr.sbin/rpcbind/rpcb_svc_com.c

1423

pmap.pm_vers = arg->r_vers;

usr.sbin/rpcbind/rpcb_svc_com.c

1432

pml->pml_map = pmap;

usr.sbin/rpcbind/security.c

67

struct pmap *pmap;

usr.sbin/rpcbind/security.c

81

pmap = (struct pmap *)args;

usr.sbin/rpcbind/security.c

82

prog = pmap->pm_prog;

usr.sbin/ypbind/yp_ping.c

112

struct pmap parms;