crypto/krb5/src/include/gssrpc/pmap_prot.h

93

extern bool_t xdr_pmap(XDR *, struct pmap *);

crypto/krb5/src/include/gssrpc/pmap_prot.h

96

struct pmap pml_map;

crypto/krb5/src/lib/rpc/pmap_clnt.c

152

struct pmap parms;

crypto/krb5/src/lib/rpc/pmap_clnt.c

71

struct pmap parms;

crypto/krb5/src/lib/rpc/pmap_getport.c

72

struct pmap parms;

crypto/krb5/src/lib/rpc/pmap_prot.c

49

xdr_pmap(XDR *xdrs, struct pmap *regs)

crypto/openssl/crypto/x509/v3_pmaps.c

101

sk_POLICY_MAPPING_push(pmaps, pmap); /* no failure as it was reserved */

crypto/openssl/crypto/x509/v3_pmaps.c

50

POLICY_MAPPING *pmap;

crypto/openssl/crypto/x509/v3_pmaps.c

56

pmap = sk_POLICY_MAPPING_value(pmaps, i);

crypto/openssl/crypto/x509/v3_pmaps.c

57

i2t_ASN1_OBJECT(obj_tmp1, 80, pmap->issuerDomainPolicy);

crypto/openssl/crypto/x509/v3_pmaps.c

58

i2t_ASN1_OBJECT(obj_tmp2, 80, pmap->subjectDomainPolicy);

crypto/openssl/crypto/x509/v3_pmaps.c

67

POLICY_MAPPING *pmap = NULL;

crypto/openssl/crypto/x509/v3_pmaps.c

93

pmap = POLICY_MAPPING_new();

crypto/openssl/crypto/x509/v3_pmaps.c

94

if (pmap == NULL) {

crypto/openssl/crypto/x509/v3_pmaps.c

98

pmap->issuerDomainPolicy = obj1;

crypto/openssl/crypto/x509/v3_pmaps.c

99

pmap->subjectDomainPolicy = obj2;

include/rpc/pmap_prot.h

94

struct pmap pml_map;

include/rpc/pmap_prot.h

99

extern bool_t xdr_pmap(XDR *, struct pmap *);

lib/libc/rpc/pmap_getport.c

70

struct pmap parms;

lib/libc/rpc/pmap_prot.c

50

xdr_pmap(XDR *xdrs, struct pmap *regs)

lib/libc/rpc/rpcb_clnt.c

733

struct pmap pmapparms;

sys/amd64/amd64/genassym.c

70

ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));

sys/amd64/amd64/mp_machdep.c

635

smp_targeted_tlb_shootdown_native(pmap_t pmap, vm_offset_t addr1,

sys/amd64/amd64/mp_machdep.c

656

CPU_COPY(pmap_invalidate_cpu_mask(pmap), &mask);

sys/amd64/amd64/mp_machdep.c

675

PCPU_SET(smp_tlb_pmap, pmap);

sys/amd64/amd64/mp_machdep.c

701

curcpu_cb(pmap, addr1, addr2);

sys/amd64/amd64/mp_machdep.c

720

curcpu_cb(pmap, addr1, addr2);

sys/amd64/amd64/mp_machdep.c

726

smp_masked_invltlb(pmap_t pmap, smp_invl_cb_t curcpu_cb)

sys/amd64/amd64/mp_machdep.c

728

if (invlpgb_works && pmap == kernel_pmap) {

sys/amd64/amd64/mp_machdep.c

742

smp_targeted_tlb_shootdown(pmap, 0, 0, curcpu_cb, invl_op_tlb);

sys/amd64/amd64/mp_machdep.c

749

smp_masked_invlpg(vm_offset_t addr, pmap_t pmap, smp_invl_cb_t curcpu_cb)

sys/amd64/amd64/mp_machdep.c

751

if (invlpgb_works && pmap == kernel_pmap) {

sys/amd64/amd64/mp_machdep.c

758

smp_targeted_tlb_shootdown(pmap, addr, 0, curcpu_cb, invl_op_pg);

sys/amd64/amd64/mp_machdep.c

765

smp_masked_invlpg_range(vm_offset_t addr1, vm_offset_t addr2, pmap_t pmap,

sys/amd64/amd64/mp_machdep.c

768

if (invlpgb_works && pmap == kernel_pmap) {

sys/amd64/amd64/mp_machdep.c

788

smp_targeted_tlb_shootdown(pmap, addr1, addr2, curcpu_cb,

sys/amd64/amd64/pmap.c

10021

pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/amd64/amd64/pmap.c

10029

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

10030

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

10031

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

10032

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

10034

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

10038

pdpe = pmap_pdpe(pmap, addr);

sys/amd64/amd64/pmap.c

10048

pdep = pmap_pde(pmap, addr);

sys/amd64/amd64/pmap.c

10077

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

10082

pmap_pcid_alloc(pmap_t pmap, struct pmap_pcid *pcidp)

sys/amd64/amd64/pmap.c

10113

pmap_pcid_alloc_checked(pmap_t pmap, struct pmap_pcid *pcidp)

sys/amd64/amd64/pmap.c

10117

cached = pmap_pcid_alloc(pmap, pcidp);

sys/amd64/amd64/pmap.c

10119

("pmap %p cpu %d pcid %#x", pmap, PCPU_GET(cpuid), pcidp->pm_pcid));

sys/amd64/amd64/pmap.c

10120

KASSERT(pcidp->pm_pcid != PMAP_PCID_KERN || pmap == kernel_pmap,

sys/amd64/amd64/pmap.c

10122

pmap, PCPU_GET(cpuid), pcidp->pm_pcid));

sys/amd64/amd64/pmap.c

10127

pmap_activate_sw_pti_post(struct thread *td, pmap_t pmap)

sys/amd64/amd64/pmap.c

10130

PCPU_GET(tssp)->tss_rsp0 = pmap->pm_ucr3 != PMAP_NO_CR3 ?

sys/amd64/amd64/pmap.c

10135

pmap_activate_sw_pcid_pti(struct thread *td, pmap_t pmap, u_int cpuid)

sys/amd64/amd64/pmap.c

10153

pcidp = zpcpu_get_cpu(pmap->pm_pcidp, cpuid);

sys/amd64/amd64/pmap.c

10154

cached = pmap_pcid_alloc_checked(pmap, pcidp);

sys/amd64/amd64/pmap.c

10156

if ((cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3)

sys/amd64/amd64/pmap.c

10157

load_cr3(pmap->pm_cr3 | pcidp->pm_pcid);

sys/amd64/amd64/pmap.c

10158

PCPU_SET(curpmap, pmap);

sys/amd64/amd64/pmap.c

10159

kcr3 = pmap->pm_cr3 | pcidp->pm_pcid;

sys/amd64/amd64/pmap.c

10160

ucr3 = pmap->pm_ucr3 | pcidp->pm_pcid | PMAP_PCID_USER_PT;

sys/amd64/amd64/pmap.c

10162

if (!cached && pmap->pm_ucr3 != PMAP_NO_CR3)

sys/amd64/amd64/pmap.c

10170

pmap_activate_sw_pti_post(td, pmap);

sys/amd64/amd64/pmap.c

10174

pmap_activate_sw_pcid_nopti(struct thread *td __unused, pmap_t pmap,

sys/amd64/amd64/pmap.c

10183

pcidp = zpcpu_get_cpu(pmap->pm_pcidp, cpuid);

sys/amd64/amd64/pmap.c

10184

cached = pmap_pcid_alloc_checked(pmap, pcidp);

sys/amd64/amd64/pmap.c

10186

if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3)

sys/amd64/amd64/pmap.c

10187

load_cr3(pmap->pm_cr3 | pcidp->pm_pcid | cached);

sys/amd64/amd64/pmap.c

10188

PCPU_SET(curpmap, pmap);

sys/amd64/amd64/pmap.c

10194

pmap_activate_sw_nopcid_nopti(struct thread *td __unused, pmap_t pmap,

sys/amd64/amd64/pmap.c

10198

load_cr3(pmap->pm_cr3);

sys/amd64/amd64/pmap.c

10199

PCPU_SET(curpmap, pmap);

sys/amd64/amd64/pmap.c

10203

pmap_activate_sw_nopcid_pti(struct thread *td, pmap_t pmap,

sys/amd64/amd64/pmap.c

10207

pmap_activate_sw_nopcid_nopti(td, pmap, cpuid);

sys/amd64/amd64/pmap.c

10208

PCPU_SET(kcr3, pmap->pm_cr3);

sys/amd64/amd64/pmap.c

10209

PCPU_SET(ucr3, pmap->pm_ucr3);

sys/amd64/amd64/pmap.c

10210

pmap_activate_sw_pti_post(td, pmap);

sys/amd64/amd64/pmap.c

10230

pmap_t oldpmap, pmap;

sys/amd64/amd64/pmap.c

10234

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/amd64/amd64/pmap.c

10235

if (oldpmap == pmap) {

sys/amd64/amd64/pmap.c

10241

CPU_SET_ATOMIC(cpuid, &pmap->pm_active);

sys/amd64/amd64/pmap.c

10242

pmap_activate_sw_mode(td, pmap, cpuid);

sys/amd64/amd64/pmap.c

10272

pmap_activate_boot(pmap_t pmap)

sys/amd64/amd64/pmap.c

10281

MPASS(pmap != kernel_pmap);

sys/amd64/amd64/pmap.c

10284

CPU_SET_ATOMIC(cpuid, &pmap->pm_active);

sys/amd64/amd64/pmap.c

10285

PCPU_SET(curpmap, pmap);

sys/amd64/amd64/pmap.c

10287

kcr3 = pmap->pm_cr3;

sys/amd64/amd64/pmap.c

10289

kcr3 |= pmap_get_pcid(pmap) | CR3_PCID_SAVE;

sys/amd64/amd64/pmap.c

10298

pmap_active_cpus(pmap_t pmap, cpuset_t *res)

sys/amd64/amd64/pmap.c

10300

*res = pmap->pm_active;

sys/amd64/amd64/pmap.c

10351

pmap_emulate_accessed_dirty(pmap_t pmap, vm_offset_t va, int ftype)

sys/amd64/amd64/pmap.c

10364

if (!pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

10367

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

10368

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

10369

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

10370

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

10374

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

10376

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

10424

pmap_promote_pde(pmap, pde, va, mpte, &lock)) {

sys/amd64/amd64/pmap.c

10441

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

10446

pmap_get_mapping(pmap_t pmap, vm_offset_t va, uint64_t *ptr, int *num)

sys/amd64/amd64/pmap.c

10455

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

10456

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

10458

pml4 = pmap_pml4e(pmap, va);

sys/amd64/amd64/pmap.c

10479

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

11426

pmap_pkru_assign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, u_int keyidx,

sys/amd64/amd64/pmap.c

11432

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11433

MPASS(pmap->pm_type == PT_X86);

sys/amd64/amd64/pmap.c

11436

!rangeset_check_empty(&pmap->pm_pkru, sva, eva))

sys/amd64/amd64/pmap.c

11443

error = rangeset_insert(&pmap->pm_pkru, sva, eva, ppr);

sys/amd64/amd64/pmap.c

11450

pmap_pkru_deassign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

11453

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11454

MPASS(pmap->pm_type == PT_X86);

sys/amd64/amd64/pmap.c

11456

return (rangeset_remove(&pmap->pm_pkru, sva, eva));

sys/amd64/amd64/pmap.c

11460

pmap_pkru_deassign_all(pmap_t pmap)

sys/amd64/amd64/pmap.c

11463

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11464

if (pmap->pm_type == PT_X86 &&

sys/amd64/amd64/pmap.c

11466

rangeset_remove_all(&pmap->pm_pkru);

sys/amd64/amd64/pmap.c

11475

pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte)

sys/amd64/amd64/pmap.c

11481

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11482

KASSERT(pmap->pm_type != PT_X86 || (*pte & X86_PG_PKU_MASK) == 0,

sys/amd64/amd64/pmap.c

11484

if (pmap->pm_type != PT_X86 ||

sys/amd64/amd64/pmap.c

11489

ppr = rangeset_containing(&pmap->pm_pkru, sva);

sys/amd64/amd64/pmap.c

11491

return (rangeset_empty(&pmap->pm_pkru, sva, eva));

sys/amd64/amd64/pmap.c

11494

if ((ppr = rangeset_beginning(&pmap->pm_pkru, va)) == NULL ||

sys/amd64/amd64/pmap.c

11503

pmap_pkru_get(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

11507

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11508

if (pmap->pm_type != PT_X86 ||

sys/amd64/amd64/pmap.c

11512

ppr = rangeset_containing(&pmap->pm_pkru, va);

sys/amd64/amd64/pmap.c

11528

pmap_pkru_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

11531

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11532

if (pmap->pm_type == PT_X86 &&

sys/amd64/amd64/pmap.c

11534

rangeset_remove_pred(&pmap->pm_pkru, sva, eva,

sys/amd64/amd64/pmap.c

11554

pmap_pkru_update_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/amd64/pmap.c

11564

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

11565

MPASS(pmap->pm_type == PT_X86);

sys/amd64/amd64/pmap.c

11569

pml4e = pmap_pml4e(pmap, va);

sys/amd64/amd64/pmap.c

11620

} else if (!pmap_demote_pde(pmap, pde, va)) {

sys/amd64/amd64/pmap.c

11641

pmap_invalidate_range(pmap, sva, eva);

sys/amd64/amd64/pmap.c

11645

pmap_pkru_check_uargs(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/amd64/pmap.c

11649

if (pmap->pm_type != PT_X86 || keyidx > PMAP_MAX_PKRU_IDX ||

sys/amd64/amd64/pmap.c

11658

pmap_pkru_set(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, u_int keyidx,

sys/amd64/amd64/pmap.c

11665

error = pmap_pkru_check_uargs(pmap, sva, eva, keyidx, flags);

sys/amd64/amd64/pmap.c

11669

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

11670

error = pmap_pkru_assign(pmap, sva, eva, keyidx, flags);

sys/amd64/amd64/pmap.c

11672

pmap_pkru_update_range(pmap, sva, eva, keyidx);

sys/amd64/amd64/pmap.c

11673

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

11682

pmap_pkru_clear(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

11688

error = pmap_pkru_check_uargs(pmap, sva, eva, 0, 0);

sys/amd64/amd64/pmap.c

11692

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

11693

error = pmap_pkru_deassign(pmap, sva, eva);

sys/amd64/amd64/pmap.c

11695

pmap_pkru_update_range(pmap, sva, eva, 0);

sys/amd64/amd64/pmap.c

11696

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

12193

pmap_t pmap;

sys/amd64/amd64/pmap.c

12208

pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace);

sys/amd64/amd64/pmap.c

12210

pmap = PCPU_GET(curpmap);

sys/amd64/amd64/pmap.c

12212

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

12215

if (pmap_is_la57(pmap)) {

sys/amd64/amd64/pmap.c

12216

pml5 = pmap_pml5e(pmap, va);

sys/amd64/amd64/pmap.c

12224

pml4 = pmap_pml4e(pmap, va);

sys/amd64/amd64/pmap.c

12318

pmap_t pmap;

sys/amd64/amd64/pmap.c

12325

pmap = (pmap_t)addr;

sys/amd64/amd64/pmap.c

12327

pmap = PCPU_GET(curpmap);

sys/amd64/amd64/pmap.c

12329

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

12331

if (pmap_is_la57(pmap)) {

sys/amd64/amd64/pmap.c

12332

pml5 = pmap->pm_pmltop;

sys/amd64/amd64/pmap.c

12345

ptpages_show_pml4(DMAP_TO_VM_PAGE(pmap->pm_pmltop), NUP4ML4E,

sys/amd64/amd64/pmap.c

1324

static void free_pv_entry(pmap_t pmap, pv_entry_t pv);

sys/amd64/amd64/pmap.c

1325

static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp);

sys/amd64/amd64/pmap.c

1328

static void reserve_pv_entries(pmap_t pmap, int needed,

sys/amd64/amd64/pmap.c

1330

static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/amd64/amd64/pmap.c

1332

static bool pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde,

sys/amd64/amd64/pmap.c

1335

static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/amd64/amd64/pmap.c

1338

static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);

sys/amd64/amd64/pmap.c

1339

static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,

sys/amd64/amd64/pmap.c

1342

static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);

sys/amd64/amd64/pmap.c

1345

static bool pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);

sys/amd64/amd64/pmap.c

1346

static bool pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde,

sys/amd64/amd64/pmap.c

1348

static bool pmap_demote_pde_mpte(pmap_t pmap, pd_entry_t *pde,

sys/amd64/amd64/pmap.c

1350

static bool pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe,

sys/amd64/amd64/pmap.c

1352

static int pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/amd64/amd64/pmap.c

1354

static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,

sys/amd64/amd64/pmap.c

1356

static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,

sys/amd64/amd64/pmap.c

1359

static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/amd64/amd64/pmap.c

1365

static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,

sys/amd64/amd64/pmap.c

1371

static bool pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,

sys/amd64/amd64/pmap.c

1374

static bool pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,

sys/amd64/amd64/pmap.c

1382

static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,

sys/amd64/amd64/pmap.c

1384

static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,

sys/amd64/amd64/pmap.c

1386

static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);

sys/amd64/amd64/pmap.c

1387

static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,

sys/amd64/amd64/pmap.c

1389

static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/amd64/pmap.c

1392

static bool pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,

sys/amd64/amd64/pmap.c

1394

static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,

sys/amd64/amd64/pmap.c

1398

static pd_entry_t *pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp,

sys/amd64/amd64/pmap.c

1400

static vm_page_t pmap_allocpte_alloc(pmap_t pmap, vm_pindex_t ptepindex,

sys/amd64/amd64/pmap.c

1402

static vm_page_t pmap_allocpte_nosleep(pmap_t pmap, vm_pindex_t ptepindex,

sys/amd64/amd64/pmap.c

1404

static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,

sys/amd64/amd64/pmap.c

1407

static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/amd64/amd64/pmap.c

1447

pmap_pml5e(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1450

MPASS(pmap_is_la57(pmap));

sys/amd64/amd64/pmap.c

1451

return (&pmap->pm_pmltop[pmap_pml5e_index(va)]);

sys/amd64/amd64/pmap.c

1455

pmap_pml5e_u(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1458

MPASS(pmap_is_la57(pmap));

sys/amd64/amd64/pmap.c

1459

return (&pmap->pm_pmltopu[pmap_pml5e_index(va)]);

sys/amd64/amd64/pmap.c

1474

pmap_pml4e(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1480

if (pmap_is_la57(pmap)) {

sys/amd64/amd64/pmap.c

1481

pml5e = pmap_pml5e(pmap, va);

sys/amd64/amd64/pmap.c

1482

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

1487

pml4e = pmap->pm_pmltop;

sys/amd64/amd64/pmap.c

1493

pmap_pml4e_u(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1495

MPASS(!pmap_is_la57(pmap));

sys/amd64/amd64/pmap.c

1496

return (&pmap->pm_pmltopu[pmap_pml4e_index(va)]);

sys/amd64/amd64/pmap.c

1511

pmap_pdpe(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1516

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

1517

pml4e = pmap_pml4e(pmap, va);

sys/amd64/amd64/pmap.c

1537

pmap_pde(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1542

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

1543

pdpe = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

1547

("pmap_pde for 1G page, pmap %p va %#lx", pmap, va));

sys/amd64/amd64/pmap.c

1565

pmap_pte(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

1570

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

1571

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

1580

pmap_resident_count_adj(pmap_t pmap, int count)

sys/amd64/amd64/pmap.c

1583

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

1584

KASSERT(pmap->pm_stats.resident_count + count >= 0,

sys/amd64/amd64/pmap.c

1585

("pmap %p resident count underflow %ld %d", pmap,

sys/amd64/amd64/pmap.c

1586

pmap->pm_stats.resident_count, count));

sys/amd64/amd64/pmap.c

1587

pmap->pm_stats.resident_count += count;

sys/amd64/amd64/pmap.c

1591

pmap_pt_page_count_pinit(pmap_t pmap, int count)

sys/amd64/amd64/pmap.c

1593

KASSERT(pmap->pm_stats.resident_count + count >= 0,

sys/amd64/amd64/pmap.c

1594

("pmap %p resident count underflow %ld %d", pmap,

sys/amd64/amd64/pmap.c

1595

pmap->pm_stats.resident_count, count));

sys/amd64/amd64/pmap.c

1596

pmap->pm_stats.resident_count += count;

sys/amd64/amd64/pmap.c

1600

pmap_pt_page_count_adj(pmap_t pmap, int count)

sys/amd64/amd64/pmap.c

1602

if (pmap == kernel_pmap)

sys/amd64/amd64/pmap.c

1605

if (pmap != NULL)

sys/amd64/amd64/pmap.c

1606

pmap_resident_count_adj(pmap, count);

sys/amd64/amd64/pmap.c

177

pmap_type_guest(pmap_t pmap)

sys/amd64/amd64/pmap.c

180

return ((pmap->pm_type == PT_EPT) || (pmap->pm_type == PT_RVI));

sys/amd64/amd64/pmap.c

184

pmap_emulate_ad_bits(pmap_t pmap)

sys/amd64/amd64/pmap.c

187

return ((pmap->pm_flags & PMAP_EMULATE_AD_BITS) != 0);

sys/amd64/amd64/pmap.c

191

pmap_valid_bit(pmap_t pmap)

sys/amd64/amd64/pmap.c

195

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

201

if (pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

207

panic("pmap_valid_bit: invalid pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

214

pmap_rw_bit(pmap_t pmap)

sys/amd64/amd64/pmap.c

218

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

224

if (pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

230

panic("pmap_rw_bit: invalid pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

2384

pmap_allow_2m_x_page(pmap_t pmap, bool executable)

sys/amd64/amd64/pmap.c

2387

return (pmap->pm_type != PT_EPT || !executable ||

sys/amd64/amd64/pmap.c

239

pmap_global_bit(pmap_t pmap)

sys/amd64/amd64/pmap.c

243

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

252

panic("pmap_global_bit: invalid pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

259

pmap_accessed_bit(pmap_t pmap)

sys/amd64/amd64/pmap.c

263

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

269

if (pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

2712

pmap_swap_pat(pmap_t pmap, pt_entry_t entry)

sys/amd64/amd64/pmap.c

2716

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

2734

panic("pmap_switch_pat_bits: bad pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

2741

pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)

sys/amd64/amd64/pmap.c

275

panic("pmap_accessed_bit: invalid pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

2753

pmap_cache_bits(pmap_t pmap, int mode, bool is_pde)

sys/amd64/amd64/pmap.c

2757

if (!pmap_is_valid_memattr(pmap, mode))

sys/amd64/amd64/pmap.c

2760

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

2784

panic("unsupported pmap type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

2791

pmap_cache_mask(pmap_t pmap, bool is_pde)

sys/amd64/amd64/pmap.c

2795

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

2804

panic("pmap_cache_mask: invalid pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

2811

pmap_pat_index(pmap_t pmap, pt_entry_t pte, bool is_pde)

sys/amd64/amd64/pmap.c

2816

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

282

pmap_modified_bit(pmap_t pmap)

sys/amd64/amd64/pmap.c

2846

pmap_ps_enabled(pmap_t pmap)

sys/amd64/amd64/pmap.c

2849

return (pg_ps_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0);

sys/amd64/amd64/pmap.c

2853

pmap_update_pde_store(pmap_t pmap, pd_entry_t *pde, pd_entry_t newpde)

sys/amd64/amd64/pmap.c

2856

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

286

switch (pmap->pm_type) {

sys/amd64/amd64/pmap.c

2875

atomic_add_long(&pmap->pm_eptgen, 1);

sys/amd64/amd64/pmap.c

2878

panic("pmap_update_pde_store: bad pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

2891

pmap_update_pde_invalidate(pmap_t pmap, vm_offset_t va, pd_entry_t newpde)

sys/amd64/amd64/pmap.c

2895

if (pmap_type_guest(pmap))

sys/amd64/amd64/pmap.c

2898

KASSERT(pmap->pm_type == PT_X86,

sys/amd64/amd64/pmap.c

2899

("pmap_update_pde_invalidate: invalid type %d", pmap->pm_type));

sys/amd64/amd64/pmap.c

2901

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

2905

pmap_invlpg(pmap, va);

sys/amd64/amd64/pmap.c

292

if (pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

298

panic("pmap_modified_bit: invalid pm_type %d", pmap->pm_type);

sys/amd64/amd64/pmap.c

305

pmap_pku_mask_bit(pmap_t pmap)

sys/amd64/amd64/pmap.c

308

return (pmap->pm_type == PT_X86 ? X86_PG_PKU_MASK : 0);

sys/amd64/amd64/pmap.c

3085

pmap_invalidate_ept(pmap_t pmap)

sys/amd64/amd64/pmap.c

3091

KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active),

sys/amd64/amd64/pmap.c

3116

atomic_add_long(&pmap->pm_eptgen, 1);

sys/amd64/amd64/pmap.c

3118

goal = smr_advance(pmap->pm_eptsmr);

sys/amd64/amd64/pmap.c

312

safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte)

sys/amd64/amd64/pmap.c

3123

ipinum = pmap->pm_flags & PMAP_NESTED_IPIMASK;

sys/amd64/amd64/pmap.c

3124

ipi_selected(pmap->pm_active, ipinum);

sys/amd64/amd64/pmap.c

3131

smr_wait(pmap->pm_eptsmr, goal);

sys/amd64/amd64/pmap.c

3135

pmap_invalidate_preipi_pcid(pmap_t pmap)

sys/amd64/amd64/pmap.c

3143

if (pmap != PCPU_GET(curpmap))

sys/amd64/amd64/pmap.c

3148

pcidp = zpcpu_get_cpu(pmap->pm_pcidp, i);

sys/amd64/amd64/pmap.c

315

if (!pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

3167

pmap_invalidate_preipi_nopcid(pmap_t pmap __unused)

sys/amd64/amd64/pmap.c

3179

pmap_invalidate_page_pcid_cb(pmap_t pmap, vm_offset_t va,

sys/amd64/amd64/pmap.c

318

KASSERT(pmap->pm_type == PT_EPT, ("invalid pm_type %d", pmap->pm_type));

sys/amd64/amd64/pmap.c

3199

if (pmap->pm_ucr3 == PMAP_NO_CR3 ||

sys/amd64/amd64/pmap.c

3203

pcid = pmap_get_pcid(pmap);

sys/amd64/amd64/pmap.c

3210

kcr3 = pmap->pm_cr3 | pcid | CR3_PCID_SAVE;

sys/amd64/amd64/pmap.c

3211

ucr3 = pmap->pm_ucr3 | pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;

sys/amd64/amd64/pmap.c

3217

pmap_invalidate_page_pcid_invpcid_cb(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

3219

pmap_invalidate_page_pcid_cb(pmap, va, true);

sys/amd64/amd64/pmap.c

3223

pmap_invalidate_page_pcid_noinvpcid_cb(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

3225

pmap_invalidate_page_pcid_cb(pmap, va, false);

sys/amd64/amd64/pmap.c

3229

pmap_invalidate_page_nopcid_cb(pmap_t pmap __unused, vm_offset_t va __unused)

sys/amd64/amd64/pmap.c

3242

pmap_invalidate_page_curcpu_cb(pmap_t pmap, vm_offset_t va,

sys/amd64/amd64/pmap.c

3245

if (pmap == kernel_pmap) {

sys/amd64/amd64/pmap.c

3247

} else if (pmap == PCPU_GET(curpmap)) {

sys/amd64/amd64/pmap.c

3249

pmap_invalidate_page_cb(pmap, va);

sys/amd64/amd64/pmap.c

3254

pmap_invalidate_page(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

3256

if (pmap_type_guest(pmap)) {

sys/amd64/amd64/pmap.c

3257

pmap_invalidate_ept(pmap);

sys/amd64/amd64/pmap.c

3261

KASSERT(pmap->pm_type == PT_X86,

sys/amd64/amd64/pmap.c

3262

("pmap_invalidate_page: invalid type %d", pmap->pm_type));

sys/amd64/amd64/pmap.c

3264

pmap_invalidate_preipi(pmap);

sys/amd64/amd64/pmap.c

3265

smp_masked_invlpg(va, pmap, pmap_invalidate_page_curcpu_cb);

sys/amd64/amd64/pmap.c

3272

pmap_invalidate_range_pcid_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/amd64/pmap.c

3281

if (pmap != PCPU_GET(curpmap) ||

sys/amd64/amd64/pmap.c

3282

pmap->pm_ucr3 == PMAP_NO_CR3 ||

sys/amd64/amd64/pmap.c

3286

pcid = pmap_get_pcid(pmap);

sys/amd64/amd64/pmap.c

3293

kcr3 = pmap->pm_cr3 | pcid | CR3_PCID_SAVE;

sys/amd64/amd64/pmap.c

3294

ucr3 = pmap->pm_ucr3 | pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;

sys/amd64/amd64/pmap.c

3300

pmap_invalidate_range_pcid_invpcid_cb(pmap_t pmap, vm_offset_t sva,

sys/amd64/amd64/pmap.c

3303

pmap_invalidate_range_pcid_cb(pmap, sva, eva, true);

sys/amd64/amd64/pmap.c

3307

pmap_invalidate_range_pcid_noinvpcid_cb(pmap_t pmap, vm_offset_t sva,

sys/amd64/amd64/pmap.c

3310

pmap_invalidate_range_pcid_cb(pmap, sva, eva, false);

sys/amd64/amd64/pmap.c

3314

pmap_invalidate_range_nopcid_cb(pmap_t pmap __unused, vm_offset_t sva __unused,

sys/amd64/amd64/pmap.c

332

((pmap->pm_flags & PMAP_SUPPORTS_EXEC_ONLY) != 0))

sys/amd64/amd64/pmap.c

3329

pmap_invalidate_range_curcpu_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

3333

if (pmap == kernel_pmap) {

sys/amd64/amd64/pmap.c

3342

} else if (pmap == PCPU_GET(curpmap)) {

sys/amd64/amd64/pmap.c

3345

pmap_invalidate_range_cb(pmap, sva, eva);

sys/amd64/amd64/pmap.c

3350

pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

3353

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

3357

if (pmap_type_guest(pmap)) {

sys/amd64/amd64/pmap.c

3358

pmap_invalidate_ept(pmap);

sys/amd64/amd64/pmap.c

3362

KASSERT(pmap->pm_type == PT_X86,

sys/amd64/amd64/pmap.c

3363

("pmap_invalidate_range: invalid type %d", pmap->pm_type));

sys/amd64/amd64/pmap.c

3365

pmap_invalidate_preipi(pmap);

sys/amd64/amd64/pmap.c

3366

smp_masked_invlpg_range(sva, eva, pmap,

sys/amd64/amd64/pmap.c

3371

pmap_invalidate_all_cb_template(pmap_t pmap, bool pmap_pcid_enabled1,

sys/amd64/amd64/pmap.c

3378

if (pmap == kernel_pmap) {

sys/amd64/amd64/pmap.c

3385

} else if (pmap == PCPU_GET(curpmap)) {

sys/amd64/amd64/pmap.c

3389

pcid = pmap_get_pcid(pmap);

sys/amd64/amd64/pmap.c

3396

kcr3 = pmap->pm_cr3 | pcid;

sys/amd64/amd64/pmap.c

3399

if (pmap->pm_ucr3 != PMAP_NO_CR3)

sys/amd64/amd64/pmap.c

3408

pmap_invalidate_all_pcid_invpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused,

sys/amd64/amd64/pmap.c

3411

pmap_invalidate_all_cb_template(pmap, true, true);

sys/amd64/amd64/pmap.c

3415

pmap_invalidate_all_pcid_noinvpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused,

sys/amd64/amd64/pmap.c

3418

pmap_invalidate_all_cb_template(pmap, true, false);

sys/amd64/amd64/pmap.c

3422

pmap_invalidate_all_nopcid_invpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused,

sys/amd64/amd64/pmap.c

3425

pmap_invalidate_all_cb_template(pmap, false, true);

sys/amd64/amd64/pmap.c

3429

pmap_invalidate_all_nopcid_noinvpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused,

sys/amd64/amd64/pmap.c

3432

pmap_invalidate_all_cb_template(pmap, false, false);

sys/amd64/amd64/pmap.c

3446

pmap_invalidate_all(pmap_t pmap)

sys/amd64/amd64/pmap.c

3448

if (pmap_type_guest(pmap)) {

sys/amd64/amd64/pmap.c

3449

pmap_invalidate_ept(pmap);

sys/amd64/amd64/pmap.c

3453

KASSERT(pmap->pm_type == PT_X86,

sys/amd64/amd64/pmap.c

3454

("pmap_invalidate_all: invalid type %d", pmap->pm_type));

sys/amd64/amd64/pmap.c

3456

pmap_invalidate_preipi(pmap);

sys/amd64/amd64/pmap.c

3457

smp_masked_invltlb(pmap, pmap_invalidate_all_curcpu_cb);

sys/amd64/amd64/pmap.c

3461

pmap_invalidate_cache_curcpu_cb(pmap_t pmap __unused, vm_offset_t va __unused,

sys/amd64/amd64/pmap.c

3476

pmap_t pmap;

sys/amd64/amd64/pmap.c

3489

pmap_update_pde_store(act->pmap, act->pde, act->newpde);

sys/amd64/amd64/pmap.c

3498

pmap_update_pde_invalidate(act->pmap, act->va, act->newpde);

sys/amd64/amd64/pmap.c

3510

pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)

sys/amd64/amd64/pmap.c

3520

if (pmap == kernel_pmap || pmap_type_guest(pmap))

sys/amd64/amd64/pmap.c

3523

active = pmap->pm_active;

sys/amd64/amd64/pmap.c

3529

act.pmap = pmap;

sys/amd64/amd64/pmap.c

3537

pmap_update_pde_store(pmap, pde, newpde);

sys/amd64/amd64/pmap.c

3539

pmap_update_pde_invalidate(pmap, va, newpde);

sys/amd64/amd64/pmap.c

3545

pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde)

sys/amd64/amd64/pmap.c

3560

pmap_invalidate_range(pmap, va, va + NBPDR - 1);

sys/amd64/amd64/pmap.c

3562

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

3749

pmap_extract(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

3757

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

3758

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

3759

pdpe = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

3777

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

3789

pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/amd64/amd64/pmap.c

3797

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

3798

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

3799

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

3801

pdpep = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

3831

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

4036

pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/amd64/amd64/pmap.c

404

struct pmap kernel_pmap_store = {

sys/amd64/amd64/pmap.c

4040

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

4044

return (vm_radix_insert(&pmap->pm_root, mpte));

sys/amd64/amd64/pmap.c

4054

pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

4057

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

4058

return (vm_radix_remove(&pmap->pm_root, pmap_pde_pindex(va)));

sys/amd64/amd64/pmap.c

4068

pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/amd64/amd64/pmap.c

4073

_pmap_unwire_ptp(pmap, va, m, free);

sys/amd64/amd64/pmap.c

4080

_pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/amd64/amd64/pmap.c

4088

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

4095

MPASS(pmap_is_la57(pmap));

sys/amd64/amd64/pmap.c

4096

pml5 = pmap_pml5e(pmap, va);

sys/amd64/amd64/pmap.c

4098

if (pmap->pm_pmltopu != NULL && va <= VM_MAXUSER_ADDRESS) {

sys/amd64/amd64/pmap.c

4099

pml5 = pmap_pml5e_u(pmap, va);

sys/amd64/amd64/pmap.c

4104

pml4 = pmap_pml4e(pmap, va);

sys/amd64/amd64/pmap.c

4106

if (!pmap_is_la57(pmap) && pmap->pm_pmltopu != NULL &&

sys/amd64/amd64/pmap.c

4108

pml4 = pmap_pml4e_u(pmap, va);

sys/amd64/amd64/pmap.c

4113

pdp = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

4117

pd = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

4122

pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME);

sys/amd64/amd64/pmap.c

4123

pmap_unwire_ptp(pmap, va, pdpg, free);

sys/amd64/amd64/pmap.c

4126

pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME);

sys/amd64/amd64/pmap.c

4127

pmap_unwire_ptp(pmap, va, pdppg, free);

sys/amd64/amd64/pmap.c

4128

} else if (m->pindex < NUPDE + NUPDPE + NUPML4E && pmap_is_la57(pmap)) {

sys/amd64/amd64/pmap.c

4130

pml4pg = PHYS_TO_VM_PAGE(*pmap_pml5e(pmap, va) & PG_FRAME);

sys/amd64/amd64/pmap.c

4131

pmap_unwire_ptp(pmap, va, pml4pg, free);

sys/amd64/amd64/pmap.c

4134

pmap_pt_page_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

4148

pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde,

sys/amd64/amd64/pmap.c

4157

return (pmap_unwire_ptp(pmap, va, mpte, free));

sys/amd64/amd64/pmap.c

4165

pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte)

sys/amd64/amd64/pmap.c

4170

if (pmap_unwire_ptp(pmap, va, mpte, &free)) {

sys/amd64/amd64/pmap.c

4176

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

4182

pmap_pinit_pcids(pmap_t pmap, uint32_t pcid, int gen)

sys/amd64/amd64/pmap.c

4188

pcidp = zpcpu_get_cpu(pmap->pm_pcidp, i);

sys/amd64/amd64/pmap.c

4195

pmap_pinit0(pmap_t pmap)

sys/amd64/amd64/pmap.c

420

static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,

sys/amd64/amd64/pmap.c

4200

PMAP_LOCK_INIT(pmap);

sys/amd64/amd64/pmap.c

4201

pmap->pm_pmltop = kernel_pmap->pm_pmltop;

sys/amd64/amd64/pmap.c

4202

pmap->pm_pmltopu = NULL;

sys/amd64/amd64/pmap.c

4203

pmap->pm_cr3 = kernel_pmap->pm_cr3;

sys/amd64/amd64/pmap.c

4205

pmap->pm_ucr3 = PMAP_NO_CR3;

sys/amd64/amd64/pmap.c

4206

vm_radix_init(&pmap->pm_root);

sys/amd64/amd64/pmap.c

4207

CPU_ZERO(&pmap->pm_active);

sys/amd64/amd64/pmap.c

4208

TAILQ_INIT(&pmap->pm_pvchunk);

sys/amd64/amd64/pmap.c

4209

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/amd64/amd64/pmap.c

4210

pmap->pm_flags = pmap_flags;

sys/amd64/amd64/pmap.c

4211

pmap->pm_pcidp = uma_zalloc_pcpu(pcpu_zone_8, M_WAITOK);

sys/amd64/amd64/pmap.c

4212

pmap_pinit_pcids(pmap, PMAP_PCID_KERN + 1, 1);

sys/amd64/amd64/pmap.c

4213

pmap_activate_boot(pmap);

sys/amd64/amd64/pmap.c

4317

pmap_alloc_pt_page(pmap_t pmap, vm_pindex_t pindex, int flags)

sys/amd64/amd64/pmap.c

4325

pmap_pt_page_count_adj(pmap, 1);

sys/amd64/amd64/pmap.c

4330

pmap_free_pt_page(pmap_t pmap, vm_page_t m, bool zerofilled)

sys/amd64/amd64/pmap.c

4346

pmap_pt_page_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

4356

pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags)

sys/amd64/amd64/pmap.c

4361

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/amd64/amd64/pmap.c

4378

pmap_pt_page_count_pinit(pmap, 1);

sys/amd64/amd64/pmap.c

4381

pmap->pm_pmltop = PHYS_TO_DMAP(pmltop_phys);

sys/amd64/amd64/pmap.c

4384

if (pmap->pm_pcidp == NULL)

sys/amd64/amd64/pmap.c

4385

pmap->pm_pcidp = uma_zalloc_pcpu(pcpu_zone_8,

sys/amd64/amd64/pmap.c

4387

pmap_pinit_pcids(pmap, PMAP_PCID_NONE, 0);

sys/amd64/amd64/pmap.c

4389

pmap->pm_cr3 = PMAP_NO_CR3; /* initialize to an invalid value */

sys/amd64/amd64/pmap.c

4390

pmap->pm_ucr3 = PMAP_NO_CR3;

sys/amd64/amd64/pmap.c

4391

pmap->pm_pmltopu = NULL;

sys/amd64/amd64/pmap.c

4393

pmap->pm_type = pm_type;

sys/amd64/amd64/pmap.c

4403

pmap->pm_cr3 = pmltop_phys;

sys/amd64/amd64/pmap.c

4404

if (pmap_is_la57(pmap))

sys/amd64/amd64/pmap.c

4416

pmap_pt_page_count_pinit(pmap, 1);

sys/amd64/amd64/pmap.c

4417

pmap->pm_pmltopu = PHYS_TO_DMAP(

sys/amd64/amd64/pmap.c

4419

if (pmap_is_la57(pmap))

sys/amd64/amd64/pmap.c

442

pmap_is_la57(pmap_t pmap)

sys/amd64/amd64/pmap.c

4423

pmap->pm_ucr3 = VM_PAGE_TO_PHYS(pmltop_pgu);

sys/amd64/amd64/pmap.c

4426

rangeset_init(&pmap->pm_pkru, pkru_dup_range,

sys/amd64/amd64/pmap.c

4427

pkru_free_range, pmap, M_NOWAIT);

sys/amd64/amd64/pmap.c

4432

pmap->pm_eptsmr = smr_create("pmap", 0, 0);

sys/amd64/amd64/pmap.c

4436

vm_radix_init(&pmap->pm_root);

sys/amd64/amd64/pmap.c

4437

CPU_ZERO(&pmap->pm_active);

sys/amd64/amd64/pmap.c

4438

TAILQ_INIT(&pmap->pm_pvchunk);

sys/amd64/amd64/pmap.c

4439

pmap->pm_flags = flags;

sys/amd64/amd64/pmap.c

444

if (pmap->pm_type == PT_X86)

sys/amd64/amd64/pmap.c

4440

pmap->pm_eptgen = 0;

sys/amd64/amd64/pmap.c

4446

pmap_pinit(pmap_t pmap)

sys/amd64/amd64/pmap.c

4449

return (pmap_pinit_type(pmap, PT_X86, pmap_flags));

sys/amd64/amd64/pmap.c

4453

pmap_allocpte_free_unref(pmap_t pmap, vm_offset_t va, pt_entry_t *pte)

sys/amd64/amd64/pmap.c

4462

_pmap_unwire_ptp(pmap, va, mpg, &free);

sys/amd64/amd64/pmap.c

4463

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

4468

pmap_allocpte_getpml4(pmap_t pmap, struct rwlock **lockp, vm_offset_t va,

sys/amd64/amd64/pmap.c

4478

if (!pmap_is_la57(pmap))

sys/amd64/amd64/pmap.c

4479

return (&pmap->pm_pmltop[pmap_pml4e_index(va)]);

sys/amd64/amd64/pmap.c

4481

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

4483

pml5 = &pmap->pm_pmltop[pml5index];

sys/amd64/amd64/pmap.c

4485

if (pmap_allocpte_nosleep(pmap, pmap_pml5e_pindex(va), lockp,

sys/amd64/amd64/pmap.c

4505

pmap_allocpte_getpdp(pmap_t pmap, struct rwlock **lockp, vm_offset_t va,

sys/amd64/amd64/pmap.c

4514

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

4516

pml4 = pmap_allocpte_getpml4(pmap, lockp, va, false);

sys/amd64/amd64/pmap.c

4522

if (pmap_allocpte_nosleep(pmap, pmap_pml4e_pindex(va), lockp,

sys/amd64/amd64/pmap.c

4524

if (pmap_is_la57(pmap))

sys/amd64/amd64/pmap.c

4525

pmap_allocpte_free_unref(pmap, va,

sys/amd64/amd64/pmap.c

4526

pmap_pml5e(pmap, va));

sys/amd64/amd64/pmap.c

4579

pmap_allocpte_nosleep(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp,

sys/amd64/amd64/pmap.c

4590

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

4592

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

4593

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

4594

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

4595

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

4600

m = pmap_alloc_pt_page(pmap, ptepindex,

sys/amd64/amd64/pmap.c

4610

MPASS(pmap_is_la57(pmap));

sys/amd64/amd64/pmap.c

4613

pml5 = &pmap->pm_pmltop[pml5index];

sys/amd64/amd64/pmap.c

4615

("pmap %p va %#lx pml5 %#lx", pmap, va, *pml5));

sys/amd64/amd64/pmap.c

4618

if (pmap->pm_pmltopu != NULL && pml5index < NUPML5E) {

sys/amd64/amd64/pmap.c

4619

MPASS(pmap->pm_ucr3 != PMAP_NO_CR3);

sys/amd64/amd64/pmap.c

4622

pml5u = &pmap->pm_pmltopu[pml5index];

sys/amd64/amd64/pmap.c

4629

pml4 = pmap_allocpte_getpml4(pmap, lockp, va, true);

sys/amd64/amd64/pmap.c

4631

pmap_free_pt_page(pmap, m, true);

sys/amd64/amd64/pmap.c

4635

("pmap %p va %#lx pml4 %#lx", pmap, va, *pml4));

sys/amd64/amd64/pmap.c

4638

if (!pmap_is_la57(pmap) && pmap->pm_pmltopu != NULL &&

sys/amd64/amd64/pmap.c

4640

MPASS(pmap->pm_ucr3 != PMAP_NO_CR3);

sys/amd64/amd64/pmap.c

4651

pml4u = &pmap->pm_pmltopu[pml4index];

sys/amd64/amd64/pmap.c

4657

pdp = pmap_allocpte_getpdp(pmap, lockp, va, true);

sys/amd64/amd64/pmap.c

4659

pmap_free_pt_page(pmap, m, true);

sys/amd64/amd64/pmap.c

4663

("pmap %p va %#lx pdp %#lx", pmap, va, *pdp));

sys/amd64/amd64/pmap.c

4667

pdp = pmap_allocpte_getpdp(pmap, lockp, va, false);

sys/amd64/amd64/pmap.c

4669

pmap_free_pt_page(pmap, m, true);

sys/amd64/amd64/pmap.c

4674

if (pmap_allocpte_nosleep(pmap, pmap_pdpe_pindex(va),

sys/amd64/amd64/pmap.c

4676

pmap_allocpte_free_unref(pmap, va,

sys/amd64/amd64/pmap.c

4677

pmap_pml4e(pmap, va));

sys/amd64/amd64/pmap.c

4678

pmap_free_pt_page(pmap, m, true);

sys/amd64/amd64/pmap.c

4691

("pmap %p va %#lx pd %#lx", pmap, va, *pd));

sys/amd64/amd64/pmap.c

4708

pmap_allocpte_alloc(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp,

sys/amd64/amd64/pmap.c

4713

m = pmap_allocpte_nosleep(pmap, ptepindex, lockp, va);

sys/amd64/amd64/pmap.c

4716

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

4719

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

4725

pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp,

sys/amd64/amd64/pmap.c

4733

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

4736

pdpe = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

4748

pdpg = pmap_allocpte_alloc(pmap, NUPDE + pdpindex, lockp, va);

sys/amd64/amd64/pmap.c

4765

pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)

sys/amd64/amd64/pmap.c

4771

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

4781

pd = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

4788

if (!pmap_demote_pde_locked(pmap, pd, va, lockp)) {

sys/amd64/amd64/pmap.c

4809

m = pmap_allocpte_alloc(pmap, ptepindex, lockp, va);

sys/amd64/amd64/pmap.c

4826

pmap_release(pmap_t pmap)

sys/amd64/amd64/pmap.c

4831

KASSERT(vm_radix_is_empty(&pmap->pm_root),

sys/amd64/amd64/pmap.c

4833

pmap));

sys/amd64/amd64/pmap.c

4834

KASSERT(CPU_EMPTY(&pmap->pm_active),

sys/amd64/amd64/pmap.c

4835

("releasing active pmap %p", pmap));

sys/amd64/amd64/pmap.c

4837

m = DMAP_TO_VM_PAGE(pmap->pm_pmltop);

sys/amd64/amd64/pmap.c

4839

if (pmap_is_la57(pmap)) {

sys/amd64/amd64/pmap.c

4841

pmap->pm_pmltop[i] = 0;

sys/amd64/amd64/pmap.c

4844

pmap->pm_pmltop[KPML4BASE + i] = 0;

sys/amd64/amd64/pmap.c

4847

pmap->pm_pmltop[KASANPML4I + i] = 0;

sys/amd64/amd64/pmap.c

4851

pmap->pm_pmltop[KMSANSHADPML4I + i] = 0;

sys/amd64/amd64/pmap.c

4853

pmap->pm_pmltop[KMSANORIGPML4I + i] = 0;

sys/amd64/amd64/pmap.c

4856

pmap->pm_pmltop[DMPML4I + i] = 0;

sys/amd64/amd64/pmap.c

4857

pmap->pm_pmltop[PML4PML4I] = 0; /* Recursive Mapping */

sys/amd64/amd64/pmap.c

4859

pmap->pm_pmltop[LMSPML4I + i] = 0;

sys/amd64/amd64/pmap.c

4863

pmap_pt_page_count_pinit(pmap, -1);

sys/amd64/amd64/pmap.c

4865

if (pmap->pm_pmltopu != NULL) {

sys/amd64/amd64/pmap.c

4866

m = DMAP_TO_VM_PAGE(pmap->pm_pmltopu);

sys/amd64/amd64/pmap.c

4868

pmap_pt_page_count_pinit(pmap, -1);

sys/amd64/amd64/pmap.c

4870

if (pmap->pm_type == PT_X86 &&

sys/amd64/amd64/pmap.c

4872

rangeset_fini(&pmap->pm_pkru);

sys/amd64/amd64/pmap.c

4874

KASSERT(pmap->pm_stats.resident_count == 0,

sys/amd64/amd64/pmap.c

4876

pmap, pmap->pm_stats.resident_count));

sys/amd64/amd64/pmap.c

5173

reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap, bool start_di)

sys/amd64/amd64/pmap.c

5176

if (pmap == NULL)

sys/amd64/amd64/pmap.c

5178

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

5179

if (pmap != locked_pmap)

sys/amd64/amd64/pmap.c

5180

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

5204

pmap_t next_pmap, pmap;

sys/amd64/amd64/pmap.c

5217

pmap = NULL;

sys/amd64/amd64/pmap.c

5257

if (pmap != next_pmap) {

sys/amd64/amd64/pmap.c

5259

reclaim_pv_chunk_leave_pmap(pmap, locked_pmap,

sys/amd64/amd64/pmap.c

5261

pmap = next_pmap;

sys/amd64/amd64/pmap.c

5263

if (pmap > locked_pmap) {

sys/amd64/amd64/pmap.c

5265

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

5270

} else if (pmap != locked_pmap) {

sys/amd64/amd64/pmap.c

5271

if (PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

5277

pmap = NULL; /* pmap is not locked */

sys/amd64/amd64/pmap.c

5287

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

5288

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

5289

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

5290

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

5305

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

5313

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

5332

pmap_unuse_pt(pmap, va, *pde, &free);

sys/amd64/amd64/pmap.c

5341

pmap_resident_count_adj(pmap, -freed);

sys/amd64/amd64/pmap.c

5345

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5357

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5360

if (pmap == locked_pmap)

sys/amd64/amd64/pmap.c

5365

if (pvc->active_reclaims == 1 && pmap != NULL) {

sys/amd64/amd64/pmap.c

5384

reclaim_pv_chunk_leave_pmap(pmap, locked_pmap, start_di);

sys/amd64/amd64/pmap.c

5416

free_pv_entry(pmap_t pmap, pv_entry_t pv)

sys/amd64/amd64/pmap.c

5421

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5432

if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {

sys/amd64/amd64/pmap.c

5433

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5434

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5438

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5504

get_pv_entry(pmap_t pmap, struct rwlock **lockp)

sys/amd64/amd64/pmap.c

5512

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5515

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/amd64/amd64/pmap.c

5528

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5529

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,

sys/amd64/amd64/pmap.c

5544

m = reclaim_pv_chunk(pmap, lockp);

sys/amd64/amd64/pmap.c

5553

pc->pc_pmap = pmap;

sys/amd64/amd64/pmap.c

5562

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5604

reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)

sys/amd64/amd64/pmap.c

5613

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5626

TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {

sys/amd64/amd64/pmap.c

5643

m = reclaim_pv_chunk(pmap, lockp);

sys/amd64/amd64/pmap.c

5653

pc->pc_pmap = pmap;

sys/amd64/amd64/pmap.c

5657

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5686

pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

5691

if (pmap == PV_PMAP(pv) && va == pv->pv_va) {

sys/amd64/amd64/pmap.c

5706

pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/amd64/amd64/pmap.c

5716

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5728

pv = pmap_pvh_remove(pvh, pmap, va);

sys/amd64/amd64/pmap.c

5737

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/amd64/amd64/pmap.c

5755

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5756

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5760

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5761

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

5774

pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/amd64/amd64/pmap.c

5795

pv = pmap_pvh_remove(&m->md, pmap, va);

sys/amd64/amd64/pmap.c

5805

pmap_pvh_free(&m->md, pmap, va);

sys/amd64/amd64/pmap.c

5816

pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/amd64/amd64/pmap.c

5820

pv = pmap_pvh_remove(pvh, pmap, va);

sys/amd64/amd64/pmap.c

5822

free_pv_entry(pmap, pv);

sys/amd64/amd64/pmap.c

5830

pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/amd64/amd64/pmap.c

5835

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5837

if ((pv = get_pv_entry(pmap, NULL)) != NULL) {

sys/amd64/amd64/pmap.c

5853

pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, u_int flags,

sys/amd64/amd64/pmap.c

5860

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5862

if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?

sys/amd64/amd64/pmap.c

5893

pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)

sys/amd64/amd64/pmap.c

5899

rv = pmap_demote_pde_locked(pmap, pde, va, &lock);

sys/amd64/amd64/pmap.c

5933

pmap_demote_pde_abort(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,

sys/amd64/amd64/pmap.c

5941

pmap_remove_pde(pmap, pde, sva, true, &free, lockp);

sys/amd64/amd64/pmap.c

5942

if ((oldpde & pmap_global_bit(pmap)) == 0)

sys/amd64/amd64/pmap.c

5943

pmap_invalidate_pde_page(pmap, sva, oldpde);

sys/amd64/amd64/pmap.c

5946

va, pmap);

sys/amd64/amd64/pmap.c

5950

pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,

sys/amd64/amd64/pmap.c

5953

return (pmap_demote_pde_mpte(pmap, pde, va, lockp, NULL));

sys/amd64/amd64/pmap.c

5957

pmap_demote_pde_mpte(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,

sys/amd64/amd64/pmap.c

5967

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

5968

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

5969

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

5970

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

5971

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

5972

PG_PTE_CACHE = pmap_cache_mask(pmap, false);

sys/amd64/amd64/pmap.c

5973

PG_PKU_MASK = pmap_pku_mask_bit(pmap);

sys/amd64/amd64/pmap.c

5975

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

5989

pmap_demote_pde_abort(pmap, va, pde, oldpde,

sys/amd64/amd64/pmap.c

5993

mpte = pmap_remove_pt_page(pmap, va);

sys/amd64/amd64/pmap.c

5996

} else if ((mpte = pmap_remove_pt_page(pmap, va)) == NULL) {

sys/amd64/amd64/pmap.c

6019

mpte = pmap_alloc_pt_page(pmap, pmap_pde_pindex(va),

sys/amd64/amd64/pmap.c

6028

pmap_demote_pde_abort(pmap, va, pde, oldpde,

sys/amd64/amd64/pmap.c

6043

newpte = pmap_swap_pat(pmap, newpte);

sys/amd64/amd64/pmap.c

6070

reserve_pv_entries(pmap, NPTEPG - 1, lockp);

sys/amd64/amd64/pmap.c

6080

pmap_update_pde(pmap, va, pde, newpde);

sys/amd64/amd64/pmap.c

6088

pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));

sys/amd64/amd64/pmap.c

6094

pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME, lockp);

sys/amd64/amd64/pmap.c

6098

va, pmap);

sys/amd64/amd64/pmap.c

6106

pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)

sys/amd64/amd64/pmap.c

6112

KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));

sys/amd64/amd64/pmap.c

6113

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6114

mpte = pmap_remove_pt_page(pmap, va);

sys/amd64/amd64/pmap.c

6131

pmap_update_pde(pmap, va, pde, newpde);

sys/amd64/amd64/pmap.c

6138

pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));

sys/amd64/amd64/pmap.c

6145

pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, bool demote_kpde,

sys/amd64/amd64/pmap.c

6154

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

6155

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

6156

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

6157

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

6159

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6164

pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;

sys/amd64/amd64/pmap.c

6167

pmap_resident_count_adj(pmap, -NBPDR / PAGE_SIZE);

sys/amd64/amd64/pmap.c

6171

pmap_pvh_free(pvh, pmap, sva);

sys/amd64/amd64/pmap.c

6185

if (pmap != kernel_pmap) {

sys/amd64/amd64/pmap.c

6186

mpte = pmap_remove_pt_page(pmap, sva);

sys/amd64/amd64/pmap.c

6190

pmap_pt_page_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

6197

pmap_remove_kernel_pde(pmap, pdq, sva);

sys/amd64/amd64/pmap.c

6199

mpte = vm_radix_lookup(&pmap->pm_root, pmap_pde_pindex(sva));

sys/amd64/amd64/pmap.c

6205

return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free));

sys/amd64/amd64/pmap.c

6212

pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,

sys/amd64/amd64/pmap.c

6219

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

6220

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

6221

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

6223

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6226

pmap->pm_stats.wired_count -= 1;

sys/amd64/amd64/pmap.c

6227

pmap_resident_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

6235

pmap_pvh_free(&m->md, pmap, va);

sys/amd64/amd64/pmap.c

6244

return (pmap_unuse_pt(pmap, va, ptepde, free));

sys/amd64/amd64/pmap.c

6251

pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,

sys/amd64/amd64/pmap.c

6257

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

6258

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6265

pmap_remove_pte(pmap, pte, va, *pde, free, &lock);

sys/amd64/amd64/pmap.c

6268

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

6275

pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/amd64/pmap.c

6282

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6283

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

6290

pmap_invalidate_range(pmap, va, sva);

sys/amd64/amd64/pmap.c

6299

if (pmap_remove_pte(pmap, pte, sva, *pde, free, lockp)) {

sys/amd64/amd64/pmap.c

6305

pmap_invalidate_range(pmap, va, sva);

sys/amd64/amd64/pmap.c

6310

pmap_remove1(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, bool map_delete)

sys/amd64/amd64/pmap.c

6323

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

6324

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

6333

if (pmap->pm_stats.resident_count <= 1 + (pmap->pm_pmltopu != NULL ?

sys/amd64/amd64/pmap.c

6341

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

6343

pmap_pkru_on_remove(pmap, sva, eva);

sys/amd64/amd64/pmap.c

6351

pde = pmap_pde(pmap, sva);

sys/amd64/amd64/pmap.c

6353

pmap_remove_page(pmap, sva, pde, &free);

sys/amd64/amd64/pmap.c

6360

if (pmap->pm_stats.resident_count == 0)

sys/amd64/amd64/pmap.c

6363

if (pmap_is_la57(pmap)) {

sys/amd64/amd64/pmap.c

6364

pml5e = pmap_pml5e(pmap, sva);

sys/amd64/amd64/pmap.c

637

static bool pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/amd64/pmap.c

6373

pml4e = pmap_pml4e(pmap, sva);

sys/amd64/amd64/pmap.c

639

static pt_entry_t pmap_pkru_get(pmap_t pmap, vm_offset_t va);

sys/amd64/amd64/pmap.c

6393

MPASS(pmap != kernel_pmap); /* XXXKIB */

sys/amd64/amd64/pmap.c

6397

pmap_resident_count_adj(pmap, -NBPDP / PAGE_SIZE);

sys/amd64/amd64/pmap.c

6398

mt = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, sva) & PG_FRAME);

sys/amd64/amd64/pmap.c

6399

pmap_unwire_ptp(pmap, sva, mt, &free);

sys/amd64/amd64/pmap.c

640

static void pmap_pkru_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);

sys/amd64/amd64/pmap.c

6434

pmap_remove_pde(pmap, pde, sva, true, &free,

sys/amd64/amd64/pmap.c

6437

} else if (!pmap_demote_pde_locked(pmap, pde, sva,

sys/amd64/amd64/pmap.c

644

static int pmap_pkru_deassign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);

sys/amd64/amd64/pmap.c

645

static void pmap_pkru_deassign_all(pmap_t pmap);

sys/amd64/amd64/pmap.c

6453

if (pmap_remove_ptes(pmap, sva, va_next, pde, &free, &lock))

sys/amd64/amd64/pmap.c

6460

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

6461

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

6473

pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

6475

pmap_remove1(pmap, sva, eva, false);

sys/amd64/amd64/pmap.c

6485

pmap_map_delete(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

6487

pmap_remove1(pmap, sva, eva, true);

sys/amd64/amd64/pmap.c

6508

pmap_t pmap;

sys/amd64/amd64/pmap.c

6525

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

6526

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

6529

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

6532

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

6537

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

6538

(void)pmap_demote_pde_locked(pmap, pde, va, &lock);

sys/amd64/amd64/pmap.c

6539

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

6542

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

6543

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

6547

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

6550

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

6554

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

6555

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

6556

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

6557

pmap_resident_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

6558

pde = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

6564

pmap->pm_stats.wired_count--;

sys/amd64/amd64/pmap.c

6573

pmap_unuse_pt(pmap, pv->pv_va, *pde, &free);

sys/amd64/amd64/pmap.c

6574

pmap_invalidate_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

6577

free_pv_entry(pmap, pv);

sys/amd64/amd64/pmap.c

6578

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

6590

pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)

sys/amd64/amd64/pmap.c

6597

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

6598

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

6599

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

6601

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6639

pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)

sys/amd64/amd64/pmap.c

6652

pmap_remove(pmap, sva, eva);

sys/amd64/amd64/pmap.c

6660

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

6661

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

6662

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

6663

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

6686

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

6688

pml4e = pmap_pml4e(pmap, sva);

sys/amd64/amd64/pmap.c

6710

MPASS(pmap != kernel_pmap); /* XXXKIB */

sys/amd64/amd64/pmap.c

6750

if (pmap_protect_pde(pmap, pde, sva, prot))

sys/amd64/amd64/pmap.c

6764

if (ptpaddr == pbits || !pmap_demote_pde(pmap, pde,

sys/amd64/amd64/pmap.c

6800

pmap_invalidate_page(pmap, sva);

sys/amd64/amd64/pmap.c

6807

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

6808

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

6812

pmap_pde_ept_executable(pmap_t pmap, pd_entry_t pde)

sys/amd64/amd64/pmap.c

6815

if (pmap->pm_type != PT_EPT)

sys/amd64/amd64/pmap.c

6829

pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, vm_page_t mpte,

sys/amd64/amd64/pmap.c

6837

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

6838

if (!pmap_ps_enabled(pmap))

sys/amd64/amd64/pmap.c

6841

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

6842

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

6843

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

6844

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

6845

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

6846

PG_PKU_MASK = pmap_pku_mask_bit(pmap);

sys/amd64/amd64/pmap.c

6847

PG_PTE_CACHE = pmap_cache_mask(pmap, false);

sys/amd64/amd64/pmap.c

6856

if (!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap, newpde)))

sys/amd64/amd64/pmap.c

6861

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

6891

" in pmap %p", va & ~PDRMASK, pmap);

sys/amd64/amd64/pmap.c

6906

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

6920

(va & ~PDRMASK), pmap);

sys/amd64/amd64/pmap.c

6925

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

6944

KASSERT((newpde & PG_A) != 0 || safe_to_clear_referenced(pmap, newpde),

sys/amd64/amd64/pmap.c

6962

if (pmap_insert_pt_page(pmap, mpte, true, allpte_PG_A != 0)) {

sys/amd64/amd64/pmap.c

6966

pmap);

sys/amd64/amd64/pmap.c

6974

pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME, lockp);

sys/amd64/amd64/pmap.c

6979

newpde = pmap_swap_pat(pmap, newpde);

sys/amd64/amd64/pmap.c

6985

pmap_update_pde(pmap, va, pde, PG_PS | newpde);

sys/amd64/amd64/pmap.c

6991

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

6997

pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags,

sys/amd64/amd64/pmap.c

7003

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

7016

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7020

if (!pmap_pkru_same(pmap, va, va + pagesizes[psind], &pten))

sys/amd64/amd64/pmap.c

7024

pml4e = pmap_pml4e(pmap, va);

sys/amd64/amd64/pmap.c

7026

mp = pmap_allocpte_alloc(pmap, pmap_pml4e_pindex(va),

sys/amd64/amd64/pmap.c

7045

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

7047

mp = pmap_allocpte_alloc(pmap, pmap_pdpe_pindex(va),

sys/amd64/amd64/pmap.c

7058

pdpe = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

7071

pmap->pm_stats.wired_count += pagesizes[psind] / PAGE_SIZE;

sys/amd64/amd64/pmap.c

7073

pmap->pm_stats.wired_count -= pagesizes[psind] / PAGE_SIZE;

sys/amd64/amd64/pmap.c

7075

pmap_resident_count_adj(pmap, pagesizes[psind] / PAGE_SIZE);

sys/amd64/amd64/pmap.c

7082

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

7084

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

7105

pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/amd64/amd64/pmap.c

7118

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

7119

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

7120

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

7121

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7122

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

7149

if (pmap == kernel_pmap)

sys/amd64/amd64/pmap.c

7151

newpte |= pmap_cache_bits(pmap, m->md.pat_mode, psind > 0);

sys/amd64/amd64/pmap.c

7165

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

7169

rv = pmap_enter_largepage(pmap, va, newpte | PG_PS, flags,

sys/amd64/amd64/pmap.c

7177

rv = pmap_enter_pde(pmap, va, newpte | PG_PS, flags, m, &lock);

sys/amd64/amd64/pmap.c

7187

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

7189

pmap_demote_pde_locked(pmap, pde, va, &lock))) {

sys/amd64/amd64/pmap.c

7201

mpte = pmap_allocpte_alloc(pmap, pmap_pde_pindex(va),

sys/amd64/amd64/pmap.c

7213

if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86)

sys/amd64/amd64/pmap.c

7214

newpte |= pmap_pkru_get(pmap, va);

sys/amd64/amd64/pmap.c

7227

pmap->pm_stats.wired_count++;

sys/amd64/amd64/pmap.c

7229

pmap->pm_stats.wired_count--;

sys/amd64/amd64/pmap.c

7281

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

7285

pv = pmap_pvh_remove(&om->md, pmap, va);

sys/amd64/amd64/pmap.c

7289

free_pv_entry(pmap, pv);

sys/amd64/amd64/pmap.c

7300

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

7308

pmap->pm_stats.wired_count++;

sys/amd64/amd64/pmap.c

7309

pmap_resident_count_adj(pmap, 1);

sys/amd64/amd64/pmap.c

7317

pv = get_pv_entry(pmap, &lock);

sys/amd64/amd64/pmap.c

7352

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

7366

(void)pmap_promote_pde(pmap, pde, va, mpte, &lock);

sys/amd64/amd64/pmap.c

7373

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

7384

pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/amd64/amd64/pmap.c

7390

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

7391

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7393

pmap_cache_bits(pmap, m->md.pat_mode, true) | PG_PS | PG_V;

sys/amd64/amd64/pmap.c

7400

return (pmap_enter_pde(pmap, va, newpde, PMAP_ENTER_NOSLEEP |

sys/amd64/amd64/pmap.c

7441

pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags,

sys/amd64/amd64/pmap.c

7450

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

7451

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

7452

KASSERT((newpde & (pmap_modified_bit(pmap) | PG_RW)) != PG_RW,

sys/amd64/amd64/pmap.c

7457

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7458

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

7460

if (!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap,

sys/amd64/amd64/pmap.c

7463

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

7466

if ((pde = pmap_alloc_pde(pmap, va, &pdpg, (flags &

sys/amd64/amd64/pmap.c

7469

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

7478

if (!pmap_pkru_same(pmap, va, va + NBPDR, &newpde)) {

sys/amd64/amd64/pmap.c

7479

pmap_abort_ptp(pmap, va, pdpg);

sys/amd64/amd64/pmap.c

7496

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

7504

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

7518

(void)pmap_remove_pde(pmap, pde, va, false, &free,

sys/amd64/amd64/pmap.c

7521

pmap_invalidate_pde_page(pmap, va, oldpde);

sys/amd64/amd64/pmap.c

7530

if (pmap_insert_pt_page(pmap, mt, false,

sys/amd64/amd64/pmap.c

7544

if (pmap_remove_ptes(pmap, va, va + NBPDR, pde, &free,

sys/amd64/amd64/pmap.c

7546

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

7563

if ((newpde & PG_W) != 0 && pmap != kernel_pmap) {

sys/amd64/amd64/pmap.c

7564

uwptpg = pmap_alloc_pt_page(pmap, pmap_pde_pindex(va),

sys/amd64/amd64/pmap.c

7567

pmap_abort_ptp(pmap, va, pdpg);

sys/amd64/amd64/pmap.c

7570

if (pmap_insert_pt_page(pmap, uwptpg, true, false)) {

sys/amd64/amd64/pmap.c

7571

pmap_free_pt_page(pmap, uwptpg, false);

sys/amd64/amd64/pmap.c

7572

pmap_abort_ptp(pmap, va, pdpg);

sys/amd64/amd64/pmap.c

7582

if (!pmap_pv_insert_pde(pmap, va, newpde, flags, lockp)) {

sys/amd64/amd64/pmap.c

7584

pmap_abort_ptp(pmap, va, pdpg);

sys/amd64/amd64/pmap.c

7589

mt = pmap_remove_pt_page(pmap, va);

sys/amd64/amd64/pmap.c

7594

mt = pmap_remove_pt_page(pmap, va);

sys/amd64/amd64/pmap.c

7599

pmap_free_pt_page(pmap, uwptpg, false);

sys/amd64/amd64/pmap.c

7602

" in pmap %p", va, pmap);

sys/amd64/amd64/pmap.c

7615

pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE;

sys/amd64/amd64/pmap.c

7616

pmap_resident_count_adj(pmap, NBPDR / PAGE_SIZE);

sys/amd64/amd64/pmap.c

7626

va, pmap);

sys/amd64/amd64/pmap.c

7643

pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,

sys/amd64/amd64/pmap.c

7659

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

7663

m->psind == 1 && pmap_ps_enabled(pmap) &&

sys/amd64/amd64/pmap.c

7664

((rv = pmap_enter_2mpage(pmap, va, m, prot, &lock)) ==

sys/amd64/amd64/pmap.c

7668

mpte = pmap_enter_quick_locked(pmap, va, m, prot,

sys/amd64/amd64/pmap.c

7675

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

7688

pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/amd64/amd64/pmap.c

7693

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

7694

(void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock);

sys/amd64/amd64/pmap.c

7697

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

7701

pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/amd64/amd64/pmap.c

7710

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7711

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

7737

pdpe = pmap_pdpe(pmap, va);

sys/amd64/amd64/pmap.c

7748

mpte = pmap_allocpte_alloc(pmap,

sys/amd64/amd64/pmap.c

7754

mpte = pmap_allocpte_alloc(pmap, ptepindex,

sys/amd64/amd64/pmap.c

7776

!pmap_try_insert_pv_entry(pmap, va, m, lockp)) {

sys/amd64/amd64/pmap.c

7778

pmap_abort_ptp(pmap, va, mpte);

sys/amd64/amd64/pmap.c

7785

pmap_resident_count_adj(pmap, 1);

sys/amd64/amd64/pmap.c

7788

pmap_cache_bits(pmap, m->md.pat_mode, false);

sys/amd64/amd64/pmap.c

7794

newpte |= PG_U | pmap_pkru_get(pmap, va);

sys/amd64/amd64/pmap.c

7807

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

7813

if (pmap_promote_pde(pmap, pde, va, mpte, lockp))

sys/amd64/amd64/pmap.c

7842

pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,

sys/amd64/amd64/pmap.c

7852

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

7853

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

7854

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7855

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

7861

if (!pmap_ps_enabled(pmap))

sys/amd64/amd64/pmap.c

7899

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

7900

for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, true);

sys/amd64/amd64/pmap.c

7902

pde = pmap_alloc_pde(pmap, addr, &pdpg, NULL);

sys/amd64/amd64/pmap.c

7917

pmap_resident_count_adj(pmap, NBPDR / PAGE_SIZE);

sys/amd64/amd64/pmap.c

7928

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

7945

pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/amd64/amd64/pmap.c

7953

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

7954

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

7955

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

7957

pml4e = pmap_pml4e(pmap, sva);

sys/amd64/amd64/pmap.c

7976

MPASS(pmap != kernel_pmap); /* XXXKIB */

sys/amd64/amd64/pmap.c

7979

pmap->pm_stats.wired_count -= NBPDP / PAGE_SIZE;

sys/amd64/amd64/pmap.c

8000

pmap->pm_stats.wired_count -= NBPDR /

sys/amd64/amd64/pmap.c

8003

} else if (!pmap_demote_pde(pmap, pde, sva))

sys/amd64/amd64/pmap.c

8022

pmap->pm_stats.wired_count--;

sys/amd64/amd64/pmap.c

8025

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8337

pmap_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/amd64/amd64/pmap.c

8351

if (PV_PMAP(pv) == pmap) {

sys/amd64/amd64/pmap.c

8362

if (PV_PMAP(pv) == pmap) {

sys/amd64/amd64/pmap.c

8386

pmap_t pmap;

sys/amd64/amd64/pmap.c

8398

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8399

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8402

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8405

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8409

pte = pmap_pte(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8412

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8417

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8418

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8422

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8426

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8430

pte = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8433

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8487

pmap_remove_pages(pmap_t pmap)

sys/amd64/amd64/pmap.c

8513

KASSERT(pmap == PCPU_GET(curpmap), ("non-current pmap %p", pmap));

sys/amd64/amd64/pmap.c

8521

CPU_AND(&other_cpus, &other_cpus, &pmap->pm_active);

sys/amd64/amd64/pmap.c

8523

KASSERT(CPU_EMPTY(&other_cpus), ("pmap active %p", pmap));

sys/amd64/amd64/pmap.c

8528

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

8529

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

8530

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

8535

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8536

TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {

sys/amd64/amd64/pmap.c

8550

pte = pmap_pdpe(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8627

pmap_resident_count_adj(pmap, -NBPDR / PAGE_SIZE);

sys/amd64/amd64/pmap.c

8637

mpte = pmap_remove_pt_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8641

pmap_pt_page_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

8648

pmap_resident_count_adj(pmap, -1);

sys/amd64/amd64/pmap.c

8659

pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);

sys/amd64/amd64/pmap.c

8669

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/amd64/amd64/pmap.c

8675

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

8676

pmap_pkru_deassign_all(pmap);

sys/amd64/amd64/pmap.c

8678

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8690

pmap_t pmap;

sys/amd64/amd64/pmap.c

8699

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8700

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8703

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8706

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8710

pte = pmap_pte(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8713

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

8714

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

8718

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

8719

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

8723

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8730

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8731

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8735

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8739

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8743

pte = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8746

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

8747

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

8751

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

8752

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

8756

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8794

pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/amd64/amd64/pmap.c

8800

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

8807

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8808

pde = pmap_pde(pmap, addr);

sys/amd64/amd64/pmap.c

8813

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8839

pmap_t pmap;

sys/amd64/amd64/pmap.c

8860

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8861

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8864

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8867

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8871

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

8873

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

8875

(void)pmap_demote_pde_locked(pmap, pde, va, &lock);

sys/amd64/amd64/pmap.c

8879

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8882

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8883

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8887

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8891

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8895

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

8896

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

8897

pde = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8909

pmap_invalidate_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

8911

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8943

pmap_t pmap;

sys/amd64/amd64/pmap.c

8969

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

8970

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

8973

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

8976

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

8980

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

8981

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

8982

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

8984

pde = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9014

(uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&

sys/amd64/amd64/pmap.c

9016

if (safe_to_clear_referenced(pmap, oldpde)) {

sys/amd64/amd64/pmap.c

9018

pmap_invalidate_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9020

} else if (pmap_demote_pde_locked(pmap, pde,

sys/amd64/amd64/pmap.c

9034

pmap_remove_pte(pmap, pte, va, *pde,

sys/amd64/amd64/pmap.c

9036

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

9057

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9074

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

9075

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

9079

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

9082

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9086

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

9087

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

9088

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

9089

pde = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9097

if (safe_to_clear_referenced(pmap, *pte)) {

sys/amd64/amd64/pmap.c

9099

pmap_invalidate_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9108

pmap_remove_pte(pmap, pte, pv->pv_va,

sys/amd64/amd64/pmap.c

9110

pmap_invalidate_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9121

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9142

pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)

sys/amd64/amd64/pmap.c

9162

if (pmap_emulate_ad_bits(pmap))

sys/amd64/amd64/pmap.c

9165

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

9166

PG_G = pmap_global_bit(pmap);

sys/amd64/amd64/pmap.c

9167

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

9168

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

9169

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

9172

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

9174

pml4e = pmap_pml4e(pmap, sva);

sys/amd64/amd64/pmap.c

9202

if (!pmap_demote_pde_locked(pmap, pde, sva, &lock)) {

sys/amd64/amd64/pmap.c

9231

pmap_remove_pte(pmap, pte, va, *pde, NULL,

sys/amd64/amd64/pmap.c

9269

pmap_invalidate_range(pmap, va, sva);

sys/amd64/amd64/pmap.c

9274

pmap_invalidate_range(pmap, va, sva);

sys/amd64/amd64/pmap.c

9277

pmap_invalidate_all(pmap);

sys/amd64/amd64/pmap.c

9278

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9289

pmap_t pmap;

sys/amd64/amd64/pmap.c

9309

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

9310

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

9313

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

9316

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9320

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

9321

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

9323

pde = pmap_pde(pmap, va);

sys/amd64/amd64/pmap.c

9327

pmap_demote_pde_locked(pmap, pde, va, &lock) &&

sys/amd64/amd64/pmap.c

9337

pmap_invalidate_page(pmap, va);

sys/amd64/amd64/pmap.c

9339

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9342

pmap = PV_PMAP(pv);

sys/amd64/amd64/pmap.c

9343

if (!PMAP_TRYLOCK(pmap)) {

sys/amd64/amd64/pmap.c

9347

PMAP_LOCK(pmap);

sys/amd64/amd64/pmap.c

9350

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9354

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

9355

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

9356

pde = pmap_pde(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9362

pmap_invalidate_page(pmap, pv->pv_va);

sys/amd64/amd64/pmap.c

9364

PMAP_UNLOCK(pmap);

sys/amd64/amd64/pmap.c

9535

pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va, vm_page_t m)

sys/amd64/amd64/pmap.c

9543

PG_A = pmap_accessed_bit(pmap);

sys/amd64/amd64/pmap.c

9544

PG_M = pmap_modified_bit(pmap);

sys/amd64/amd64/pmap.c

9545

PG_V = pmap_valid_bit(pmap);

sys/amd64/amd64/pmap.c

9546

PG_RW = pmap_rw_bit(pmap);

sys/amd64/amd64/pmap.c

9548

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/amd64/amd64/pmap.c

9553

pdpg = pmap_alloc_pt_page(pmap, va >> PDPSHIFT,

sys/amd64/amd64/pmap.c

9558

va, pmap);

sys/amd64/amd64/pmap.c

9564

pmap_pt_page_count_adj(pmap, 1);

sys/amd64/amd64/pmap.c

9591

pmap_invalidate_page(pmap, (vm_offset_t)vtopde(va));

sys/amd64/amd64/pmap.c

9595

" in pmap %p", va, pmap);

sys/amd64/include/pcpu.h

59

struct pmap *pc_curpmap; \

sys/amd64/include/pmap.h

346

typedef struct pmap *pmap_t;

sys/amd64/include/pmap.h

349

extern struct pmap kernel_pmap_store;

sys/amd64/include/pmap.h

352

#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)

sys/amd64/include/pmap.h

353

#define PMAP_LOCK_ASSERT(pmap, type) \

sys/amd64/include/pmap.h

354

mtx_assert(&(pmap)->pm_mtx, (type))

sys/amd64/include/pmap.h

355

#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)

sys/amd64/include/pmap.h

356

#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \

sys/amd64/include/pmap.h

358

#define PMAP_LOCKED(pmap) mtx_owned(&(pmap)->pm_mtx)

sys/amd64/include/pmap.h

359

#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)

sys/amd64/include/pmap.h

360

#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)

sys/amd64/include/pmap.h

361

#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)

sys/amd64/include/pmap.h

363

int pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags);

sys/amd64/include/pmap.h

364

int pmap_emulate_accessed_dirty(pmap_t pmap, vm_offset_t va, int ftype);

sys/amd64/include/pmap.h

390

void pmap_activate_boot(pmap_t pmap);

sys/amd64/include/pmap.h

394

int pmap_cache_bits(pmap_t pmap, int mode, bool is_pde);

sys/amd64/include/pmap.h

418

bool pmap_ps_enabled(pmap_t pmap);

sys/amd64/include/pmap.h

427

void pmap_get_mapping(pmap_t pmap, vm_offset_t va, uint64_t *ptr, int *num);

sys/amd64/include/pmap.h

437

int pmap_pkru_clear(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);

sys/amd64/include/pmap.h

438

int pmap_pkru_set(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/amd64/include/pmap.h

455

pmap_invalidate_cpu_mask(pmap_t pmap)

sys/amd64/include/pmap.h

457

return (&pmap->pm_active);

sys/amd64/include/pmap.h

468

pmap_invlpg(pmap_t pmap, vm_offset_t va)

sys/amd64/include/pmap.h

470

if (pmap == kernel_pmap && PCPU_GET(pcid_invlpg_workaround)) {

sys/amd64/include/pmap.h

483

pmap_get_pcid(pmap_t pmap)

sys/amd64/include/pmap.h

488

pcidp = zpcpu_get(pmap->pm_pcidp);

sys/amd64/include/pmap.h

511

typedef void (*smp_invl_local_cb_t)(struct pmap *, vm_offset_t addr1,

sys/amd64/include/vmm.h

166

struct pmap;

sys/amd64/include/vmm.h

177

DECLARE_VMMOPS_FUNC(void *, init, (struct vm *vm, struct pmap *pmap));

sys/amd64/include/vmm.h

179

struct pmap *pmap, struct vm_eventinfo *info));

sys/amd64/vmm/amd/npt.c

67

npt_pinit(pmap_t pmap)

sys/amd64/vmm/amd/npt.c

70

return (pmap_pinit_type(pmap, PT_RVI, npt_flags));

sys/amd64/vmm/amd/svm.c

1944

svm_pmap_activate(struct svm_vcpu *vcpu, pmap_t pmap)

sys/amd64/vmm/amd/svm.c

1952

CPU_SET_ATOMIC(cpu, &pmap->pm_active);

sys/amd64/vmm/amd/svm.c

1953

smr_enter(pmap->pm_eptsmr);

sys/amd64/vmm/amd/svm.c

1993

eptgen = atomic_load_long(&pmap->pm_eptgen);

sys/amd64/vmm/amd/svm.c

2047

svm_pmap_deactivate(pmap_t pmap)

sys/amd64/vmm/amd/svm.c

2049

smr_exit(pmap->pm_eptsmr);

sys/amd64/vmm/amd/svm.c

2050

CPU_CLR_ATOMIC(curcpu, &pmap->pm_active);

sys/amd64/vmm/amd/svm.c

2125

svm_run(void *vcpui, register_t rip, pmap_t pmap, struct vm_eventinfo *evinfo)

sys/amd64/vmm/amd/svm.c

2234

svm_pmap_activate(vcpu, pmap);

sys/amd64/vmm/amd/svm.c

2246

svm_pmap_deactivate(pmap);

sys/amd64/vmm/amd/svm.c

591

svm_init(struct vm *vm, pmap_t pmap)

sys/amd64/vmm/amd/svm.c

607

svm_sc->nptp = vtophys(pmap->pm_pmltop);

sys/amd64/vmm/intel/ept.c

172

ept_pinit(pmap_t pmap)

sys/amd64/vmm/intel/ept.c

175

return (pmap_pinit_type(pmap, PT_EPT, ept_pmap_flags));

sys/amd64/vmm/intel/vmx.c

1056

vmx_init(struct vm *vm, pmap_t pmap)

sys/amd64/vmm/intel/vmx.c

1064

vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pmltop));

sys/amd64/vmm/intel/vmx.c

1127

vmx->pmap = pmap;

sys/amd64/vmm/intel/vmx.c

1245

vcpu->ctx.pmap = vmx->pmap;

sys/amd64/vmm/intel/vmx.c

1289

vmx_invvpid(struct vmx *vmx, struct vmx_vcpu *vcpu, pmap_t pmap, int running)

sys/amd64/vmm/intel/vmx.c

1327

if (atomic_load_long(&pmap->pm_eptgen) == vmx->eptgen[curcpu]) {

sys/amd64/vmm/intel/vmx.c

1346

vmx_set_pcpu_defaults(struct vmx *vmx, struct vmx_vcpu *vcpu, pmap_t pmap)

sys/amd64/vmm/intel/vmx.c

1361

vmx_invvpid(vmx, vcpu, pmap, 1);

sys/amd64/vmm/intel/vmx.c

3013

vmx_pmap_activate(struct vmx *vmx, pmap_t pmap)

sys/amd64/vmm/intel/vmx.c

3020

CPU_SET_ATOMIC(cpu, &pmap->pm_active);

sys/amd64/vmm/intel/vmx.c

3021

smr_enter(pmap->pm_eptsmr);

sys/amd64/vmm/intel/vmx.c

3022

eptgen = atomic_load_long(&pmap->pm_eptgen);

sys/amd64/vmm/intel/vmx.c

3031

vmx_pmap_deactivate(struct vmx *vmx, pmap_t pmap)

sys/amd64/vmm/intel/vmx.c

3033

smr_exit(pmap->pm_eptsmr);

sys/amd64/vmm/intel/vmx.c

3034

CPU_CLR_ATOMIC(curcpu, &pmap->pm_active);

sys/amd64/vmm/intel/vmx.c

3038

vmx_run(void *vcpui, register_t rip, pmap_t pmap, struct vm_eventinfo *evinfo)

sys/amd64/vmm/intel/vmx.c

3059

KASSERT(vmxctx->pmap == pmap,

sys/amd64/vmm/intel/vmx.c

3060

("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));

sys/amd64/vmm/intel/vmx.c

3077

vmx_set_pcpu_defaults(vmx, vcpu, pmap);

sys/amd64/vmm/intel/vmx.c

3190

vmx_pmap_activate(vmx, pmap);

sys/amd64/vmm/intel/vmx.c

3195

vmx_pmap_deactivate(vmx, pmap);

sys/amd64/vmm/intel/vmx.c

3442

pmap_t pmap;

sys/amd64/vmm/intel/vmx.c

3498

pmap = vcpu->ctx.pmap;

sys/amd64/vmm/intel/vmx.c

3499

vmx_invvpid(vmx, vcpu, pmap, running);

sys/amd64/vmm/intel/vmx.h

147

pmap_t pmap;

sys/amd64/vmm/intel/vmx.h

38

struct pmap;

sys/amd64/vmm/intel/vmx.h

86

struct pmap *pmap;

sys/amd64/vmm/intel/vmx_genassym.c

77

ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));

sys/amd64/vmm/intel/vmx_genassym.c

78

ASSYM(PM_EPTGEN, offsetof(struct pmap, pm_eptgen));

sys/amd64/vmm/vmm.c

1132

pmap_t pmap;

sys/amd64/vmm/vmm.c

1142

pmap = vmspace_pmap(vm_vmspace(vm));

sys/amd64/vmm/vmm.c

1150

KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active),

sys/amd64/vmm/vmm.c

1161

error = vmmops_run(vcpu->cookie, vcpu->nextrip, pmap, &evinfo);

sys/amd64/vmm/vmm.c

133

DEFINE_VMMOPS_IFUNC(void *, init, (struct vm *vm, struct pmap *pmap))

sys/amd64/vmm/vmm.c

134

DEFINE_VMMOPS_IFUNC(int, run, (void *vcpui, register_t rip, struct pmap *pmap,

sys/amd64/vmm/vmm.c

470

pmap_t pmap;

sys/amd64/vmm/vmm.c

477

pmap = vmspace_pmap(vm_vmspace(vm));

sys/amd64/vmm/vmm.c

491

hpa = pmap_extract(pmap, gpa);

sys/arm/arm/busdma_machdep.c

158

static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, pmap_t pmap,

sys/arm/arm/busdma_machdep.c

813

_bus_dmamap_count_pages(bus_dma_tag_t dmat, pmap_t pmap, bus_dmamap_t map,

sys/arm/arm/busdma_machdep.c

836

if (__predict_true(pmap == kernel_pmap))

sys/arm/arm/busdma_machdep.c

839

paddr = pmap_extract(pmap, vaddr);

sys/arm/arm/busdma_machdep.c

943

bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,

sys/arm/arm/busdma_machdep.c

972

_bus_dmamap_count_pages(dmat, pmap, map, buf, buflen, flags);

sys/arm/arm/busdma_machdep.c

990

if (__predict_true(pmap == kernel_pmap)) {

sys/arm/arm/busdma_machdep.c

994

curaddr = pmap_extract(pmap, (vm_offset_t)vaddr);

sys/arm/arm/db_interface.c

160

struct pmap *pmap;

sys/arm/arm/db_interface.c

162

if (!p || !p->p_vmspace || !p->p_vmspace->vm_map.pmap ||

sys/arm/arm/db_interface.c

169

pmap = kernel_pmap;

sys/arm/arm/db_interface.c

171

pmap = p->p_vmspace->vm_map.pmap;

sys/arm/arm/db_interface.c

173

return (pmap_extract(pmap, addr) == 0);

sys/arm/arm/genassym.c

123

ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));

sys/arm/arm/pmap-v6.c

1361

pmap_t pmap;

sys/arm/arm/pmap-v6.c

1365

LIST_FOREACH(pmap, &allpmaps, pm_list) {

sys/arm/arm/pmap-v6.c

1366

pte2p = pmap_pt2tab_entry(pmap, va);

sys/arm/arm/pmap-v6.c

1379

pmap_t pmap;

sys/arm/arm/pmap-v6.c

1383

LIST_FOREACH(pmap, &allpmaps, pm_list) {

sys/arm/arm/pmap-v6.c

1384

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

1488

pmap_tlb_flush(pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

1491

if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))

sys/arm/arm/pmap-v6.c

1496

pmap_tlb_flush_range(pmap_t pmap, vm_offset_t sva, vm_size_t size)

sys/arm/arm/pmap-v6.c

1499

if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))

sys/arm/arm/pmap-v6.c

1564

SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,

sys/arm/arm/pmap-v6.c

1586

pmap_ps_enabled(pmap_t pmap __unused)

sys/arm/arm/pmap-v6.c

1620

pmap_ttb_get(pmap_t pmap)

sys/arm/arm/pmap-v6.c

1623

return (vtophys(pmap->pm_pt1) | ttb_flags);

sys/arm/arm/pmap-v6.c

1702

pmap_pt2pg_init(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/arm/arm/pmap-v6.c

1719

if (pmap == kernel_pmap)

sys/arm/arm/pmap-v6.c

1722

pte2p = pmap_pt2tab_entry(pmap, va);

sys/arm/arm/pmap-v6.c

1871

pmap_is_current(pmap_t pmap)

sys/arm/arm/pmap-v6.c

1874

return (pmap == kernel_pmap ||

sys/arm/arm/pmap-v6.c

1875

(pmap == vmspace_pmap(curthread->td_proc->p_vmspace)));

sys/arm/arm/pmap-v6.c

1883

pmap_pte2(pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

1888

pte1 = pte1_load(pmap_pte1(pmap, va));

sys/arm/arm/pmap-v6.c

1893

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

1931

pmap_pte2_quick(pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

1936

pte1 = pte1_load(pmap_pte1(pmap, va));

sys/arm/arm/pmap-v6.c

1941

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

1976

pmap_extract(pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

1982

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

1983

pte1 = pte1_load(pmap_pte1(pmap, va));

sys/arm/arm/pmap-v6.c

1987

pte2p = pmap_pte2(pmap, va);

sys/arm/arm/pmap-v6.c

1992

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

2004

pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/arm/arm/pmap-v6.c

2012

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

2013

pte1 = pte1_load(pmap_pte1(pmap, va));

sys/arm/arm/pmap-v6.c

2022

pte2p = pmap_pte2(pmap, va);

sys/arm/arm/pmap-v6.c

2033

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

2165

pmap_pinit0(pmap_t pmap)

sys/arm/arm/pmap-v6.c

2167

PDEBUG(1, printf("%s: pmap = %p\n", __func__, pmap));

sys/arm/arm/pmap-v6.c

2169

PMAP_LOCK_INIT(pmap);

sys/arm/arm/pmap-v6.c

2180

pmap->pm_pt1 = kern_pt1;

sys/arm/arm/pmap-v6.c

2181

pmap->pm_pt2tab = kern_pt2tab;

sys/arm/arm/pmap-v6.c

2182

CPU_ZERO(&pmap->pm_active);

sys/arm/arm/pmap-v6.c

2183

PCPU_SET(curpmap, pmap);

sys/arm/arm/pmap-v6.c

2184

TAILQ_INIT(&pmap->pm_pvchunk);

sys/arm/arm/pmap-v6.c

2185

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/arm/arm/pmap-v6.c

2186

CPU_SET(0, &pmap->pm_active);

sys/arm/arm/pmap-v6.c

2216

pmap_pinit(pmap_t pmap)

sys/arm/arm/pmap-v6.c

2223

PDEBUG(6, printf("%s: pmap = %p, pm_pt1 = %p\n", __func__, pmap,

sys/arm/arm/pmap-v6.c

2224

pmap->pm_pt1));

sys/arm/arm/pmap-v6.c

2246

if (pmap->pm_pt1 == NULL) {

sys/arm/arm/pmap-v6.c

2247

pmap->pm_pt1 = kmem_alloc_contig(NB_IN_PT1,

sys/arm/arm/pmap-v6.c

2249

if (pmap->pm_pt1 == NULL)

sys/arm/arm/pmap-v6.c

2252

if (pmap->pm_pt2tab == NULL) {

sys/arm/arm/pmap-v6.c

2263

pmap->pm_pt2tab = kmem_alloc_attr(NB_IN_PT2TAB,

sys/arm/arm/pmap-v6.c

2265

if (pmap->pm_pt2tab == NULL) {

sys/arm/arm/pmap-v6.c

2271

kmem_free(pmap->pm_pt1, NB_IN_PT1);

sys/arm/arm/pmap-v6.c

2272

pmap->pm_pt1 = NULL;

sys/arm/arm/pmap-v6.c

2290

pte1_copy_nosync(kern_pt1, pmap->pm_pt1, KERNBASE,

sys/arm/arm/pmap-v6.c

2292

pte1_copy_nosync(kern_pt1, pmap->pm_pt1, vm_max_kernel_address,

sys/arm/arm/pmap-v6.c

2294

pt2tab_copy_nosync(kern_pt2tab, pmap->pm_pt2tab, KERNBASE,

sys/arm/arm/pmap-v6.c

2296

pt2tab_copy_nosync(kern_pt2tab, pmap->pm_pt2tab, vm_max_kernel_address,

sys/arm/arm/pmap-v6.c

2298

LIST_INSERT_HEAD(&allpmaps, pmap, pm_list);

sys/arm/arm/pmap-v6.c

2306

pt2tab_pa = vtophys(pmap->pm_pt2tab);

sys/arm/arm/pmap-v6.c

2307

pte2p = pmap_pt2tab_entry(pmap, (vm_offset_t)PT2MAP);

sys/arm/arm/pmap-v6.c

2313

pte1p = pmap_pte1(pmap, (vm_offset_t)PT2MAP);

sys/arm/arm/pmap-v6.c

2321

pte1_sync_range(pmap->pm_pt1, NB_IN_PT1);

sys/arm/arm/pmap-v6.c

2322

pte2_sync_range(pmap->pm_pt2tab, NB_IN_PT2TAB);

sys/arm/arm/pmap-v6.c

2324

CPU_ZERO(&pmap->pm_active);

sys/arm/arm/pmap-v6.c

2325

TAILQ_INIT(&pmap->pm_pvchunk);

sys/arm/arm/pmap-v6.c

2326

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/arm/arm/pmap-v6.c

2349

pmap_release(pmap_t pmap)

sys/arm/arm/pmap-v6.c

2354

KASSERT(pmap->pm_stats.resident_count == 0,

sys/arm/arm/pmap-v6.c

2356

pmap->pm_stats.resident_count));

sys/arm/arm/pmap-v6.c

2357

KASSERT(pt2tab_user_is_empty(pmap->pm_pt2tab),

sys/arm/arm/pmap-v6.c

2359

KASSERT(CPU_EMPTY(&pmap->pm_active),

sys/arm/arm/pmap-v6.c

2360

("%s: pmap %p is active on some CPU(s)", __func__, pmap));

sys/arm/arm/pmap-v6.c

2363

LIST_REMOVE(pmap, pm_list);

sys/arm/arm/pmap-v6.c

2369

bzero((char *)pmap->pm_pt1 + start, end - start);

sys/arm/arm/pmap-v6.c

2373

bzero((char *)pmap->pm_pt2tab + start, end - start);

sys/arm/arm/pmap-v6.c

2502

_pmap_allocpte2(pmap_t pmap, vm_offset_t va, u_int flags)

sys/arm/arm/pmap-v6.c

2511

pte1p = pmap->pm_pt1 + pte1_idx;

sys/arm/arm/pmap-v6.c

2517

pte2 = pt2tab_load(pmap_pt2tab_entry(pmap, va));

sys/arm/arm/pmap-v6.c

2525

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

2529

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

2539

pmap->pm_stats.resident_count++;

sys/arm/arm/pmap-v6.c

2540

pt2pg_pa = pmap_pt2pg_init(pmap, va, m);

sys/arm/arm/pmap-v6.c

2554

pmap_allocpte2(pmap_t pmap, vm_offset_t va, u_int flags)

sys/arm/arm/pmap-v6.c

2562

pte1p = pmap->pm_pt1 + pte1_idx;

sys/arm/arm/pmap-v6.c

257

struct pmap kernel_pmap_store;

sys/arm/arm/pmap-v6.c

2570

(void)pmap_demote_pte1(pmap, pte1p, va);

sys/arm/arm/pmap-v6.c

258

LIST_HEAD(pmaplist, pmap);

sys/arm/arm/pmap-v6.c

2592

m = _pmap_allocpte2(pmap, va, flags);

sys/arm/arm/pmap-v6.c

2624

pmap_unwire_pt2pg(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/arm/arm/pmap-v6.c

2631

("%s: pmap %p PT2PG %p wired", __func__, pmap, m));

sys/arm/arm/pmap-v6.c

2640

("%s: pmap %p va %#x PT2PG %p bad index", __func__, pmap, va, m));

sys/arm/arm/pmap-v6.c

2641

pte1p = pmap->pm_pt1 + m->pindex;

sys/arm/arm/pmap-v6.c

2644

("%s: pmap %p PT2 %u (PG %p) wired", __func__, pmap, i, m));

sys/arm/arm/pmap-v6.c

2651

pmap_tlb_flush(pmap, (m->pindex + i) << PTE1_SHIFT);

sys/arm/arm/pmap-v6.c

2657

pmap, va, opte1, i));

sys/arm/arm/pmap-v6.c

2664

pte2p = pmap_pt2tab_entry(pmap, va);

sys/arm/arm/pmap-v6.c

2666

pmap_tlb_flush(pmap, pt2map_pt2pg(va));

sys/arm/arm/pmap-v6.c

2669

pmap->pm_stats.resident_count--;

sys/arm/arm/pmap-v6.c

2687

pmap_unwire_pt2(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/arm/arm/pmap-v6.c

2697

pmap_unwire_pt2pg(pmap, va, m);

sys/arm/arm/pmap-v6.c

2710

pmap_unwire_pt2_all(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm/arm/pmap-v6.c

2745

pmap_unwire_pt2pg(pmap, va, m);

sys/arm/arm/pmap-v6.c

2755

pmap_unuse_pt2(pmap_t pmap, vm_offset_t va, struct spglist *free)

sys/arm/arm/pmap-v6.c

2762

pte1 = pte1_load(pmap_pte1(pmap, va));

sys/arm/arm/pmap-v6.c

2764

return (pmap_unwire_pt2(pmap, va, mpte, free));

sys/arm/arm/pmap-v6.c

2844

pmap_t pmap;

sys/arm/arm/pmap-v6.c

2854

pmap = NULL;

sys/arm/arm/pmap-v6.c

2861

if (pmap != pc->pc_pmap) {

sys/arm/arm/pmap-v6.c

2862

if (pmap != NULL) {

sys/arm/arm/pmap-v6.c

2863

if (pmap != locked_pmap)

sys/arm/arm/pmap-v6.c

2864

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

2866

pmap = pc->pc_pmap;

sys/arm/arm/pmap-v6.c

2868

if (pmap > locked_pmap)

sys/arm/arm/pmap-v6.c

2869

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

2870

else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {

sys/arm/arm/pmap-v6.c

2871

pmap = NULL;

sys/arm/arm/pmap-v6.c

2887

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

2890

pte2p = pmap_pte2(pmap, va);

sys/arm/arm/pmap-v6.c

2899

pmap, va));

sys/arm/arm/pmap-v6.c

2900

pmap_tlb_flush(pmap, va);

sys/arm/arm/pmap-v6.c

2916

pmap_unuse_pt2(pmap, va, &free);

sys/arm/arm/pmap-v6.c

2925

pmap->pm_stats.resident_count -= freed;

sys/arm/arm/pmap-v6.c

2929

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

2932

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,

sys/arm/arm/pmap-v6.c

2940

if (pmap == locked_pmap)

sys/arm/arm/pmap-v6.c

2957

if (pmap != NULL) {

sys/arm/arm/pmap-v6.c

2958

if (pmap != locked_pmap)

sys/arm/arm/pmap-v6.c

2959

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

2993

free_pv_entry(pmap_t pmap, pv_entry_t pv)

sys/arm/arm/pmap-v6.c

2999

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3014

if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=

sys/arm/arm/pmap-v6.c

3016

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

3017

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,

sys/arm/arm/pmap-v6.c

3022

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

3031

get_pv_entry(pmap_t pmap, bool try)

sys/arm/arm/pmap-v6.c

3041

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3050

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/arm/arm/pmap-v6.c

3067

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

3068

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

3085

m = pmap_pv_reclaim(pmap);

sys/arm/arm/pmap-v6.c

3093

pc->pc_pmap = pmap;

sys/arm/arm/pmap-v6.c

3099

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

3109

pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/arm/arm/pmap-v6.c

3114

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3115

pv = get_pv_entry(pmap, false);

sys/arm/arm/pmap-v6.c

3121

pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

3127

if (pmap == PV_PMAP(pv) && va == pv->pv_va) {

sys/arm/arm/pmap-v6.c

3136

pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

3140

pv = pmap_pvh_remove(pvh, pmap, va);

sys/arm/arm/pmap-v6.c

3142

free_pv_entry(pmap, pv);

sys/arm/arm/pmap-v6.c

3146

pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)

sys/arm/arm/pmap-v6.c

3151

pmap_pvh_free(&m->md, pmap, va);

sys/arm/arm/pmap-v6.c

3160

pmap_pv_demote_pte1(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)

sys/arm/arm/pmap-v6.c

3177

pv = pmap_pvh_remove(pvh, pmap, va);

sys/arm/arm/pmap-v6.c

3188

pmap_insert_entry(pmap, va, m);

sys/arm/arm/pmap-v6.c

3194

pmap_pv_promote_pte1(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)

sys/arm/arm/pmap-v6.c

3214

pv = pmap_pvh_remove(&m->md, pmap, va);

sys/arm/arm/pmap-v6.c

3223

pmap_pvh_free(&m->md, pmap, va);

sys/arm/arm/pmap-v6.c

3232

pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/arm/arm/pmap-v6.c

3237

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3239

(pv = get_pv_entry(pmap, true)) != NULL) {

sys/arm/arm/pmap-v6.c

3251

pmap_pv_insert_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t pte1, u_int flags)

sys/arm/arm/pmap-v6.c

3260

(pv = get_pv_entry(pmap, noreclaim)) == NULL)

sys/arm/arm/pmap-v6.c

3269

pmap_tlb_flush_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t npte1)

sys/arm/arm/pmap-v6.c

3274

pmap_tlb_flush_range(pmap, pte1_trunc(va), PTE1_SIZE);

sys/arm/arm/pmap-v6.c

3276

pmap_tlb_flush(pmap, pte1_trunc(va));

sys/arm/arm/pmap-v6.c

328

static bool pmap_demote_pte1(pmap_t pmap, pt1_entry_t *pte1p,

sys/arm/arm/pmap-v6.c

3295

pmap_t pmap;

sys/arm/arm/pmap-v6.c

330

static int pmap_enter_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t pte1,

sys/arm/arm/pmap-v6.c

3302

pmap = PCPU_GET(curpmap);

sys/arm/arm/pmap-v6.c

3303

if (pmap == NULL)

sys/arm/arm/pmap-v6.c

3304

pmap = kernel_pmap;

sys/arm/arm/pmap-v6.c

3313

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

3330

LIST_FOREACH(pmap, &allpmaps, pm_list) {

sys/arm/arm/pmap-v6.c

3331

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

3376

pmap_change_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va,

sys/arm/arm/pmap-v6.c

3380

if (pmap == kernel_pmap) {

sys/arm/arm/pmap-v6.c

3403

pmap_tlb_flush_pte1(pmap, va, npte1);

sys/arm/arm/pmap-v6.c

3410

pmap_change_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va,

sys/arm/arm/pmap-v6.c

3414

if (pmap == kernel_pmap) {

sys/arm/arm/pmap-v6.c

3428

pmap_tlb_flush_pte1(pmap, va, npte1);

sys/arm/arm/pmap-v6.c

3449

pmap_promote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va)

sys/arm/arm/pmap-v6.c

3458

pmap, va, pte1_load(pte1p), pte1p));

sys/arm/arm/pmap-v6.c

3460

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3467

fpte2p = pmap_pte2_quick(pmap, pte1_trunc(va));

sys/arm/arm/pmap-v6.c

3473

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3476

if (pte2_is_managed(fpte2) && pmap == kernel_pmap) {

sys/arm/arm/pmap-v6.c

3479

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3503

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3516

__func__, pteva, pmap);

sys/arm/arm/pmap-v6.c

3521

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3549

pmap_pv_promote_pte1(pmap, va, pte1_pa(npte1));

sys/arm/arm/pmap-v6.c

3554

pmap_change_pte1(pmap, pte1p, va, npte1);

sys/arm/arm/pmap-v6.c

3558

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3561

__func__, pmap, va, npte1, pte1_load(pte1p), pte1p));

sys/arm/arm/pmap-v6.c

3582

pmap_remove_kernel_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va)

sys/arm/arm/pmap-v6.c

3589

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3590

m = pmap_pt2_page(pmap, va);

sys/arm/arm/pmap-v6.c

3624

pmap_remove_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva,

sys/arm/arm/pmap-v6.c

3632

PDEBUG(6, printf("%s(%p): va %#x pte1 %#x at %p\n", __func__, pmap, sva,

sys/arm/arm/pmap-v6.c

3635

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3645

pmap_tlb_flush(pmap, sva);

sys/arm/arm/pmap-v6.c

3648

pmap->pm_stats.wired_count -= PTE1_SIZE / PAGE_SIZE;

sys/arm/arm/pmap-v6.c

3649

pmap->pm_stats.resident_count -= PTE1_SIZE / PAGE_SIZE;

sys/arm/arm/pmap-v6.c

3652

pmap_pvh_free(pvh, pmap, sva);

sys/arm/arm/pmap-v6.c

3665

if (pmap == kernel_pmap) {

sys/arm/arm/pmap-v6.c

3670

pmap_remove_kernel_pte1(pmap, pte1p, sva);

sys/arm/arm/pmap-v6.c

3676

m = pmap_pt2_page(pmap, sva);

sys/arm/arm/pmap-v6.c

3678

pmap_unwire_pt2_all(pmap, sva, m, free);

sys/arm/arm/pmap-v6.c

3701

pmap_demote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va)

sys/arm/arm/pmap-v6.c

3711

pmap, va, pte1_load(pte1p), pte1p));

sys/arm/arm/pmap-v6.c

3713

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

3718

if ((opte1 & PTE1_A) == 0 || (m = pmap_pt2_page(pmap, va)) == NULL) {

sys/arm/arm/pmap-v6.c

3730

pmap_remove_pte1(pmap, pte1p, pte1_trunc(va), &free);

sys/arm/arm/pmap-v6.c

3733

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3738

pmap->pm_stats.resident_count++;

sys/arm/arm/pmap-v6.c

3747

pt2pg_pa = pmap_pt2pg_init(pmap, va, m);

sys/arm/arm/pmap-v6.c

3771

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

3837

pmap_change_pte1(pmap, pte1p, va, npte1);

sys/arm/arm/pmap-v6.c

3849

pmap_pv_demote_pte1(pmap, va, pte1_pa(opte1));

sys/arm/arm/pmap-v6.c

3853

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

3856

__func__, pmap, va, npte1, pte1_load(pte1p), pte1p));

sys/arm/arm/pmap-v6.c

3873

pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/arm/arm/pmap-v6.c

3909

if (pmap != kernel_pmap)

sys/arm/arm/pmap-v6.c

3913

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

3920

rv = pmap_enter_pte1(pmap, va, PTE1_PA(pa) | ATTR_TO_L1(npte2) |

sys/arm/arm/pmap-v6.c

3930

mpte2 = pmap_allocpte2(pmap, va, flags);

sys/arm/arm/pmap-v6.c

3939

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

3942

pte2p = pmap_pte2_quick(pmap, va);

sys/arm/arm/pmap-v6.c

3960

pmap->pm_stats.wired_count++;

sys/arm/arm/pmap-v6.c

3962

pmap->pm_stats.wired_count--;

sys/arm/arm/pmap-v6.c

3983

__func__, pmap, va, opte2, opa, pa, flags, prot));

sys/arm/arm/pmap-v6.c

3993

pmap->pm_stats.wired_count--;

sys/arm/arm/pmap-v6.c

3998

pv = pmap_pvh_remove(&om->md, pmap, va);

sys/arm/arm/pmap-v6.c

4006

pmap->pm_stats.resident_count++;

sys/arm/arm/pmap-v6.c

4013

pv = get_pv_entry(pmap, false);

sys/arm/arm/pmap-v6.c

4018

free_pv_entry(pmap, pv);

sys/arm/arm/pmap-v6.c

402

pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)

sys/arm/arm/pmap-v6.c

4024

pmap->pm_stats.wired_count++;

sys/arm/arm/pmap-v6.c

4058

if ((prot & VM_PROT_EXECUTE) && pmap != kernel_pmap &&

sys/arm/arm/pmap-v6.c

4066

pmap_tlb_flush(pmap, va);

sys/arm/arm/pmap-v6.c

4091

panic("%s: pmap %p va %#x opte2 %x npte2 %x !!", __func__, pmap,

sys/arm/arm/pmap-v6.c

4104

pmap_promote_pte1(pmap, pte1p, va);

sys/arm/arm/pmap-v6.c

4111

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4119

pmap_remove_pte2(pmap_t pmap, pt2_entry_t *pte2p, vm_offset_t va,

sys/arm/arm/pmap-v6.c

4126

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

4130

pmap_tlb_flush(pmap, va);

sys/arm/arm/pmap-v6.c

4133

__func__, pmap, va, opte2));

sys/arm/arm/pmap-v6.c

4136

pmap->pm_stats.wired_count -= 1;

sys/arm/arm/pmap-v6.c

4137

pmap->pm_stats.resident_count -= 1;

sys/arm/arm/pmap-v6.c

4144

pmap_remove_entry(pmap, m, va);

sys/arm/arm/pmap-v6.c

4146

return (pmap_unuse_pt2(pmap, va, free));

sys/arm/arm/pmap-v6.c

4153

pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free)

sys/arm/arm/pmap-v6.c

4160

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

4161

if ((pte2p = pmap_pte2_quick(pmap, va)) == NULL ||

sys/arm/arm/pmap-v6.c

4164

pmap_remove_pte2(pmap, pte2p, va, free);

sys/arm/arm/pmap-v6.c

4174

pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm/arm/pmap-v6.c

4184

if (pmap->pm_stats.resident_count == 0)

sys/arm/arm/pmap-v6.c

4191

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4198

pte1 = pte1_load(pmap_pte1(pmap, sva));

sys/arm/arm/pmap-v6.c

4200

pmap_remove_page(pmap, sva, &free);

sys/arm/arm/pmap-v6.c

4212

if (pmap->pm_stats.resident_count == 0)

sys/arm/arm/pmap-v6.c

4215

pte1p = pmap_pte1(pmap, sva);

sys/arm/arm/pmap-v6.c

4231

pmap_remove_pte1(pmap, pte1p, sva, &free);

sys/arm/arm/pmap-v6.c

4233

} else if (!pmap_demote_pte1(pmap, pte1p, sva)) {

sys/arm/arm/pmap-v6.c

4246

" is not link", __func__, pmap, sva, pte1, pte1p));

sys/arm/arm/pmap-v6.c

4256

for (pte2p = pmap_pte2_quick(pmap, sva); sva != nextva;

sys/arm/arm/pmap-v6.c

4261

if (pmap_remove_pte2(pmap, pte2p, sva, &free))

sys/arm/arm/pmap-v6.c

4268

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4290

pmap_t pmap;

sys/arm/arm/pmap-v6.c

4306

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

4307

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4308

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

4309

(void)pmap_demote_pte1(pmap, pte1p, va);

sys/arm/arm/pmap-v6.c

4310

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4314

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

4315

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4316

pmap->pm_stats.resident_count--;

sys/arm/arm/pmap-v6.c

4317

pte1p = pmap_pte1(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

4320

pte2p = pmap_pte2_quick(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

4322

pmap_tlb_flush(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

4324

__func__, pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

4326

pmap->pm_stats.wired_count--;

sys/arm/arm/pmap-v6.c

4335

pmap_unuse_pt2(pmap, pv->pv_va, &free);

sys/arm/arm/pmap-v6.c

4337

free_pv_entry(pmap, pv);

sys/arm/arm/pmap-v6.c

4338

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4351

pmap_remove_pte1_quick(pmap_t pmap, pt1_entry_t pte1, pv_entry_t pv,

sys/arm/arm/pmap-v6.c

4372

pmap->pm_stats.resident_count -= PTE1_SIZE / PAGE_SIZE;

sys/arm/arm/pmap-v6.c

4380

mpt2pg = pmap_pt2_page(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

4382

pmap_unwire_pt2_all(pmap, pv->pv_va, mpt2pg, free);

sys/arm/arm/pmap-v6.c

4390

pmap_remove_pte2_quick(pmap_t pmap, pt2_entry_t pte2, pv_entry_t pv,

sys/arm/arm/pmap-v6.c

4409

pmap->pm_stats.resident_count--;

sys/arm/arm/pmap-v6.c

4416

pmap_unuse_pt2(pmap, pv->pv_va, free);

sys/arm/arm/pmap-v6.c

4427

pmap_remove_pages(pmap_t pmap)

sys/arm/arm/pmap-v6.c

4444

KASSERT(pmap == vmspace_pmap(curthread->td_proc->p_vmspace),

sys/arm/arm/pmap-v6.c

4445

("%s: non-current pmap %p", __func__, pmap));

sys/arm/arm/pmap-v6.c

4451

other_cpus = pmap->pm_active;

sys/arm/arm/pmap-v6.c

4455

("%s: pmap %p active on other cpus", __func__, pmap));

sys/arm/arm/pmap-v6.c

4460

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4462

TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {

sys/arm/arm/pmap-v6.c

4463

KASSERT(pc->pc_pmap == pmap, ("%s: wrong pmap %p %p",

sys/arm/arm/pmap-v6.c

4464

__func__, pmap, pc->pc_pmap));

sys/arm/arm/pmap-v6.c

4479

pte1p = pmap_pte1(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

4487

pmap_remove_pte1_quick(pmap, pte1, pv,

sys/arm/arm/pmap-v6.c

4497

pmap, pv->pv_va, pte2);

sys/arm/arm/pmap-v6.c

4506

pmap_remove_pte2_quick(pmap, pte2, pv,

sys/arm/arm/pmap-v6.c

4510

__func__, pmap, pv->pv_va, pte1);

sys/arm/arm/pmap-v6.c

4522

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm/arm/pmap-v6.c

4529

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4542

pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm/arm/pmap-v6.c

4554

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

4569

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

4601

mpt2pg = _pmap_allocpte2(pmap, va,

sys/arm/arm/pmap-v6.c

4634

!pmap_try_insert_pv_entry(pmap, va, m)) {

sys/arm/arm/pmap-v6.c

4637

if (pmap_unwire_pt2(pmap, va, mpt2pg, &free)) {

sys/arm/arm/pmap-v6.c

4638

pmap_tlb_flush(pmap, va);

sys/arm/arm/pmap-v6.c

4650

pmap->pm_stats.resident_count++;

sys/arm/arm/pmap-v6.c

4661

else if (m->md.pat_mode == VM_MEMATTR_WB_WA && pmap != kernel_pmap) {

sys/arm/arm/pmap-v6.c

4674

pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/arm/arm/pmap-v6.c

4678

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4679

(void)pmap_enter_quick_locked(pmap, va, m, prot, NULL);

sys/arm/arm/pmap-v6.c

4681

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4691

pmap_enter_1mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/arm/arm/pmap-v6.c

4696

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

4703

if (pmap != kernel_pmap)

sys/arm/arm/pmap-v6.c

4705

return (pmap_enter_pte1(pmap, va, pte1, PMAP_ENTER_NOSLEEP |

sys/arm/arm/pmap-v6.c

4718

pmap_enter_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t pte1, u_int flags,

sys/arm/arm/pmap-v6.c

4731

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

4732

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

4737

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

4747

pmap_remove_pte1(pmap, pte1p, va, &free);

sys/arm/arm/pmap-v6.c

4751

for (cur = va, pte2p = pmap_pte2_quick(pmap, va);

sys/arm/arm/pmap-v6.c

4756

if (pmap_remove_pte2(pmap, pte2p, cur, &free))

sys/arm/arm/pmap-v6.c

4767

if (!pmap_pv_insert_pte1(pmap, va, pte1, flags)) {

sys/arm/arm/pmap-v6.c

4769

__func__, va, pmap);

sys/arm/arm/pmap-v6.c

4782

pmap->pm_stats.wired_count += PTE1_SIZE / PAGE_SIZE;

sys/arm/arm/pmap-v6.c

4783

pmap->pm_stats.resident_count += PTE1_SIZE / PAGE_SIZE;

sys/arm/arm/pmap-v6.c

4790

pmap != kernel_pmap && (!pte1_is_section(opte1) ||

sys/arm/arm/pmap-v6.c

4801

pmap);

sys/arm/arm/pmap-v6.c

4818

pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,

sys/arm/arm/pmap-v6.c

4826

__func__, pmap, start, end, m_start, prot));

sys/arm/arm/pmap-v6.c

4835

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4840

pmap_enter_1mpage(pmap, va, m, prot)) {

sys/arm/arm/pmap-v6.c

4843

mpt2pg = pmap_enter_quick_locked(pmap, va, m, prot,

sys/arm/arm/pmap-v6.c

4849

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4858

pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,

sys/arm/arm/pmap-v6.c

4912

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

4914

pte1p = pmap_pte1(pmap, addr);

sys/arm/arm/pmap-v6.c

4917

pmap->pm_stats.resident_count += PTE1_SIZE /

sys/arm/arm/pmap-v6.c

4924

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

4932

pmap_protect_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva,

sys/arm/arm/pmap-v6.c

4939

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm/arm/pmap-v6.c

4963

pmap_tlb_flush(pmap, sva);

sys/arm/arm/pmap-v6.c

4972

pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)

sys/arm/arm/pmap-v6.c

4981

pmap_remove(pmap, sva, eva);

sys/arm/arm/pmap-v6.c

4989

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

4998

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5007

pte1p = pmap_pte1(pmap, sva);

sys/arm/arm/pmap-v6.c

5023

pmap_protect_pte1(pmap, pte1p, sva, prot);

sys/arm/arm/pmap-v6.c

5029

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5034

if (!pmap_demote_pte1(pmap, pte1p, sva)) {

sys/arm/arm/pmap-v6.c

5051

" is not link", __func__, pmap, sva, pte1, pte1p));

sys/arm/arm/pmap-v6.c

5061

for (pte2p = pmap_pte2_quick(pmap, sva); sva != nextva; pte2p++,

sys/arm/arm/pmap-v6.c

5089

pmap_tlb_flush(pmap, sva);

sys/arm/arm/pmap-v6.c

5097

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5108

pmap_t pmap;

sys/arm/arm/pmap-v6.c

5116

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5117

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5118

pte1 = pte1_load(pmap_pte1(pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

5125

pte2 = pte2_load(pmap_pte2_quick(pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

5129

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5170

pmap_t pmap;

sys/arm/arm/pmap-v6.c

5177

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5178

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5179

pte1 = pte1_load(pmap_pte1(pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

5185

pte2 = pte2_load(pmap_pte2_quick(pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

5188

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5230

pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/arm/arm/pmap-v6.c

5237

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5238

pte1 = pte1_load(pmap_pte1(pmap, addr));

sys/arm/arm/pmap-v6.c

5243

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5258

pmap_t pmap;

sys/arm/arm/pmap-v6.c

5265

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5266

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5267

pte1 = pte1_load(pmap_pte1(pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

5271

pte2 = pte2_load(pmap_pte2_quick(pmap, pv->pv_va));

sys/arm/arm/pmap-v6.c

5274

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5324

pmap_t pmap;

sys/arm/arm/pmap-v6.c

5341

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5342

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5343

pte1p = pmap_pte1(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5373

(uintptr_t)pmap) & (NPTE2_IN_PG - 1)) == 0 &&

sys/arm/arm/pmap-v6.c

5376

pmap_tlb_flush(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5380

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5394

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5395

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5396

pte1p = pmap_pte1(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5400

pte2p = pmap_pte2_quick(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5406

pmap_tlb_flush(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5409

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5433

pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm/arm/pmap-v6.c

5440

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

5448

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5454

pte1p = pmap_pte1(pmap, sva);

sys/arm/arm/pmap-v6.c

5474

pmap->pm_stats.wired_count -= PTE1_SIZE /

sys/arm/arm/pmap-v6.c

5481

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5487

if (!pmap_demote_pte1(pmap, pte1p, sva))

sys/arm/arm/pmap-v6.c

5499

" is not link", __func__, pmap, sva, pte1, pte1p));

sys/arm/arm/pmap-v6.c

5509

for (pte2p = pmap_pte2_quick(pmap, sva); sva != nextva; pte2p++,

sys/arm/arm/pmap-v6.c

5524

pmap->pm_stats.wired_count--;

sys/arm/arm/pmap-v6.c

5531

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5542

pmap_t pmap;

sys/arm/arm/pmap-v6.c

5560

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5561

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5562

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

5564

(void)pmap_demote_pte1(pmap, pte1p, va);

sys/arm/arm/pmap-v6.c

5565

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5569

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5570

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5571

pte1p = pmap_pte1(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5574

pte2p = pmap_pte2_quick(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5580

pmap_tlb_flush(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5582

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5595

pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)

sys/arm/arm/pmap-v6.c

5605

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

5613

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5618

pte1p = pmap_pte1(pmap, sva);

sys/arm/arm/pmap-v6.c

5628

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5633

if (!pmap_demote_pte1(pmap, pte1p, sva)) {

sys/arm/arm/pmap-v6.c

5648

pte2p = pmap_pte2_quick(pmap, sva);

sys/arm/arm/pmap-v6.c

5651

pmap_remove_pte2(pmap, pte2p, sva, NULL);

sys/arm/arm/pmap-v6.c

5656

for (pte2p = pmap_pte2_quick(pmap, sva); sva != pdnxt; pte2p++,

sys/arm/arm/pmap-v6.c

5677

pmap_tlb_flush(pmap, sva);

sys/arm/arm/pmap-v6.c

5684

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5695

pmap_t pmap;

sys/arm/arm/pmap-v6.c

5713

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5714

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5715

pte1p = pmap_pte1(pmap, va);

sys/arm/arm/pmap-v6.c

5718

if (pmap_demote_pte1(pmap, pte1p, va) &&

sys/arm/arm/pmap-v6.c

5726

pte2p = pmap_pte2_quick(pmap, va);

sys/arm/arm/pmap-v6.c

5731

pmap_tlb_flush(pmap, va);

sys/arm/arm/pmap-v6.c

5735

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5739

pmap = PV_PMAP(pv);

sys/arm/arm/pmap-v6.c

5740

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

5741

pte1p = pmap_pte1(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5744

pte2p = pmap_pte2_quick(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5747

pmap_tlb_flush(pmap, pv->pv_va);

sys/arm/arm/pmap-v6.c

5749

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

5835

pmap_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/arm/arm/pmap-v6.c

5847

if (PV_PMAP(pv) == pmap) {

sys/arm/arm/pmap-v6.c

5858

if (PV_PMAP(pv) == pmap) {

sys/arm/arm/pmap-v6.c

6204

pmap_t pmap, oldpmap;

sys/arm/arm/pmap-v6.c

6210

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/arm/arm/pmap-v6.c

6216

CPU_SET_ATOMIC(cpuid, &pmap->pm_active);

sys/arm/arm/pmap-v6.c

6219

CPU_SET(cpuid, &pmap->pm_active);

sys/arm/arm/pmap-v6.c

6222

ttb = pmap_ttb_get(pmap);

sys/arm/arm/pmap-v6.c

6229

PCPU_SET(curpmap, pmap);

sys/arm/arm/pmap-v6.c

6234

pmap_active_cpus(pmap_t pmap, cpuset_t *res)

sys/arm/arm/pmap-v6.c

6236

*res = pmap->pm_active;

sys/arm/arm/pmap-v6.c

6245

pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/arm/arm/pmap-v6.c

6253

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

6254

pte1p = pmap_pte1(pmap, addr);

sys/arm/arm/pmap-v6.c

6265

pte2p = pmap_pte2(pmap, addr);

sys/arm/arm/pmap-v6.c

6283

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6311

pmap_set_pcb_pagedir(pmap_t pmap, struct pcb *pcb)

sys/arm/arm/pmap-v6.c

6314

pcb->pcb_pagedir = pmap_ttb_get(pmap);

sys/arm/arm/pmap-v6.c

6370

pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t size)

sys/arm/arm/pmap-v6.c

6383

pa = pmap_extract(pmap, va); /* offset is preserved */

sys/arm/arm/pmap-v6.c

6412

pmap_fault(pmap_t pmap, vm_offset_t far, uint32_t fsr, int idx, bool usermode)

sys/arm/arm/pmap-v6.c

6417

if (pmap == NULL)

sys/arm/arm/pmap-v6.c

6418

pmap = kernel_pmap;

sys/arm/arm/pmap-v6.c

6426

if (__predict_false(IN_RANGE2(far, pmap->pm_pt1, NB_IN_PT1))) {

sys/arm/arm/pmap-v6.c

6435

__func__, pmap, pmap->pm_pt1, far);

sys/arm/arm/pmap-v6.c

6451

__func__, pmap, PT2MAP, far);

sys/arm/arm/pmap-v6.c

6478

PMAP_LOCK(pmap);

sys/arm/arm/pmap-v6.c

6480

pte1 = pte1_load(pmap_pte1(pmap, far));

sys/arm/arm/pmap-v6.c

6489

pte2 = pt2tab_load(pmap_pt2tab_entry(pmap, far));

sys/arm/arm/pmap-v6.c

6492

__func__, pmap, far);

sys/arm/arm/pmap-v6.c

6503

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6515

pte1 = pte1_load(pmap_pte1(pmap, far));

sys/arm/arm/pmap-v6.c

6522

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6531

__func__, pmap, far);

sys/arm/arm/pmap-v6.c

6532

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6537

pte1p = pmap_pte1(pmap, far);

sys/arm/arm/pmap-v6.c

6541

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6549

__func__, pmap, far);

sys/arm/arm/pmap-v6.c

6550

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6564

pte1 = pte1_load(pmap_pte1(pmap, far));

sys/arm/arm/pmap-v6.c

6573

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6582

__func__, pmap, far);

sys/arm/arm/pmap-v6.c

6583

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6588

pte1p = pmap_pte1(pmap, far);

sys/arm/arm/pmap-v6.c

6594

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6603

__func__, pmap, far);

sys/arm/arm/pmap-v6.c

6604

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6614

PMAP_UNLOCK(pmap);

sys/arm/arm/pmap-v6.c

6651

pmap_t pmap;

sys/arm/arm/pmap-v6.c

6661

pmap = vmspace_pmap(p->p_vmspace);

sys/arm/arm/pmap-v6.c

6670

pte1 = pte1_load(&pmap->pm_pt1[i]);

sys/arm/arm/pmap-v6.c

6687

pte2p = pmap_pte2(pmap, va);

sys/arm/arm/pmap-v6.c

6718

pmap_pte2_ddb(pmap_t pmap, vm_offset_t va)

sys/arm/arm/pmap-v6.c

6723

pte1 = pte1_load(pmap_pte1(pmap, va));

sys/arm/arm/pmap-v6.c

6727

if (pmap_is_current(pmap))

sys/arm/arm/pmap-v6.c

6749

dump_pmap(pmap_t pmap)

sys/arm/arm/pmap-v6.c

6752

printf("pmap %p\n", pmap);

sys/arm/arm/pmap-v6.c

6753

printf(" pm_pt1: %p\n", pmap->pm_pt1);

sys/arm/arm/pmap-v6.c

6754

printf(" pm_pt2tab: %p\n", pmap->pm_pt2tab);

sys/arm/arm/pmap-v6.c

6755

printf(" pm_active: 0x%08lX\n", pmap->pm_active.__bits[0]);

sys/arm/arm/pmap-v6.c

6761

pmap_t pmap;

sys/arm/arm/pmap-v6.c

6762

LIST_FOREACH(pmap, &allpmaps, pm_list) {

sys/arm/arm/pmap-v6.c

6763

dump_pmap(pmap);

sys/arm/arm/pmap-v6.c

6778

dump_section(pmap_t pmap, uint32_t pte1_idx)

sys/arm/arm/pmap-v6.c

6783

dump_link(pmap_t pmap, uint32_t pte1_idx, bool invalid_ok)

sys/arm/arm/pmap-v6.c

6791

pte2p = pmap_pte2_ddb(pmap, va);

sys/arm/arm/pmap-v6.c

6825

DB_SHOW_COMMAND(pmap, pmap_pmap_print)

sys/arm/arm/pmap-v6.c

6828

pmap_t pmap = (pmap_t)addr;

sys/arm/arm/pmap-v6.c

6840

if (pm == pmap) break;

sys/arm/arm/pmap-v6.c

6842

printf("given pmap %p is not in allpmaps list\n", pmap);

sys/arm/arm/pmap-v6.c

6846

pmap = PCPU_GET(curpmap);

sys/arm/arm/pmap-v6.c

6851

printf("pmap: 0x%08X\n", (uint32_t)pmap);

sys/arm/arm/pmap-v6.c

6853

printf("pt2tab: 0x%08X\n", (uint32_t)pmap->pm_pt2tab);

sys/arm/arm/pmap-v6.c

6856

pte1 = pte1_load(&pmap->pm_pt1[i]);

sys/arm/arm/pmap-v6.c

6866

dump_section(pmap, i);

sys/arm/arm/pmap-v6.c

6870

pte2 = pte2_load(pmap_pt2tab_entry(pmap, va));

sys/arm/arm/pmap-v6.c

6891

dump_link(pmap, i, invalid_ok);

sys/arm/arm/pmap-v6.c

6898

dump_pt2tab(pmap_t pmap)

sys/arm/arm/pmap-v6.c

6908

pte2 = pte2_load(&pmap->pm_pt2tab[i]);

sys/arm/arm/pmap-v6.c

6926

pmap_t pmap = (pmap_t)addr;

sys/arm/arm/pmap-v6.c

6937

pmap = PCPU_GET(curpmap);

sys/arm/arm/pmap-v6.c

6938

printf("curpmap: 0x%08X\n", (uint32_t)pmap);

sys/arm/arm/pmap-v6.c

6940

printf("pt2tab: 0x%08X\n", (uint32_t)pmap->pm_pt2tab);

sys/arm/arm/pmap-v6.c

6944

pte1 = pte1_load(&pmap->pm_pt1[i]);

sys/arm/arm/pmap-v6.c

6951

dump_section(pmap, i);

sys/arm/arm/pmap-v6.c

6953

pte2 = pte2_load(pmap_pt2tab_entry(pmap, va));

sys/arm/arm/pmap-v6.c

6961

dump_pt2tab(pmap);

sys/arm/include/pcpu.h

52

struct pmap *pc_curpmap; \

sys/arm/include/pmap.h

108

LIST_ENTRY(pmap) pm_list; /* List of all pmaps */

sys/arm/include/pmap.h

111

typedef struct pmap *pmap_t;

sys/arm/include/pmap.h

114

extern struct pmap kernel_pmap_store;

sys/arm/include/pmap.h

117

#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)

sys/arm/include/pmap.h

118

#define PMAP_LOCK_ASSERT(pmap, type) \

sys/arm/include/pmap.h

119

mtx_assert(&(pmap)->pm_mtx, (type))

sys/arm/include/pmap.h

120

#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)

sys/arm/include/pmap.h

121

#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \

sys/arm/include/pmap.h

123

#define PMAP_LOCKED(pmap) mtx_owned(&(pmap)->pm_mtx)

sys/arm/include/pmap.h

124

#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)

sys/arm/include/pmap.h

125

#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)

sys/arm/include/pmap.h

126

#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)

sys/arm/include/pmap.h

141

bool pmap_ps_enabled(pmap_t pmap);

sys/arm/include/pmap.h

172

#define pmap_map_delete(pmap, sva, eva) pmap_remove(pmap, sva, eva)

sys/arm/include/pmap_var.h

446

pmap_pte1(pmap_t pmap, vm_offset_t va)

sys/arm/include/pmap_var.h

449

return (pte1_ptr(pmap->pm_pt1, va));

sys/arm/include/pmap_var.h

460

pmap_pt2tab_entry(pmap_t pmap, vm_offset_t va)

sys/arm/include/pmap_var.h

463

return (pt2tab_entry(pmap->pm_pt2tab, va));

sys/arm/include/pmap_var.h

474

pmap_pt2_page(pmap_t pmap, vm_offset_t va)

sys/arm/include/pmap_var.h

478

pte2 = pte2_load(pmap_pt2tab_entry(pmap, va));

sys/arm64/arm64/busdma_bounce.c

121

pmap_t pmap, void *buf, bus_size_t buflen, int flags);

sys/arm64/arm64/busdma_bounce.c

697

_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,

sys/arm64/arm64/busdma_bounce.c

722

if (pmap == kernel_pmap)

sys/arm64/arm64/busdma_bounce.c

725

paddr = pmap_extract(pmap, vaddr);

sys/arm64/arm64/busdma_bounce.c

825

bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,

sys/arm64/arm64/busdma_bounce.c

846

_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);

sys/arm64/arm64/busdma_bounce.c

868

if (__predict_true(pmap == kernel_pmap)) {

sys/arm64/arm64/busdma_bounce.c

872

curaddr = pmap_extract(pmap, (vm_offset_t)vaddr);

sys/arm64/arm64/pmap.c

164

#define PMAP_ASSERT_STAGE1(pmap) MPASS((pmap)->pm_stage == PM_STAGE1)

sys/arm64/arm64/pmap.c

165

#define PMAP_ASSERT_STAGE2(pmap) MPASS((pmap)->pm_stage == PM_STAGE2)

sys/arm64/arm64/pmap.c

1998

pmap_s1_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only)

sys/arm64/arm64/pmap.c

2002

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

2006

if (pmap == kernel_pmap) {

sys/arm64/arm64/pmap.c

2009

r |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));

sys/arm64/arm64/pmap.c

2021

pmap_s2_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only)

sys/arm64/arm64/pmap.c

2023

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

2025

pmap_stage2_invalidate_range(pmap_to_ttbr0(pmap), va, va + PAGE_SIZE,

sys/arm64/arm64/pmap.c

2030

pmap_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only)

sys/arm64/arm64/pmap.c

2032

if (pmap->pm_stage == PM_STAGE1)

sys/arm64/arm64/pmap.c

2033

pmap_s1_invalidate_page(pmap, va, final_only);

sys/arm64/arm64/pmap.c

2035

pmap_s2_invalidate_page(pmap, va, final_only);

sys/arm64/arm64/pmap.c

2043

pmap_s1_invalidate_strided(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/arm64/arm64/pmap.c

2048

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

2051

if (pmap == kernel_pmap) {

sys/arm64/arm64/pmap.c

2057

start = end = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));

sys/arm64/arm64/pmap.c

2076

pmap_s1_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/arm64/arm64/pmap.c

2079

pmap_s1_invalidate_strided(pmap, sva, eva, L3_SIZE, final_only);

sys/arm64/arm64/pmap.c

2083

pmap_s2_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/arm64/arm64/pmap.c

2086

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

2088

pmap_stage2_invalidate_range(pmap_to_ttbr0(pmap), sva, eva, final_only);

sys/arm64/arm64/pmap.c

2092

pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/arm64/arm64/pmap.c

2095

if (pmap->pm_stage == PM_STAGE1)

sys/arm64/arm64/pmap.c

2096

pmap_s1_invalidate_range(pmap, sva, eva, final_only);

sys/arm64/arm64/pmap.c

2098

pmap_s2_invalidate_range(pmap, sva, eva, final_only);

sys/arm64/arm64/pmap.c

2119

pmap_s1_invalidate_all(pmap_t pmap)

sys/arm64/arm64/pmap.c

2123

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

2126

if (pmap == kernel_pmap) {

sys/arm64/arm64/pmap.c

2129

r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));

sys/arm64/arm64/pmap.c

2141

pmap_s2_invalidate_all(pmap_t pmap)

sys/arm64/arm64/pmap.c

2143

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

2145

pmap_stage2_invalidate_all(pmap_to_ttbr0(pmap));

sys/arm64/arm64/pmap.c

2149

pmap_invalidate_all(pmap_t pmap)

sys/arm64/arm64/pmap.c

2151

if (pmap->pm_stage == PM_STAGE1)

sys/arm64/arm64/pmap.c

2152

pmap_s1_invalidate_all(pmap);

sys/arm64/arm64/pmap.c

2154

pmap_s2_invalidate_all(pmap);

sys/arm64/arm64/pmap.c

2164

pmap_extract(pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

2171

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

2176

pte = pmap_pte(pmap, va, &lvl);

sys/arm64/arm64/pmap.c

2202

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

2214

pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/arm64/arm64/pmap.c

2223

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

2224

pte = pmap_pte(pmap, va, &lvl);

sys/arm64/arm64/pmap.c

2242

else if (pmap->pm_stage == PM_STAGE1 &&

sys/arm64/arm64/pmap.c

2245

else if (pmap->pm_stage == PM_STAGE2 &&

sys/arm64/arm64/pmap.c

2267

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

2726

pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/arm64/arm64/pmap.c

2731

_pmap_unwire_l3(pmap, va, m, free);

sys/arm64/arm64/pmap.c

2738

_pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/arm64/arm64/pmap.c

2741

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

2749

l0 = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

2755

l1 = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

2761

l2 = pmap_l2(pmap, va);

sys/arm64/arm64/pmap.c

2764

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

2770

l1 = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

2773

pmap_unwire_l3(pmap, va, l2pg, free);

sys/arm64/arm64/pmap.c

2779

l0 = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

2782

pmap_unwire_l3(pmap, va, l1pg, free);

sys/arm64/arm64/pmap.c

2784

pmap_invalidate_page(pmap, va, false);

sys/arm64/arm64/pmap.c

2798

pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde,

sys/arm64/arm64/pmap.c

2809

return (pmap_unwire_l3(pmap, va, mpte, free));

sys/arm64/arm64/pmap.c

2817

pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte)

sys/arm64/arm64/pmap.c

2822

if (pmap_unwire_l3(pmap, va, mpte, &free))

sys/arm64/arm64/pmap.c

2827

pmap_pinit0(pmap_t pmap)

sys/arm64/arm64/pmap.c

2830

PMAP_LOCK_INIT(pmap);

sys/arm64/arm64/pmap.c

2831

bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));

sys/arm64/arm64/pmap.c

2832

pmap->pm_l0_paddr = READ_SPECIALREG(ttbr0_el1);

sys/arm64/arm64/pmap.c

2833

pmap->pm_l0 = PHYS_TO_DMAP(pmap->pm_l0_paddr);

sys/arm64/arm64/pmap.c

2834

TAILQ_INIT(&pmap->pm_pvchunk);

sys/arm64/arm64/pmap.c

2835

vm_radix_init(&pmap->pm_root);

sys/arm64/arm64/pmap.c

2836

pmap->pm_cookie = COOKIE_FROM(ASID_RESERVED_FOR_PID_0, INT_MIN);

sys/arm64/arm64/pmap.c

2837

pmap->pm_stage = PM_STAGE1;

sys/arm64/arm64/pmap.c

2838

pmap->pm_levels = 4;

sys/arm64/arm64/pmap.c

2839

pmap->pm_ttbr = pmap->pm_l0_paddr;

sys/arm64/arm64/pmap.c

2840

pmap->pm_asid_set = &asids;

sys/arm64/arm64/pmap.c

2841

pmap->pm_bti = NULL;

sys/arm64/arm64/pmap.c

2843

PCPU_SET(curpmap, pmap);

sys/arm64/arm64/pmap.c

2847

pmap_pinit_stage(pmap_t pmap, enum pmap_stage stage, int levels)

sys/arm64/arm64/pmap.c

2856

pmap->pm_l0_paddr = VM_PAGE_TO_PHYS(m);

sys/arm64/arm64/pmap.c

2857

pmap->pm_l0 = PHYS_TO_DMAP(pmap->pm_l0_paddr);

sys/arm64/arm64/pmap.c

2859

TAILQ_INIT(&pmap->pm_pvchunk);

sys/arm64/arm64/pmap.c

2860

vm_radix_init(&pmap->pm_root);

sys/arm64/arm64/pmap.c

2861

bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));

sys/arm64/arm64/pmap.c

2862

pmap->pm_cookie = COOKIE_FROM(-1, INT_MAX);

sys/arm64/arm64/pmap.c

2865

pmap->pm_levels = levels;

sys/arm64/arm64/pmap.c

2866

pmap->pm_stage = stage;

sys/arm64/arm64/pmap.c

2867

pmap->pm_bti = NULL;

sys/arm64/arm64/pmap.c

2870

pmap->pm_asid_set = &asids;

sys/arm64/arm64/pmap.c

2872

pmap->pm_bti = malloc(sizeof(struct rangeset), M_DEVBUF,

sys/arm64/arm64/pmap.c

2874

rangeset_init(pmap->pm_bti, bti_dup_range,

sys/arm64/arm64/pmap.c

2875

bti_free_range, pmap, M_NOWAIT);

sys/arm64/arm64/pmap.c

2879

pmap->pm_asid_set = &vmids;

sys/arm64/arm64/pmap.c

2887

pmap_alloc_asid(pmap);

sys/arm64/arm64/pmap.c

2894

if (pmap->pm_levels == 3) {

sys/arm64/arm64/pmap.c

2895

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

2896

m = _pmap_alloc_l3(pmap, NUL2E + NUL1E, NULL);

sys/arm64/arm64/pmap.c

2897

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

2899

pmap->pm_ttbr = VM_PAGE_TO_PHYS(m);

sys/arm64/arm64/pmap.c

2905

pmap_pinit(pmap_t pmap)

sys/arm64/arm64/pmap.c

2908

return (pmap_pinit_stage(pmap, PM_STAGE1, 4));

sys/arm64/arm64/pmap.c

2923

_pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)

sys/arm64/arm64/pmap.c

2927

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

2935

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

2937

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

2968

l0p = &pmap->pm_l0[l0index];

sys/arm64/arm64/pmap.c

2979

if (pmap == kernel_pmap)

sys/arm64/arm64/pmap.c

2992

l0 = &pmap->pm_l0[l0index];

sys/arm64/arm64/pmap.c

2996

if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index,

sys/arm64/arm64/pmap.c

3020

l0 = &pmap->pm_l0[l0index];

sys/arm64/arm64/pmap.c

3024

if (_pmap_alloc_l3(pmap, NUL2E + l1index,

sys/arm64/arm64/pmap.c

3039

if (_pmap_alloc_l3(pmap, NUL2E + l1index,

sys/arm64/arm64/pmap.c

3058

pmap_resident_count_inc(pmap, 1);

sys/arm64/arm64/pmap.c

3064

pmap_alloc_l2(pmap_t pmap, vm_offset_t va, vm_page_t *l2pgp,

sys/arm64/arm64/pmap.c

3075

l1 = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

3087

l2pg = _pmap_alloc_l3(pmap, NUL2E + l2pindex, lockp);

sys/arm64/arm64/pmap.c

3104

pmap_alloc_l3(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)

sys/arm64/arm64/pmap.c

311

struct pmap kernel_pmap_store;

sys/arm64/arm64/pmap.c

3122

pde = pmap_pde(pmap, va, &lvl);

sys/arm64/arm64/pmap.c

3161

m = _pmap_alloc_l3(pmap, ptepindex, lockp);

sys/arm64/arm64/pmap.c

3178

pmap_release(pmap_t pmap)

sys/arm64/arm64/pmap.c

3186

if (pmap->pm_levels != 4) {

sys/arm64/arm64/pmap.c

3187

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

3188

KASSERT(pmap->pm_stats.resident_count == 1,

sys/arm64/arm64/pmap.c

3190

pmap->pm_stats.resident_count));

sys/arm64/arm64/pmap.c

3191

KASSERT((pmap->pm_l0[0] & ATTR_DESCR_VALID) == ATTR_DESCR_VALID,

sys/arm64/arm64/pmap.c

3192

("pmap_release: Invalid l0 entry: %lx", pmap->pm_l0[0]));

sys/arm64/arm64/pmap.c

3195

m = PHYS_TO_VM_PAGE(pmap->pm_ttbr);

sys/arm64/arm64/pmap.c

3196

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

3197

rv = pmap_unwire_l3(pmap, 0, m, &freelist);

sys/arm64/arm64/pmap.c

3198

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

3203

KASSERT(pmap->pm_stats.resident_count == 0,

sys/arm64/arm64/pmap.c

3205

pmap->pm_stats.resident_count));

sys/arm64/arm64/pmap.c

3206

KASSERT(vm_radix_is_empty(&pmap->pm_root),

sys/arm64/arm64/pmap.c

3209

set = pmap->pm_asid_set;

sys/arm64/arm64/pmap.c

3218

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

3220

if (COOKIE_TO_EPOCH(pmap->pm_cookie) == set->asid_epoch) {

sys/arm64/arm64/pmap.c

3221

asid = COOKIE_TO_ASID(pmap->pm_cookie);

sys/arm64/arm64/pmap.c

3229

if (pmap->pm_bti != NULL) {

sys/arm64/arm64/pmap.c

3230

rangeset_fini(pmap->pm_bti);

sys/arm64/arm64/pmap.c

3231

free(pmap->pm_bti, M_DEVBUF);

sys/arm64/arm64/pmap.c

3235

m = PHYS_TO_VM_PAGE(pmap->pm_l0_paddr);

sys/arm64/arm64/pmap.c

3390

pmap_t next_pmap, pmap;

sys/arm64/arm64/pmap.c

3402

pmap = NULL;

sys/arm64/arm64/pmap.c

3434

if (pmap != next_pmap) {

sys/arm64/arm64/pmap.c

3435

if (pmap != NULL && pmap != locked_pmap)

sys/arm64/arm64/pmap.c

3436

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

3437

pmap = next_pmap;

sys/arm64/arm64/pmap.c

3439

if (pmap > locked_pmap) {

sys/arm64/arm64/pmap.c

3441

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

3444

} else if (pmap != locked_pmap) {

sys/arm64/arm64/pmap.c

3445

if (PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

3449

pmap = NULL; /* pmap is not locked */

sys/arm64/arm64/pmap.c

3470

pde = pmap_pde(pmap, va, &lvl);

sys/arm64/arm64/pmap.c

3478

(void)pmap_demote_l3c(pmap, pte, va);

sys/arm64/arm64/pmap.c

3481

if (pmap_pte_dirty(pmap, tpte))

sys/arm64/arm64/pmap.c

3484

pmap_s1_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

3499

pmap_unuse_pt(pmap, va, pmap_load(pde), &free);

sys/arm64/arm64/pmap.c

3508

pmap_resident_count_dec(pmap, freed);

sys/arm64/arm64/pmap.c

3512

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3524

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3527

if (pmap == locked_pmap)

sys/arm64/arm64/pmap.c

3533

if (pvc->active_reclaims == 1 && pmap != NULL) {

sys/arm64/arm64/pmap.c

3552

if (pmap != NULL && pmap != locked_pmap)

sys/arm64/arm64/pmap.c

3553

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

3585

free_pv_entry(pmap_t pmap, pv_entry_t pv)

sys/arm64/arm64/pmap.c

3590

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3601

if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {

sys/arm64/arm64/pmap.c

3602

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3603

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3607

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3672

get_pv_entry(pmap_t pmap, struct rwlock **lockp)

sys/arm64/arm64/pmap.c

3680

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3683

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/arm64/arm64/pmap.c

3696

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3697

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,

sys/arm64/arm64/pmap.c

370

static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,

sys/arm64/arm64/pmap.c

3712

m = reclaim_pv_chunk(pmap, lockp);

sys/arm64/arm64/pmap.c

3720

pc->pc_pmap = pmap;

sys/arm64/arm64/pmap.c

3728

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3741

reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)

sys/arm64/arm64/pmap.c

3750

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3763

TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {

sys/arm64/arm64/pmap.c

3775

m = reclaim_pv_chunk(pmap, lockp);

sys/arm64/arm64/pmap.c

3784

pc->pc_pmap = pmap;

sys/arm64/arm64/pmap.c

3786

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3815

pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

3820

if (pmap == PV_PMAP(pv) && va == pv->pv_va) {

sys/arm64/arm64/pmap.c

3835

pmap_pv_demote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/arm64/arm64/pmap.c

3845

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3858

pv = pmap_pvh_remove(pvh, pmap, va);

sys/arm64/arm64/pmap.c

3867

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/arm64/arm64/pmap.c

3885

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3886

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3890

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3891

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

3903

pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

3907

pv = pmap_pvh_remove(pvh, pmap, va);

sys/arm64/arm64/pmap.c

3909

free_pv_entry(pmap, pv);

sys/arm64/arm64/pmap.c

3917

pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm64/arm64/pmap.c

3922

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3924

if ((pv = get_pv_entry(pmap, NULL)) != NULL) {

sys/arm64/arm64/pmap.c

3940

pmap_pv_insert_l2(pmap_t pmap, vm_offset_t va, pd_entry_t l2e, u_int flags,

sys/arm64/arm64/pmap.c

3947

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3949

if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?

sys/arm64/arm64/pmap.c

3966

pmap_pv_insert_l3c(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm64/arm64/pmap.c

3974

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

3983

pv = get_pv_entry(pmap, NULL);

sys/arm64/arm64/pmap.c

3988

pmap_pvh_free(&mt->md, pmap, tva);

sys/arm64/arm64/pmap.c

4000

pmap_remove_kernel_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va)

sys/arm64/arm64/pmap.c

4007

KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));

sys/arm64/arm64/pmap.c

4008

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4010

ml3 = pmap_remove_pt_page(pmap, va);

sys/arm64/arm64/pmap.c

4036

pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva, pd_entry_t l1e,

sys/arm64/arm64/pmap.c

4043

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4053

pmap_s1_invalidate_page(pmap, sva, true);

sys/arm64/arm64/pmap.c

4056

pmap->pm_stats.wired_count -= L2_SIZE / PAGE_SIZE;

sys/arm64/arm64/pmap.c

4057

pmap_resident_count_dec(pmap, L2_SIZE / PAGE_SIZE);

sys/arm64/arm64/pmap.c

4062

pmap_pvh_free(pvh, pmap, sva);

sys/arm64/arm64/pmap.c

4064

if (pmap_pte_dirty(pmap, old_l2))

sys/arm64/arm64/pmap.c

4073

if (pmap != kernel_pmap) {

sys/arm64/arm64/pmap.c

4074

ml3 = pmap_remove_pt_page(pmap, sva);

sys/arm64/arm64/pmap.c

4078

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

4085

pmap_remove_kernel_l2(pmap, l2, sva);

sys/arm64/arm64/pmap.c

4087

ml3 = vm_radix_lookup(&pmap->pm_root, pmap_l2_pindex(sva));

sys/arm64/arm64/pmap.c

4093

return (pmap_unuse_pt(pmap, sva, l1e, free));

sys/arm64/arm64/pmap.c

4100

pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va,

sys/arm64/arm64/pmap.c

4107

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4110

(void)pmap_demote_l3c(pmap, l3, va);

sys/arm64/arm64/pmap.c

4112

pmap_s1_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

4114

pmap->pm_stats.wired_count -= 1;

sys/arm64/arm64/pmap.c

4115

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

4118

if (pmap_pte_dirty(pmap, old_l3))

sys/arm64/arm64/pmap.c

4123

pmap_pvh_free(&m->md, pmap, va);

sys/arm64/arm64/pmap.c

4131

return (pmap_unuse_pt(pmap, va, l2e, free));

sys/arm64/arm64/pmap.c

4141

pmap_remove_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va, vm_offset_t *vap,

sys/arm64/arm64/pmap.c

4151

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4173

pmap->pm_stats.wired_count -= L3C_ENTRIES;

sys/arm64/arm64/pmap.c

4174

pmap_resident_count_dec(pmap, L3C_ENTRIES);

sys/arm64/arm64/pmap.c

4189

pmap_invalidate_range(pmap, *vap, va,

sys/arm64/arm64/pmap.c

4201

if (pmap_pte_dirty(pmap, first_l3e))

sys/arm64/arm64/pmap.c

4205

pmap_pvh_free(&mt->md, pmap, tva);

sys/arm64/arm64/pmap.c

4216

_pmap_unwire_l3(pmap, va, ml3, free);

sys/arm64/arm64/pmap.c

4228

pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva,

sys/arm64/arm64/pmap.c

4242

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4251

pmap_invalidate_range(pmap, va, sva, true);

sys/arm64/arm64/pmap.c

4264

if (pmap_remove_l3c(pmap, l3, sva, &va, eva,

sys/arm64/arm64/pmap.c

4275

(void)pmap_demote_l3c(pmap, l3, sva);

sys/arm64/arm64/pmap.c

4279

pmap->pm_stats.wired_count--;

sys/arm64/arm64/pmap.c

4280

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

4286

if (pmap_pte_dirty(pmap, old_l3))

sys/arm64/arm64/pmap.c

4302

pmap_invalidate_range(pmap, va,

sys/arm64/arm64/pmap.c

4311

pmap_pvh_free(&m->md, pmap, sva);

sys/arm64/arm64/pmap.c

4319

if (l3pg != NULL && pmap_unwire_l3(pmap, sva, l3pg, free)) {

sys/arm64/arm64/pmap.c

4333

pmap_invalidate_range(pmap, va, sva, true);

sys/arm64/arm64/pmap.c

4337

pmap_remove1(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, bool map_delete)

sys/arm64/arm64/pmap.c

4348

if (pmap->pm_stats.resident_count == 0)

sys/arm64/arm64/pmap.c

4353

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

4355

pmap_bti_on_remove(pmap, sva, eva);

sys/arm64/arm64/pmap.c

4359

if (pmap->pm_stats.resident_count == 0)

sys/arm64/arm64/pmap.c

4362

l0 = pmap_l0(pmap, sva);

sys/arm64/arm64/pmap.c

4382

MPASS(pmap != kernel_pmap);

sys/arm64/arm64/pmap.c

4385

pmap_s1_invalidate_page(pmap, sva, true);

sys/arm64/arm64/pmap.c

4386

pmap_resident_count_dec(pmap, L1_SIZE / PAGE_SIZE);

sys/arm64/arm64/pmap.c

4387

pmap_unuse_pt(pmap, sva, pmap_load(l0), &free);

sys/arm64/arm64/pmap.c

4403

pmap_remove_l2(pmap, l2, sva, pmap_load(l1),

sys/arm64/arm64/pmap.c

4406

} else if (pmap_demote_l2_locked(pmap, l2, sva,

sys/arm64/arm64/pmap.c

4426

pmap_remove_l3_range(pmap, l3_paddr, sva, va_next, &free,

sys/arm64/arm64/pmap.c

4431

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

4442

pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm64/arm64/pmap.c

4444

pmap_remove1(pmap, sva, eva, false);

sys/arm64/arm64/pmap.c

4454

pmap_map_delete(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm64/arm64/pmap.c

4456

pmap_remove1(pmap, sva, eva, true);

sys/arm64/arm64/pmap.c

4477

pmap_t pmap;

sys/arm64/arm64/pmap.c

4493

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

4494

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

4497

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

4500

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

4505

pte = pmap_pte_exists(pmap, va, 2, __func__);

sys/arm64/arm64/pmap.c

4506

pmap_demote_l2_locked(pmap, pte, va, &lock);

sys/arm64/arm64/pmap.c

4507

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

4510

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

4511

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

4515

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

4518

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

4522

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

4524

pde = pmap_pde(pmap, pv->pv_va, &lvl);

sys/arm64/arm64/pmap.c

4534

(void)pmap_demote_l3c(pmap, pte, pv->pv_va);

sys/arm64/arm64/pmap.c

4537

pmap->pm_stats.wired_count--;

sys/arm64/arm64/pmap.c

4539

pmap_invalidate_page(pmap, pv->pv_va, true);

sys/arm64/arm64/pmap.c

4546

if (pmap_pte_dirty(pmap, tpte))

sys/arm64/arm64/pmap.c

4548

pmap_unuse_pt(pmap, pv->pv_va, tpde, &free);

sys/arm64/arm64/pmap.c

4551

free_pv_entry(pmap, pv);

sys/arm64/arm64/pmap.c

4552

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

4563

pmap_protect_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva, pt_entry_t mask,

sys/arm64/arm64/pmap.c

4569

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4570

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

4594

pmap_pte_dirty(pmap, old_l2)) {

sys/arm64/arm64/pmap.c

4604

pmap_s1_invalidate_page(pmap, sva, true);

sys/arm64/arm64/pmap.c

4614

pmap_mask_set_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va,

sys/arm64/arm64/pmap.c

4621

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4660

pmap_mask_set_locked(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t mask,

sys/arm64/arm64/pmap.c

4667

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4669

l0 = pmap_l0(pmap, sva);

sys/arm64/arm64/pmap.c

4693

pmap_s1_invalidate_page(pmap, sva, true);

sys/arm64/arm64/pmap.c

4708

pmap_protect_l2(pmap, l2, sva, mask, nbits);

sys/arm64/arm64/pmap.c

4711

pmap_demote_l2(pmap, l2, sva) == NULL)

sys/arm64/arm64/pmap.c

4736

pmap_s1_invalidate_range(pmap,

sys/arm64/arm64/pmap.c

477

static void free_pv_entry(pmap_t pmap, pv_entry_t pv);

sys/arm64/arm64/pmap.c

4770

pmap_mask_set_l3c(pmap, l3p, sva, &va,

sys/arm64/arm64/pmap.c

4777

(void)pmap_demote_l3c(pmap, l3p, sva);

sys/arm64/arm64/pmap.c

478

static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp);

sys/arm64/arm64/pmap.c

4794

pmap_pte_dirty(pmap, l3))

sys/arm64/arm64/pmap.c

480

static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);

sys/arm64/arm64/pmap.c

4801

pmap_s1_invalidate_range(pmap, va, sva, true);

sys/arm64/arm64/pmap.c

4806

pmap_mask_set(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t mask,

sys/arm64/arm64/pmap.c

4809

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

481

static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,

sys/arm64/arm64/pmap.c

4810

pmap_mask_set_locked(pmap, sva, eva, mask, nbits, invalidate);

sys/arm64/arm64/pmap.c

4811

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

4819

pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)

sys/arm64/arm64/pmap.c

4823

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

4826

pmap_remove(pmap, sva, eva);

sys/arm64/arm64/pmap.c

4839

if (pmap == kernel_pmap) {

sys/arm64/arm64/pmap.c

484

static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);

sys/arm64/arm64/pmap.c

4846

pmap_mask_set(pmap, sva, eva, mask, nbits, true);

sys/arm64/arm64/pmap.c

485

static bool pmap_activate_int(struct thread *td, pmap_t pmap);

sys/arm64/arm64/pmap.c

486

static void pmap_alloc_asid(pmap_t pmap);

sys/arm64/arm64/pmap.c

4878

pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/arm64/arm64/pmap.c

4882

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4886

return (vm_radix_insert(&pmap->pm_root, mpte));

sys/arm64/arm64/pmap.c

489

static bool pmap_copy_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va,

sys/arm64/arm64/pmap.c

4896

pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

4899

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

4900

return (vm_radix_remove(&pmap->pm_root, pmap_l2_pindex(va)));

sys/arm64/arm64/pmap.c

4909

pmap_update_entry(pmap_t pmap, pd_entry_t *ptep, pd_entry_t newpte,

sys/arm64/arm64/pmap.c

491

static pt_entry_t *pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va);

sys/arm64/arm64/pmap.c

4914

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

492

static pt_entry_t *pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2,

sys/arm64/arm64/pmap.c

4937

pmap_s1_invalidate_range(pmap, va, va + size, false);

sys/arm64/arm64/pmap.c

494

static pt_entry_t *pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va);

sys/arm64/arm64/pmap.c

495

static bool pmap_demote_l2c(pmap_t pmap, pt_entry_t *l2p, vm_offset_t va);

sys/arm64/arm64/pmap.c

4950

pmap_update_strided(pmap_t pmap, pd_entry_t *ptep, pd_entry_t *ptep_end,

sys/arm64/arm64/pmap.c

4956

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

496

static bool pmap_demote_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va);

sys/arm64/arm64/pmap.c

497

static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,

sys/arm64/arm64/pmap.c

4976

pmap_s1_invalidate_strided(pmap, va, va + size, stride, true);

sys/arm64/arm64/pmap.c

499

static int pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2,

sys/arm64/arm64/pmap.c

4995

pmap_pv_promote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/arm64/arm64/pmap.c

501

static int pmap_enter_l3c(pmap_t pmap, vm_offset_t va, pt_entry_t l3e, u_int flags,

sys/arm64/arm64/pmap.c

5016

pv = pmap_pvh_remove(&m->md, pmap, va);

sys/arm64/arm64/pmap.c

5026

pmap_pvh_free(&m->md, pmap, va);

sys/arm64/arm64/pmap.c

5038

pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, vm_page_t mpte,

sys/arm64/arm64/pmap.c

504

static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/arm64/arm64/pmap.c

5043

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

5050

if (pmap->pm_stage != PM_STAGE1 || !pmap_ps_enabled(pmap))

sys/arm64/arm64/pmap.c

5066

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

507

static void pmap_mask_set_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va,

sys/arm64/arm64/pmap.c

509

static bool pmap_pv_insert_l3c(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm64/arm64/pmap.c

5097

" in pmap %p", va & ~L2_OFFSET, pmap);

sys/arm64/arm64/pmap.c

511

static void pmap_remove_kernel_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va);

sys/arm64/arm64/pmap.c

5114

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

512

static int pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva,

sys/arm64/arm64/pmap.c

5133

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

515

static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva,

sys/arm64/arm64/pmap.c

5160

if (pmap_insert_pt_page(pmap, mpte, true, all_l3e_AF != 0)) {

sys/arm64/arm64/pmap.c

5164

pmap);

sys/arm64/arm64/pmap.c

5169

pmap_pv_promote_l2(pmap, va, PTE_TO_PHYS(newl2), lockp);

sys/arm64/arm64/pmap.c

517

static bool pmap_remove_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va,

sys/arm64/arm64/pmap.c

5171

pmap_update_entry(pmap, l2, newl2 | L2_BLOCK, va & ~L2_OFFSET, L2_SIZE);

sys/arm64/arm64/pmap.c

5175

pmap);

sys/arm64/arm64/pmap.c

5187

pmap_promote_l3c(pmap_t pmap, pd_entry_t *l3p, vm_offset_t va)

sys/arm64/arm64/pmap.c

5191

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

5198

if (pmap->pm_stage != PM_STAGE1 || !pmap_ps_enabled(pmap))

sys/arm64/arm64/pmap.c

520

static void pmap_reset_asid_set(pmap_t pmap);

sys/arm64/arm64/pmap.c

521

static bool pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,

sys/arm64/arm64/pmap.c

5221

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

524

static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex,

sys/arm64/arm64/pmap.c

5240

" in pmap %p", va & ~L3C_OFFSET, pmap);

sys/arm64/arm64/pmap.c

5255

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

527

static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm64/arm64/pmap.c

5272

(va & ~L3C_OFFSET), pmap);

sys/arm64/arm64/pmap.c

5277

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

5295

pmap_update_strided(pmap, l3p, l3p + L3C_ENTRIES, firstl3c |

sys/arm64/arm64/pmap.c

530

static void pmap_update_entry(pmap_t pmap, pd_entry_t *pte, pd_entry_t newpte,

sys/arm64/arm64/pmap.c

5300

pmap);

sys/arm64/arm64/pmap.c

5306

pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t pte, int flags,

sys/arm64/arm64/pmap.c

5312

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

532

static __inline vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);

sys/arm64/arm64/pmap.c

5321

if (!pmap_bti_same(pmap, va, va + pagesizes[psind], &newpte))

sys/arm64/arm64/pmap.c

5328

l0p = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

5330

mp = _pmap_alloc_l3(pmap, pmap_l0_pindex(va), NULL);

sys/arm64/arm64/pmap.c

5334

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

5336

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

535

static bool pmap_bti_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/arm64/arm64/pmap.c

5360

l2p = pmap_l2(pmap, va);

sys/arm64/arm64/pmap.c

5362

mp = _pmap_alloc_l3(pmap, pmap_l1_pindex(va), NULL);

sys/arm64/arm64/pmap.c

5366

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

5368

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

537

static pt_entry_t pmap_pte_bti(pmap_t pmap, vm_offset_t va);

sys/arm64/arm64/pmap.c

5375

l1p = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

538

static void pmap_bti_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);

sys/arm64/arm64/pmap.c

5391

l2p = pmap_l2(pmap, va);

sys/arm64/arm64/pmap.c

5393

mp = _pmap_alloc_l3(pmap, pmap_l2_pindex(va), NULL);

sys/arm64/arm64/pmap.c

5397

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

5399

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

542

static void pmap_bti_deassign_all(pmap_t pmap);

sys/arm64/arm64/pmap.c

5426

pmap_resident_count_inc(pmap, pagesizes[psind] / PAGE_SIZE);

sys/arm64/arm64/pmap.c

5428

pmap->pm_stats.wired_count += pagesizes[psind] / PAGE_SIZE;

sys/arm64/arm64/pmap.c

5431

pmap->pm_stats.wired_count -= pagesizes[psind] / PAGE_SIZE;

sys/arm64/arm64/pmap.c

5471

pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/arm64/arm64/pmap.c

5495

new_l3 |= pmap_pte_memattr(pmap, m->md.pv_memattr);

sys/arm64/arm64/pmap.c

5496

new_l3 |= pmap_pte_prot(pmap, prot);

sys/arm64/arm64/pmap.c

5499

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

5504

if (pmap != kernel_pmap)

sys/arm64/arm64/pmap.c

5525

if (pmap->pm_stage == PM_STAGE1)

sys/arm64/arm64/pmap.c

5537

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

5550

rv = pmap_enter_largepage(pmap, va, new_l3, flags, psind);

sys/arm64/arm64/pmap.c

5557

rv = pmap_enter_l2(pmap, va, (new_l3 & ~L3_PAGE) | L2_BLOCK,

sys/arm64/arm64/pmap.c

5565

rv = pmap_enter_l3c(pmap, va, new_l3 | ATTR_CONTIGUOUS, flags,

sys/arm64/arm64/pmap.c

5576

pde = pmap_l2(pmap, va);

sys/arm64/arm64/pmap.c

5577

(void)pmap_promote_l2(pmap, pde, va, mpte, &lock);

sys/arm64/arm64/pmap.c

5588

pde = pmap_pde(pmap, va, &lvl);

sys/arm64/arm64/pmap.c

5599

(l3 = pmap_demote_l2_locked(pmap, l2, va, &lock)) != NULL) {

sys/arm64/arm64/pmap.c

5617

mpte = _pmap_alloc_l3(pmap, pmap_l2_pindex(va),

sys/arm64/arm64/pmap.c

5632

new_l3 |= pmap_pte_bti(pmap, va);

sys/arm64/arm64/pmap.c

5646

pmap->pm_stats.wired_count++;

sys/arm64/arm64/pmap.c

5649

pmap->pm_stats.wired_count--;

sys/arm64/arm64/pmap.c

5679

(void)pmap_demote_l3c(pmap, l3, va);

sys/arm64/arm64/pmap.c

5691

if (pmap_pte_dirty(pmap, orig_l3))

sys/arm64/arm64/pmap.c

5694

pmap_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

5698

pv = pmap_pvh_remove(&om->md, pmap, va);

sys/arm64/arm64/pmap.c

5700

free_pv_entry(pmap, pv);

sys/arm64/arm64/pmap.c

5709

pmap_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

5717

pmap->pm_stats.wired_count++;

sys/arm64/arm64/pmap.c

5718

pmap_resident_count_inc(pmap, 1);

sys/arm64/arm64/pmap.c

5725

pv = get_pv_entry(pmap, &lock);

sys/arm64/arm64/pmap.c

573

pmap_l0(pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

5736

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

5748

if ((prot & VM_PROT_EXECUTE) && pmap != kernel_pmap &&

sys/arm64/arm64/pmap.c

5751

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

576

return (&pmap->pm_l0[pmap_l0_index(va)]);

sys/arm64/arm64/pmap.c

5766

(void)pmap_demote_l3c(pmap, l3, va);

sys/arm64/arm64/pmap.c

5768

pmap_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

5770

pmap_pte_dirty(pmap, orig_l3))

sys/arm64/arm64/pmap.c

5786

__func__, pmap, va, new_l3);

sys/arm64/arm64/pmap.c

5806

pmap_promote_l3c(pmap, l3, va) &&

sys/arm64/arm64/pmap.c

5808

(void)pmap_promote_l2(pmap, pde, va, mpte, &lock);

sys/arm64/arm64/pmap.c

5819

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

5830

pmap_enter_l2_rx(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/arm64/arm64/pmap.c

5835

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

5836

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

5854

if (pmap != kernel_pmap)

sys/arm64/arm64/pmap.c

5856

return (pmap_enter_l2(pmap, va, new_l2, PMAP_ENTER_NOSLEEP |

sys/arm64/arm64/pmap.c

589

pmap_l1(pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

5891

pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags,

sys/arm64/arm64/pmap.c

5899

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

5906

if ((l2 = pmap_alloc_l2(pmap, va, &l2pg, (flags &

sys/arm64/arm64/pmap.c

5909

va, pmap);

sys/arm64/arm64/pmap.c

5918

if (!pmap_bti_same(pmap, va, va + L2_SIZE, &new_l2)) {

sys/arm64/arm64/pmap.c

5920

pmap_abort_ptp(pmap, va, l2pg);

sys/arm64/arm64/pmap.c

593

l0 = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

5936

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

5944

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

5950

(void)pmap_remove_l2(pmap, l2, va,

sys/arm64/arm64/pmap.c

5951

pmap_load(pmap_l1(pmap, va)), false, &free, lockp);

sys/arm64/arm64/pmap.c

5960

if (pmap_insert_pt_page(pmap, mt, false,

sys/arm64/arm64/pmap.c

5973

pmap_remove_l3_range(pmap, old_l2, va, va + L2_SIZE,

sys/arm64/arm64/pmap.c

5984

pmap_s1_invalidate_page(pmap, va, false);

sys/arm64/arm64/pmap.c

6001

if ((new_l2 & ATTR_SW_WIRED) != 0 && pmap != kernel_pmap) {

sys/arm64/arm64/pmap.c

6004

pmap_abort_ptp(pmap, va, l2pg);

sys/arm64/arm64/pmap.c

6008

if (pmap_insert_pt_page(pmap, uwptpg, true, false)) {

sys/arm64/arm64/pmap.c

6011

pmap_abort_ptp(pmap, va, l2pg);

sys/arm64/arm64/pmap.c

6014

pmap_resident_count_inc(pmap, 1);

sys/arm64/arm64/pmap.c

6021

if (!pmap_pv_insert_l2(pmap, va, new_l2, flags, lockp)) {

sys/arm64/arm64/pmap.c

6023

pmap_abort_ptp(pmap, va, l2pg);

sys/arm64/arm64/pmap.c

6029

mt = pmap_remove_pt_page(pmap, va);

sys/arm64/arm64/pmap.c

6034

mt = pmap_remove_pt_page(pmap, va);

sys/arm64/arm64/pmap.c

6038

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

6045

va, pmap);

sys/arm64/arm64/pmap.c

6057

pmap->pm_stats.wired_count += L2_SIZE / PAGE_SIZE;

sys/arm64/arm64/pmap.c

6058

pmap->pm_stats.resident_count += L2_SIZE / PAGE_SIZE;

sys/arm64/arm64/pmap.c

6065

pmap != kernel_pmap && m->md.pv_memattr == VM_MEMATTR_WRITE_BACK) {

sys/arm64/arm64/pmap.c

6078

va, pmap);

sys/arm64/arm64/pmap.c

6089

pmap_enter_l3c_rx(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t *ml3p,

sys/arm64/arm64/pmap.c

6094

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

6095

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

6113

if (pmap != kernel_pmap)

sys/arm64/arm64/pmap.c

6115

return (pmap_enter_l3c(pmap, va, l3e, PMAP_ENTER_NOSLEEP |

sys/arm64/arm64/pmap.c

6120

pmap_enter_l3c(pmap_t pmap, vm_offset_t va, pt_entry_t l3e, u_int flags,

sys/arm64/arm64/pmap.c

6130

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

6154

pde = pmap_pde(pmap, va, &lvl);

sys/arm64/arm64/pmap.c

6166

l3p = pmap_demote_l2_locked(pmap, l2p,

sys/arm64/arm64/pmap.c

6187

*ml3p = _pmap_alloc_l3(pmap, l2pindex, (flags &

sys/arm64/arm64/pmap.c

6217

l3p = pmap_demote_l2_locked(pmap, l2p, va, lockp);

sys/arm64/arm64/pmap.c

622

pmap_l2(pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

6232

if (!pmap_bti_same(pmap, va, va + L3C_SIZE, &l3e)) {

sys/arm64/arm64/pmap.c

6235

pmap_abort_ptp(pmap, va, *ml3p);

sys/arm64/arm64/pmap.c

6257

pmap_remove_l3_range(pmap, pmap_load(pmap_l2(pmap, va)), va,

sys/arm64/arm64/pmap.c

626

l1 = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

6265

if (!pmap_pv_insert_l3c(pmap, va, m, lockp)) {

sys/arm64/arm64/pmap.c

6268

pmap_abort_ptp(pmap, va, *ml3p);

sys/arm64/arm64/pmap.c

6282

pmap->pm_stats.wired_count += L3C_ENTRIES;

sys/arm64/arm64/pmap.c

6283

pmap_resident_count_inc(pmap, L3C_ENTRIES);

sys/arm64/arm64/pmap.c

6291

if ((l3e & ATTR_S1_XN) == 0 && pmap != kernel_pmap &&

sys/arm64/arm64/pmap.c

6306

va, pmap);

sys/arm64/arm64/pmap.c

6323

pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,

sys/arm64/arm64/pmap.c

6339

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

6343

m->psind == 2 && pmap_ps_enabled(pmap) &&

sys/arm64/arm64/pmap.c

6344

((rv = pmap_enter_l2_rx(pmap, va, m, prot, &lock)) ==

sys/arm64/arm64/pmap.c

6348

m->psind >= 1 && pmap_ps_enabled(pmap) &&

sys/arm64/arm64/pmap.c

6349

((rv = pmap_enter_l3c_rx(pmap, va, m, &mpte, prot,

sys/arm64/arm64/pmap.c

6362

mpte = pmap_enter_quick_locked(pmap, va, m, prot |

sys/arm64/arm64/pmap.c

6369

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

6382

pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/arm64/arm64/pmap.c

6387

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

6388

(void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock);

sys/arm64/arm64/pmap.c

6391

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

6395

pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/arm64/arm64/pmap.c

6405

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

6406

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

6411

CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va);

sys/arm64/arm64/pmap.c

6434

l1 = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

6447

mpte = _pmap_alloc_l3(pmap, l2pindex,

sys/arm64/arm64/pmap.c

6453

mpte = _pmap_alloc_l3(pmap, l2pindex, NULL);

sys/arm64/arm64/pmap.c

6484

!pmap_try_insert_pv_entry(pmap, va, m, lockp)) {

sys/arm64/arm64/pmap.c

6486

pmap_abort_ptp(pmap, va, mpte);

sys/arm64/arm64/pmap.c

6493

pmap_resident_count_inc(pmap, 1);

sys/arm64/arm64/pmap.c

6498

l3_val |= pmap_pte_bti(pmap, va);

sys/arm64/arm64/pmap.c

6506

if (pmap != kernel_pmap)

sys/arm64/arm64/pmap.c

6518

if ((prot & VM_PROT_EXECUTE) && pmap != kernel_pmap &&

sys/arm64/arm64/pmap.c

6538

pmap_promote_l3c(pmap, l3, va) &&

sys/arm64/arm64/pmap.c

6541

l2 = pmap_l2(pmap, va);

sys/arm64/arm64/pmap.c

6547

if (pmap_promote_l2(pmap, l2, va, mpte, lockp))

sys/arm64/arm64/pmap.c

6561

pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,

sys/arm64/arm64/pmap.c

6580

pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm64/arm64/pmap.c

6587

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

6589

l0 = pmap_l0(pmap, sva);

sys/arm64/arm64/pmap.c

660

pmap_pde(pmap_t pmap, vm_offset_t va, int *level)

sys/arm64/arm64/pmap.c

6610

MPASS(pmap != kernel_pmap);

sys/arm64/arm64/pmap.c

6614

pmap->pm_stats.wired_count -= L1_SIZE / PAGE_SIZE;

sys/arm64/arm64/pmap.c

6637

pmap->pm_stats.wired_count -= L2_SIZE /

sys/arm64/arm64/pmap.c

664

l0 = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

6640

} else if (pmap_demote_l2(pmap, l2, sva) == NULL)

sys/arm64/arm64/pmap.c

6666

(void)pmap_demote_l3c(pmap, l3, sva);

sys/arm64/arm64/pmap.c

6678

pmap->pm_stats.wired_count--;

sys/arm64/arm64/pmap.c

6681

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

6689

pmap_copy_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va, pt_entry_t l3e,

sys/arm64/arm64/pmap.c

6694

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

6710

if (!pmap_pv_insert_l3c(pmap, va, PTE_TO_VM_PAGE(l3e), lockp)) {

sys/arm64/arm64/pmap.c

6712

pmap_abort_ptp(pmap, va, ml3);

sys/arm64/arm64/pmap.c

6728

pmap_resident_count_inc(pmap, L3C_ENTRIES);

sys/arm64/arm64/pmap.c

6731

va, pmap);

sys/arm64/arm64/pmap.c

695

pmap_pte(pmap_t pmap, vm_offset_t va, int *level)

sys/arm64/arm64/pmap.c

700

l1 = pmap_l1(pmap, va);

sys/arm64/arm64/pmap.c

7073

pmap_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/arm64/arm64/pmap.c

7087

if (PV_PMAP(pv) == pmap) {

sys/arm64/arm64/pmap.c

7098

if (PV_PMAP(pv) == pmap) {

sys/arm64/arm64/pmap.c

7122

pmap_t pmap;

sys/arm64/arm64/pmap.c

7134

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7135

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7138

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7141

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7145

pte = pmap_pte_exists(pmap, pv->pv_va, 3, __func__);

sys/arm64/arm64/pmap.c

7148

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7153

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7154

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7158

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7162

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7166

pte = pmap_pte_exists(pmap, pv->pv_va, 2, __func__);

sys/arm64/arm64/pmap.c

7169

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7214

pmap_remove_pages(pmap_t pmap)

sys/arm64/arm64/pmap.c

7236

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7237

TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {

sys/arm64/arm64/pmap.c

7249

pde = pmap_pde(pmap, pv->pv_va, &lvl);

sys/arm64/arm64/pmap.c

7319

if (pmap_pte_dirty(pmap, tpte)) {

sys/arm64/arm64/pmap.c

7335

pmap_resident_count_dec(pmap,

sys/arm64/arm64/pmap.c

7346

ml3 = pmap_remove_pt_page(pmap,

sys/arm64/arm64/pmap.c

7351

pmap_resident_count_dec(pmap,1);

sys/arm64/arm64/pmap.c

7360

pmap_resident_count_dec(pmap, 1);

sys/arm64/arm64/pmap.c

7374

pmap_unuse_pt(pmap, pv->pv_va, pmap_load(pde),

sys/arm64/arm64/pmap.c

7383

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/arm64/arm64/pmap.c

7390

pmap_invalidate_all(pmap);

sys/arm64/arm64/pmap.c

7391

pmap_bti_deassign_all(pmap);

sys/arm64/arm64/pmap.c

7393

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7407

pmap_t pmap;

sys/arm64/arm64/pmap.c

7416

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7417

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

7418

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7421

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7424

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7428

pte = pmap_pte_exists(pmap, pv->pv_va, 3, __func__);

sys/arm64/arm64/pmap.c

744

pmap_pte_exists(pmap_t pmap, vm_offset_t va, int level, const char *diag)

sys/arm64/arm64/pmap.c

7442

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7450

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7451

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

7452

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7456

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7460

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7464

pte = pmap_pte_exists(pmap, pv->pv_va, 2, __func__);

sys/arm64/arm64/pmap.c

7476

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7514

pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/arm64/arm64/pmap.c

7526

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7527

pde = pmap_pde(pmap, addr, &lvl);

sys/arm64/arm64/pmap.c

753

l0p = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

7532

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7558

pmap_t pmap;

sys/arm64/arm64/pmap.c

7576

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7577

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

7578

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7581

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7584

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7589

pte = pmap_pte_exists(pmap, va, 2, __func__);

sys/arm64/arm64/pmap.c

7591

(void)pmap_demote_l2_locked(pmap, pte, va, &lock);

sys/arm64/arm64/pmap.c

7595

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7598

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7599

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7603

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7607

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7611

pte = pmap_pte_exists(pmap, pv->pv_va, 3, __func__);

sys/arm64/arm64/pmap.c

7615

(void)pmap_demote_l3c(pmap, pte, pv->pv_va);

sys/arm64/arm64/pmap.c

7623

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

7641

pmap_invalidate_page(pmap, pv->pv_va, true);

sys/arm64/arm64/pmap.c

7643

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7670

pmap_t pmap;

sys/arm64/arm64/pmap.c

7693

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7694

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7697

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7700

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7705

pte = pmap_pte_exists(pmap, va, 2, __func__);

sys/arm64/arm64/pmap.c

7707

if (pmap_pte_dirty(pmap, tpte)) {

sys/arm64/arm64/pmap.c

7737

(uintptr_t)pmap) & (Ln_ENTRIES - 1)) == 0 &&

sys/arm64/arm64/pmap.c

7740

pmap_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

7745

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7762

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

7763

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

7767

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7770

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7774

pte = pmap_pte_exists(pmap, pv->pv_va, 3, __func__);

sys/arm64/arm64/pmap.c

7776

if (pmap_pte_dirty(pmap, tpte))

sys/arm64/arm64/pmap.c

7785

pmap_invalidate_page(pmap, pv->pv_va, true);

sys/arm64/arm64/pmap.c

7798

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7819

pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)

sys/arm64/arm64/pmap.c

7827

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

7832

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

7834

l0 = pmap_l0(pmap, sva);

sys/arm64/arm64/pmap.c

7864

if (!pmap_demote_l2_locked(pmap, l2, sva, &lock)) {

sys/arm64/arm64/pmap.c

787

pmap_ps_enabled(pmap_t pmap)

sys/arm64/arm64/pmap.c

7893

pmap_remove_l3(pmap, l3, va, pmap_load(l2),

sys/arm64/arm64/pmap.c

7910

else if (pmap_pte_dirty(pmap, oldl3)) {

sys/arm64/arm64/pmap.c

7926

(void)pmap_demote_l3c(pmap, l3, sva);

sys/arm64/arm64/pmap.c

794

if (pmap->pm_stage != PM_STAGE1)

sys/arm64/arm64/pmap.c

7943

pmap_remove_l3(pmap, dl3, dva,

sys/arm64/arm64/pmap.c

7979

pmap_s1_invalidate_range(pmap, va, sva, true);

sys/arm64/arm64/pmap.c

7984

pmap_s1_invalidate_range(pmap, va, sva, true);

sys/arm64/arm64/pmap.c

7986

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

7997

pmap_t pmap;

sys/arm64/arm64/pmap.c

8015

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

8016

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

8017

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

8020

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

8023

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

8028

l2 = pmap_l2(pmap, va);

sys/arm64/arm64/pmap.c

8032

pmap_demote_l2_locked(pmap, l2, va, &lock) &&

sys/arm64/arm64/pmap.c

8045

pmap_s1_invalidate_page(pmap, va, true);

sys/arm64/arm64/pmap.c

8047

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

805

if (pmap == kernel_pmap)

sys/arm64/arm64/pmap.c

8050

pmap = PV_PMAP(pv);

sys/arm64/arm64/pmap.c

8051

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

8052

if (!PMAP_TRYLOCK(pmap)) {

sys/arm64/arm64/pmap.c

8056

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

8059

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

8063

l2 = pmap_l2(pmap, pv->pv_va);

sys/arm64/arm64/pmap.c

8072

(void)pmap_demote_l3c(pmap, l3, pv->pv_va);

sys/arm64/arm64/pmap.c

8074

pmap_s1_invalidate_page(pmap, pv->pv_va, true);

sys/arm64/arm64/pmap.c

8076

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

824

pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t **l0, pd_entry_t **l1,

sys/arm64/arm64/pmap.c

829

if (pmap->pm_l0 == NULL)

sys/arm64/arm64/pmap.c

832

l0p = pmap_l0(pmap, va);

sys/arm64/arm64/pmap.c

8612

pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va)

sys/arm64/arm64/pmap.c

8620

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

8642

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

8672

pmap_update_entry(pmap, l1, l2phys | L1_TABLE, va, PAGE_SIZE);

sys/arm64/arm64/pmap.c

8726

pmap_demote_l2_abort(pmap_t pmap, vm_offset_t va, pt_entry_t *l2,

sys/arm64/arm64/pmap.c

8732

(void)pmap_remove_l2(pmap, l2, va, pmap_load(pmap_l1(pmap, va)), true,

sys/arm64/arm64/pmap.c

8741

pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_offset_t va,

sys/arm64/arm64/pmap.c

8749

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

8750

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

877

pmap_pte_memattr(pmap_t pmap, vm_memattr_t memattr)

sys/arm64/arm64/pmap.c

8775

pmap_demote_l2_abort(pmap, va, l2, lockp);

sys/arm64/arm64/pmap.c

8778

va, pmap);

sys/arm64/arm64/pmap.c

8781

ml3 = pmap_remove_pt_page(pmap, va);

sys/arm64/arm64/pmap.c

8784

} else if ((ml3 = pmap_remove_pt_page(pmap, va)) == NULL) {

sys/arm64/arm64/pmap.c

881

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

8818

pmap_demote_l2_abort(pmap, va, l2, lockp);

sys/arm64/arm64/pmap.c

8820

" in pmap %p", va, pmap);

sys/arm64/arm64/pmap.c

8827

pmap_resident_count_inc(pmap, 1);

sys/arm64/arm64/pmap.c

8876

reserve_pv_entries(pmap, Ln_ENTRIES - 1, lockp);

sys/arm64/arm64/pmap.c

8882

pmap_update_entry(pmap, l2, l3phys | L2_TABLE, va, PAGE_SIZE);

sys/arm64/arm64/pmap.c

8888

pmap_pv_demote_l2(pmap, va, PTE_TO_PHYS(oldl2), lockp);

sys/arm64/arm64/pmap.c

8892

" in pmap %p %lx", va, pmap, l3[0]);

sys/arm64/arm64/pmap.c

8905

pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va)

sys/arm64/arm64/pmap.c

8911

l3 = pmap_demote_l2_locked(pmap, l2, va, &lock);

sys/arm64/arm64/pmap.c

8921

pmap_demote_l2c(pmap_t pmap, pt_entry_t *l2p, vm_offset_t va)

sys/arm64/arm64/pmap.c

8927

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

8928

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

8982

pmap_s1_invalidate_strided(pmap, va & ~L2C_OFFSET, (va +

sys/arm64/arm64/pmap.c

9003

va, pmap);

sys/arm64/arm64/pmap.c

9011

pmap_demote_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va)

sys/arm64/arm64/pmap.c

9017

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

906

pmap_pte_prot(pmap_t pmap, vm_prot_t prot)

sys/arm64/arm64/pmap.c

9071

pmap_invalidate_range(pmap, va & ~L3C_OFFSET, (va + L3C_SIZE) &

sys/arm64/arm64/pmap.c

9092

va, pmap);

sys/arm64/arm64/pmap.c

911

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

9131

pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/arm64/arm64/pmap.c

9138

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

9139

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

9140

pte = pmap_pte(pmap, addr, &lvl);

sys/arm64/arm64/pmap.c

9163

if ((managed && pmap_pte_dirty(pmap, tpte)) || (!managed &&

sys/arm64/arm64/pmap.c

9179

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

9188

pmap_reset_asid_set(pmap_t pmap)

sys/arm64/arm64/pmap.c

9195

set = pmap->pm_asid_set;

sys/arm64/arm64/pmap.c

9196

stage = pmap->pm_stage;

sys/arm64/arm64/pmap.c

9198

set = pmap->pm_asid_set;

sys/arm64/arm64/pmap.c

9227

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

9232

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

9247

pmap_alloc_asid(pmap_t pmap)

sys/arm64/arm64/pmap.c

9252

set = pmap->pm_asid_set;

sys/arm64/arm64/pmap.c

9263

if (COOKIE_TO_EPOCH(pmap->pm_cookie) == set->asid_epoch)

sys/arm64/arm64/pmap.c

9272

pmap_reset_asid_set(pmap);

sys/arm64/arm64/pmap.c

9280

pmap->pm_cookie = COOKIE_FROM(new_asid, set->asid_epoch);

sys/arm64/arm64/pmap.c

9292

pmap_to_ttbr0(pmap_t pmap)

sys/arm64/arm64/pmap.c

9296

ttbr = pmap->pm_ttbr;

sys/arm64/arm64/pmap.c

9297

ttbr |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));

sys/arm64/arm64/pmap.c

932

pmap_pte_dirty(pmap_t pmap, pt_entry_t pte)

sys/arm64/arm64/pmap.c

937

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

9380

pmap_activate_int(struct thread *td, pmap_t pmap)

sys/arm64/arm64/pmap.c

9386

KASSERT(pmap != kernel_pmap, ("kernel pmap activation"));

sys/arm64/arm64/pmap.c

9388

if ((pmap->pm_stage == PM_STAGE1 && pmap == PCPU_GET(curpmap)) ||

sys/arm64/arm64/pmap.c

9389

(pmap->pm_stage == PM_STAGE2 && pmap == PCPU_GET(curvmpmap))) {

sys/arm64/arm64/pmap.c

9402

set = pmap->pm_asid_set;

sys/arm64/arm64/pmap.c

9409

if (pmap->pm_stage == PM_STAGE1)

sys/arm64/arm64/pmap.c

9410

PCPU_SET(curpmap, pmap);

sys/arm64/arm64/pmap.c

9412

PCPU_SET(curvmpmap, pmap);

sys/arm64/arm64/pmap.c

9414

epoch = COOKIE_TO_EPOCH(pmap->pm_cookie);

sys/arm64/arm64/pmap.c

9416

pmap_alloc_asid(pmap);

sys/arm64/arm64/pmap.c

9418

if (pmap->pm_stage == PM_STAGE1) {

sys/arm64/arm64/pmap.c

9428

set_ttbr0(pmap_to_ttbr0(pmap));

sys/arm64/arm64/pmap.c

9436

pmap_activate_vm(pmap_t pmap)

sys/arm64/arm64/pmap.c

9439

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

9441

(void)pmap_activate_int(NULL, pmap);

sys/arm64/arm64/pmap.c

9447

pmap_t pmap;

sys/arm64/arm64/pmap.c

9449

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/arm64/arm64/pmap.c

9450

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

9452

(void)pmap_activate_int(td, pmap);

sys/arm64/arm64/pmap.c

950

pmap_resident_count_inc(pmap_t pmap, int count)

sys/arm64/arm64/pmap.c

9504

pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz)

sys/arm64/arm64/pmap.c

9507

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

9523

pa = pmap_extract(pmap, va);

sys/arm64/arm64/pmap.c

953

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9537

pmap_stage2_fault(pmap_t pmap, uint64_t esr, uint64_t far)

sys/arm64/arm64/pmap.c

954

pmap->pm_stats.resident_count += count;

sys/arm64/arm64/pmap.c

9543

PMAP_ASSERT_STAGE2(pmap);

sys/arm64/arm64/pmap.c

9553

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

9554

pdep = pmap_pde(pmap, far, &lvl);

sys/arm64/arm64/pmap.c

9556

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

9578

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

9579

ptep = pmap_pte(pmap, far, &lvl);

sys/arm64/arm64/pmap.c

958

pmap_resident_count_dec(pmap_t pmap, int count)

sys/arm64/arm64/pmap.c

9596

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

9604

pmap_fault(pmap_t pmap, uint64_t esr, uint64_t far)

sys/arm64/arm64/pmap.c

961

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

962

KASSERT(pmap->pm_stats.resident_count >= count,

sys/arm64/arm64/pmap.c

9624

if (pmap->pm_stage == PM_STAGE2)

sys/arm64/arm64/pmap.c

9625

return (pmap_stage2_fault(pmap, esr, far));

sys/arm64/arm64/pmap.c

963

("pmap %p resident count underflow %ld %d", pmap,

sys/arm64/arm64/pmap.c

9632

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

9633

ptep = pmap_pte(pmap, far, &lvl);

sys/arm64/arm64/pmap.c

964

pmap->pm_stats.resident_count, count));

sys/arm64/arm64/pmap.c

9642

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

965

pmap->pm_stats.resident_count -= count;

sys/arm64/arm64/pmap.c

9650

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

9651

ptep = pmap_pte(pmap, far, &lvl);

sys/arm64/arm64/pmap.c

9657

pmap_s1_invalidate_page(pmap, far, true);

sys/arm64/arm64/pmap.c

9661

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

9671

if (pmap == kernel_pmap) {

sys/arm64/arm64/pmap.c

9688

owned = PMAP_OWNED(pmap);

sys/arm64/arm64/pmap.c

9690

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

9696

PMAP_UNLOCK(pmap);

sys/arm64/arm64/pmap.c

9833

pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)

sys/arm64/arm64/pmap.c

9860

pmap_bti_assign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm64/arm64/pmap.c

9865

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9866

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

9867

MPASS(pmap->pm_bti != NULL);

sys/arm64/arm64/pmap.c

9871

error = rangeset_insert(pmap->pm_bti, sva, eva, rs);

sys/arm64/arm64/pmap.c

9878

pmap_bti_deassign_all(pmap_t pmap)

sys/arm64/arm64/pmap.c

9881

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9882

if (pmap->pm_bti != NULL)

sys/arm64/arm64/pmap.c

9883

rangeset_remove_all(pmap->pm_bti);

sys/arm64/arm64/pmap.c

9895

pmap_bti_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte)

sys/arm64/arm64/pmap.c

9900

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9908

if (pmap == kernel_pmap) {

sys/arm64/arm64/pmap.c

9912

if (pmap->pm_bti == NULL)

sys/arm64/arm64/pmap.c

9914

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

9915

rs = rangeset_containing(pmap->pm_bti, sva);

sys/arm64/arm64/pmap.c

9917

return (rangeset_empty(pmap->pm_bti, sva, eva));

sys/arm64/arm64/pmap.c

9919

if ((rs = rangeset_beginning(pmap->pm_bti, va)) == NULL)

sys/arm64/arm64/pmap.c

9927

pmap_pte_bti(pmap_t pmap, vm_offset_t va)

sys/arm64/arm64/pmap.c

9929

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9932

if (pmap->pm_stage != PM_STAGE1)

sys/arm64/arm64/pmap.c

9934

if (pmap == kernel_pmap)

sys/arm64/arm64/pmap.c

9936

if (pmap->pm_bti != NULL &&

sys/arm64/arm64/pmap.c

9937

rangeset_containing(pmap->pm_bti, va) != NULL)

sys/arm64/arm64/pmap.c

9943

pmap_bti_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm64/arm64/pmap.c

9946

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9947

if (pmap->pm_bti != NULL)

sys/arm64/arm64/pmap.c

9948

rangeset_remove(pmap->pm_bti, sva, eva);

sys/arm64/arm64/pmap.c

9966

pmap_bti_update_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, bool set)

sys/arm64/arm64/pmap.c

9968

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/arm64/pmap.c

9969

PMAP_ASSERT_STAGE1(pmap);

sys/arm64/arm64/pmap.c

9971

pmap_mask_set_locked(pmap, sva, eva, ATTR_S1_GP, set ? ATTR_S1_GP : 0,

sys/arm64/arm64/pmap.c

9976

pmap_bti_set(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/arm64/arm64/pmap.c

9980

if (pmap->pm_bti == NULL)

sys/arm64/arm64/pmap.c

9984

if (pmap->pm_stage != PM_STAGE1)

sys/arm64/arm64/pmap.c

9992

PMAP_LOCK(pmap);

sys/arm64/arm64/pmap.c

9993

error = pmap_bti_assign(pmap, sva, eva);

sys/arm64/arm64/pmap.c

9995

pmap_bti_update_range(pmap, sva, eva, true);

sys/arm64/arm64/pmap.c

9996

PMAP_UNLOCK(pmap);

sys/arm64/arm64/trap.c

387

pmap_fault(map->pmap, esr, fault_va) == KERN_SUCCESS)

sys/arm64/include/bus_dma.h

122

bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,

sys/arm64/include/bus_dma.h

128

return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs,

sys/arm64/include/bus_dma_impl.h

68

void *buf, bus_size_t buflen, struct pmap *pmap, int flags,

sys/arm64/include/pcpu.h

49

struct pmap *pc_curpmap; \

sys/arm64/include/pcpu.h

50

struct pmap *pc_curvmpmap; \

sys/arm64/include/pmap.h

101

typedef struct pmap *pmap_t;

sys/arm64/include/pmap.h

106

extern struct pmap kernel_pmap_store;

sys/arm64/include/pmap.h

112

#define PMAP_ASSERT_LOCKED(pmap) \

sys/arm64/include/pmap.h

113

mtx_assert(&(pmap)->pm_mtx, MA_OWNED)

sys/arm64/include/pmap.h

114

#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)

sys/arm64/include/pmap.h

115

#define PMAP_LOCK_ASSERT(pmap, type) \

sys/arm64/include/pmap.h

116

mtx_assert(&(pmap)->pm_mtx, (type))

sys/arm64/include/pmap.h

117

#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)

sys/arm64/include/pmap.h

118

#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \

sys/arm64/include/pmap.h

120

#define PMAP_OWNED(pmap) mtx_owned(&(pmap)->pm_mtx)

sys/arm64/include/pmap.h

121

#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)

sys/arm64/include/pmap.h

122

#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)

sys/arm64/include/pmap.h

123

#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)

sys/arm64/include/pmap.h

166

bool pmap_ps_enabled(pmap_t pmap);

sys/arm64/include/pmap.h

168

uint64_t pmap_to_ttbr0(pmap_t pmap);

sys/arm64/include/vmm.h

149

struct pmap;

sys/arm64/include/vmm.h

153

DECLARE_VMMOPS_FUNC(void *, init, (struct vm *vm, struct pmap *pmap));

sys/arm64/include/vmm.h

156

DECLARE_VMMOPS_FUNC(int, run, (void *vcpui, register_t pc, struct pmap *pmap,

sys/arm64/iommu/iommu_pmap.c

104

smmu_pmap_l0(struct smmu_pmap *pmap, vm_offset_t va)

sys/arm64/iommu/iommu_pmap.c

107

return (&pmap->sp_l0[smmu_l0_index(va)]);

sys/arm64/iommu/iommu_pmap.c

120

smmu_pmap_l1(struct smmu_pmap *pmap, vm_offset_t va)

sys/arm64/iommu/iommu_pmap.c

124

l0 = smmu_pmap_l0(pmap, va);

sys/arm64/iommu/iommu_pmap.c

151

smmu_pmap_l2(struct smmu_pmap *pmap, vm_offset_t va)

sys/arm64/iommu/iommu_pmap.c

155

l1 = smmu_pmap_l1(pmap, va);

sys/arm64/iommu/iommu_pmap.c

187

smmu_pmap_pde(struct smmu_pmap *pmap, vm_offset_t va, int *level)

sys/arm64/iommu/iommu_pmap.c

191

l0 = smmu_pmap_l0(pmap, va);

sys/arm64/iommu/iommu_pmap.c

222

smmu_pmap_pte(struct smmu_pmap *pmap, vm_offset_t va, int *level)

sys/arm64/iommu/iommu_pmap.c

227

l1 = smmu_pmap_l1(pmap, va);

sys/arm64/iommu/iommu_pmap.c

274

smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count)

sys/arm64/iommu/iommu_pmap.c

277

SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/iommu/iommu_pmap.c

278

pmap->sp_resident_count += count;

sys/arm64/iommu/iommu_pmap.c

282

smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count)

sys/arm64/iommu/iommu_pmap.c

285

SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/iommu/iommu_pmap.c

286

KASSERT(pmap->sp_resident_count >= count,

sys/arm64/iommu/iommu_pmap.c

287

("pmap %p resident count underflow %ld %d", pmap,

sys/arm64/iommu/iommu_pmap.c

288

pmap->sp_resident_count, count));

sys/arm64/iommu/iommu_pmap.c

289

pmap->sp_resident_count -= count;

sys/arm64/iommu/iommu_pmap.c

293

smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count)

sys/arm64/iommu/iommu_pmap.c

298

smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count)

sys/arm64/iommu/iommu_pmap.c

334

smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m,

sys/arm64/iommu/iommu_pmap.c

340

_smmu_pmap_unwire_l3(pmap, va, m, free);

sys/arm64/iommu/iommu_pmap.c

347

_smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m,

sys/arm64/iommu/iommu_pmap.c

351

SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/iommu/iommu_pmap.c

359

l0 = smmu_pmap_l0(pmap, va);

sys/arm64/iommu/iommu_pmap.c

365

l1 = smmu_pmap_l1(pmap, va);

sys/arm64/iommu/iommu_pmap.c

371

l2 = smmu_pmap_l2(pmap, va);

sys/arm64/iommu/iommu_pmap.c

374

smmu_pmap_resident_count_dec(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

380

l1 = smmu_pmap_l1(pmap, va);

sys/arm64/iommu/iommu_pmap.c

383

smmu_pmap_unwire_l3(pmap, va, l2pg, free);

sys/arm64/iommu/iommu_pmap.c

389

l0 = smmu_pmap_l0(pmap, va);

sys/arm64/iommu/iommu_pmap.c

392

smmu_pmap_unwire_l3(pmap, va, l1pg, free);

sys/arm64/iommu/iommu_pmap.c

403

smmu_pmap_pinit(struct smmu_pmap *pmap)

sys/arm64/iommu/iommu_pmap.c

412

pmap->sp_l0_paddr = VM_PAGE_TO_PHYS(m);

sys/arm64/iommu/iommu_pmap.c

413

pmap->sp_l0 = PHYS_TO_DMAP(pmap->sp_l0_paddr);

sys/arm64/iommu/iommu_pmap.c

416

pmap->sp_resident_count = 0;

sys/arm64/iommu/iommu_pmap.c

418

mtx_init(&pmap->sp_mtx, "smmu pmap", NULL, MTX_DEF);

sys/arm64/iommu/iommu_pmap.c

435

_pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex)

sys/arm64/iommu/iommu_pmap.c

439

SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/arm64/iommu/iommu_pmap.c

473

l0 = &pmap->sp_l0[l0index];

sys/arm64/iommu/iommu_pmap.c

483

l0 = &pmap->sp_l0[l0index];

sys/arm64/iommu/iommu_pmap.c

487

if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index)

sys/arm64/iommu/iommu_pmap.c

509

l0 = &pmap->sp_l0[l0index];

sys/arm64/iommu/iommu_pmap.c

513

if (_pmap_alloc_l3(pmap, NUL2E + l1index) == NULL) {

sys/arm64/iommu/iommu_pmap.c

527

if (_pmap_alloc_l3(pmap, NUL2E + l1index)

sys/arm64/iommu/iommu_pmap.c

544

smmu_pmap_resident_count_inc(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

559

smmu_pmap_release(struct smmu_pmap *pmap)

sys/arm64/iommu/iommu_pmap.c

563

KASSERT(pmap->sp_resident_count == 0,

sys/arm64/iommu/iommu_pmap.c

565

pmap->sp_resident_count));

sys/arm64/iommu/iommu_pmap.c

567

m = PHYS_TO_VM_PAGE(pmap->sp_l0_paddr);

sys/arm64/iommu/iommu_pmap.c

570

mtx_destroy(&pmap->sp_mtx);

sys/arm64/iommu/iommu_pmap.c

581

pmap_gpu_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,

sys/arm64/iommu/iommu_pmap.c

609

SMMU_PMAP_LOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

616

pde = smmu_pmap_pde(pmap, va, &lvl);

sys/arm64/iommu/iommu_pmap.c

620

mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va));

sys/arm64/iommu/iommu_pmap.c

63

#define SMMU_PMAP_LOCK(pmap) mtx_lock(&(pmap)->sp_mtx)

sys/arm64/iommu/iommu_pmap.c

633

l1p = smmu_pmap_l1(pmap, va);

sys/arm64/iommu/iommu_pmap.c

634

l2p = smmu_pmap_l2(pmap, va);

sys/arm64/iommu/iommu_pmap.c

64

#define SMMU_PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->sp_mtx)

sys/arm64/iommu/iommu_pmap.c

649

smmu_pmap_resident_count_inc(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

65

#define SMMU_PMAP_LOCK_ASSERT(pmap, type) \

sys/arm64/iommu/iommu_pmap.c

654

SMMU_PMAP_UNLOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

66

mtx_assert(&(pmap)->sp_mtx, (type))

sys/arm64/iommu/iommu_pmap.c

663

pmap_gpu_remove(struct smmu_pmap *pmap, vm_offset_t va)

sys/arm64/iommu/iommu_pmap.c

672

SMMU_PMAP_LOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

674

pde = smmu_pmap_pde(pmap, va, &lvl);

sys/arm64/iommu/iommu_pmap.c

682

smmu_pmap_resident_count_dec(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

688

SMMU_PMAP_UNLOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

697

smmu_pmap_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,

sys/arm64/iommu/iommu_pmap.c

721

SMMU_PMAP_LOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

728

pde = smmu_pmap_pde(pmap, va, &lvl);

sys/arm64/iommu/iommu_pmap.c

732

mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va));

sys/arm64/iommu/iommu_pmap.c

746

smmu_pmap_resident_count_inc(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

751

SMMU_PMAP_UNLOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

760

smmu_pmap_remove(struct smmu_pmap *pmap, vm_offset_t va)

sys/arm64/iommu/iommu_pmap.c

766

SMMU_PMAP_LOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

768

pte = smmu_pmap_pte(pmap, va, &lvl);

sys/arm64/iommu/iommu_pmap.c

773

smmu_pmap_resident_count_dec(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

779

SMMU_PMAP_UNLOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

790

smmu_pmap_remove_pages(struct smmu_pmap *pmap)

sys/arm64/iommu/iommu_pmap.c

800

SMMU_PMAP_LOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

803

l0e = pmap->sp_l0[i];

sys/arm64/iommu/iommu_pmap.c

843

smmu_pmap_resident_count_dec(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

849

smmu_pmap_resident_count_dec(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

854

smmu_pmap_resident_count_dec(pmap, 1);

sys/arm64/iommu/iommu_pmap.c

856

smmu_pmap_clear(&pmap->sp_l0[i]);

sys/arm64/iommu/iommu_pmap.c

859

KASSERT(pmap->sp_resident_count == 0,

sys/arm64/iommu/iommu_pmap.c

86

static vm_page_t _pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex);

sys/arm64/iommu/iommu_pmap.c

860

("Invalid resident count %jd", pmap->sp_resident_count));

sys/arm64/iommu/iommu_pmap.c

862

SMMU_PMAP_UNLOCK(pmap);

sys/arm64/iommu/iommu_pmap.c

87

static void _smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va,

sys/arm64/iommu/iommu_pmap.h

44

int smmu_pmap_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,

sys/arm64/iommu/iommu_pmap.h

46

int smmu_pmap_remove(struct smmu_pmap *pmap, vm_offset_t va);

sys/arm64/iommu/iommu_pmap.h

49

int pmap_gpu_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,

sys/arm64/iommu/iommu_pmap.h

51

int pmap_gpu_remove(struct smmu_pmap *pmap, vm_offset_t va);

sys/arm64/iommu/iommu_pmap.h

54

void smmu_pmap_remove_pages(struct smmu_pmap *pmap);

sys/arm64/iommu/iommu_pmap.h

55

void smmu_pmap_release(struct smmu_pmap *pmap);

sys/arm64/iommu/iommu_pmap.h

56

int smmu_pmap_pinit(struct smmu_pmap *pmap);

sys/arm64/vmm/vmm.c

1001

pmap_t pmap;

sys/arm64/vmm/vmm.c

1011

pmap = vmspace_pmap(vm_vmspace(vm));

sys/arm64/vmm/vmm.c

1022

error = vmmops_run(vcpu->cookie, vcpu->nextpc, pmap, &evinfo);

sys/arm64/vmm/vmm.c

353

pmap_t pmap __diagused;

sys/arm64/vmm/vmm.c

358

pmap = vmspace_pmap(vm_vmspace(vm));

sys/arm64/vmm/vmm.c

363

MPASS(cpuid_to_pcpu[i]->pc_curvmpmap != pmap);

sys/arm64/vmm/vmm.c

915

pmap_t pmap;

sys/arm64/vmm/vmm.c

920

pmap = vmspace_pmap(vm_vmspace(vcpu->vm));

sys/arm64/vmm/vmm.c

925

if (pmap_fault(pmap, esr, addr) == KERN_SUCCESS)

sys/arm64/vmm/vmm_arm64.c

1095

vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)

sys/arm64/vmm/vmm_arm64.c

1182

pmap_activate_vm(pmap);

sys/arm64/vmm/vmm_arm64.c

1184

pmap_to_ttbr0(pmap));

sys/arm64/vmm/vmm_arm64.c

1217

pmap);

sys/arm64/vmm/vmm_arm64.c

513

vmmops_init(struct vm *vm, pmap_t pmap)

sys/arm64/vmm/vmm_arm64.c

599

arm_vmm_pinit(pmap_t pmap)

sys/arm64/vmm/vmm_arm64.c

602

pmap_pinit_stage(pmap, PM_STAGE2, vmm_pmap_levels);

sys/arm64/vmm/vmm_arm64.c

703

pmap_t pmap)

sys/arm64/vmm/vmm_arm64.c

811

struct vm_exit *vme, pmap_t pmap)

sys/arm64/vmm/vmm_arm64.c

818

handled = handle_el1_sync_excp(hypctx, vme, pmap);

sys/compat/linux/linux_mmap.c

336

pmap_remove(map->pmap, estart, eend);

sys/compat/linuxkpi/common/src/linux_page.c

285

*mp = pmap_extract_and_hold(map->pmap, va, prot);

sys/compat/x86bios/x86bios.c

151

if (x86bios_vmc.pmap[i].kva == (vm_offset_t)addr)

sys/compat/x86bios/x86bios.c

158

bzero(x86bios_vmc.pmap + i, sizeof(*x86bios_vmc.pmap) * nfree);

sys/compat/x86bios/x86bios.c

161

while (--i >= 0 && x86bios_vmc.pmap[i].kva == 0)

sys/ddb/db_ps.c

490

(p->p_vmspace != NULL) ? &p->p_vmspace->vm_map.pmap : 0);

sys/ddb/db_watch.c

231

pmap_protect(watch->map->pmap,

sys/dev/hptmv/entry.c

1372

PBUS_DMAMAP pmap = &(pAdapter->pbus_dmamap[i]);

sys/dev/hptmv/entry.c

1373

pmap->pAdapter = pAdapter;

sys/dev/hptmv/entry.c

1374

dmamap_put(pmap);

sys/dev/hptmv/entry.c

1376

if(bus_dmamap_create(pAdapter->io_dma_parent, 0, &pmap->dma_map)) {

sys/dev/hptmv/entry.c

1382

callout_init_mtx(&pmap->timeout, &pAdapter->lock, 0);

sys/dev/hptmv/entry.c

2200

PBUS_DMAMAP pmap = (PBUS_DMAMAP)ccb->ccb_adapter;

sys/dev/hptmv/entry.c

2201

IAL_ADAPTER_T * pAdapter = pmap->pAdapter;

sys/dev/hptmv/entry.c

2204

dmamap_put(pmap);

sys/dev/hptmv/entry.c

2228

PBUS_DMAMAP pmap;

sys/dev/hptmv/entry.c

2256

pmap = dmamap_get(pAdapter);

sys/dev/hptmv/entry.c

2257

HPT_ASSERT(pmap);

sys/dev/hptmv/entry.c

2258

ccb->ccb_adapter = pmap;

sys/dev/hptmv/entry.c

2259

memset((void *)pmap->psg, 0, sizeof(pmap->psg));

sys/dev/hptmv/entry.c

2598

PBUS_DMAMAP pmap = (PBUS_DMAMAP)((union ccb *)arg)->ccb_adapter;

sys/dev/hptmv/entry.c

2599

IAL_ADAPTER_T *pAdapter = pmap->pAdapter;

sys/dev/hptmv/entry.c

2612

PBUS_DMAMAP pmap = (PBUS_DMAMAP) ccb->ccb_adapter;

sys/dev/hptmv/entry.c

2613

IAL_ADAPTER_T *pAdapter = pmap->pAdapter;

sys/dev/hptmv/entry.c

2636

bus_dmamap_sync(pAdapter->io_dma_parent, pmap->dma_map,

sys/dev/hptmv/entry.c

2640

bus_dmamap_sync(pAdapter->io_dma_parent, pmap->dma_map,

sys/dev/hptmv/entry.c

2645

callout_reset(&pmap->timeout, 20 * hz, hpt_timeout, ccb);

sys/dev/hptmv/entry.c

2655

PBUS_DMAMAP pmap = (PBUS_DMAMAP)ccb->ccb_adapter;

sys/dev/hptmv/entry.c

2656

IAL_ADAPTER_T *pAdapter = pmap->pAdapter;

sys/dev/hptmv/entry.c

2777

pCmd->pSgTable = pmap->psg;

sys/dev/hptmv/entry.c

2831

pmap->dma_map,

sys/dev/hptmv/entry.c

2841

dmamap_put(pmap);

sys/dev/hptmv/entry.c

2864

PBUS_DMAMAP pmap = (PBUS_DMAMAP)ccb->ccb_adapter;

sys/dev/hptmv/entry.c

2865

IAL_ADAPTER_T *pAdapter = pmap->pAdapter;

sys/dev/hptmv/entry.c

2869

callout_stop(&pmap->timeout);

sys/dev/hptmv/entry.c

2896

bus_dmamap_sync(pAdapter->io_dma_parent, pmap->dma_map, BUS_DMASYNC_POSTREAD);

sys/dev/hptmv/entry.c

2899

bus_dmamap_sync(pAdapter->io_dma_parent, pmap->dma_map, BUS_DMASYNC_POSTWRITE);

sys/dev/hptmv/entry.c

2902

bus_dmamap_unload(pAdapter->io_dma_parent, pmap->dma_map);

sys/dev/hyperv/vmbus/hyperv_mmu.c

130

hv_vm_tlb_flush(pmap_t pmap, vm_offset_t addr1, vm_offset_t addr2,

sys/dev/hyperv/vmbus/hyperv_mmu.c

144

return smp_targeted_tlb_shootdown_native(pmap, addr1, addr2,

sys/dev/hyperv/vmbus/hyperv_mmu.c

149

return smp_targeted_tlb_shootdown_native(pmap, addr1, addr2,

sys/dev/hyperv/vmbus/hyperv_mmu.c

164

CPU_COPY(pmap_invalidate_cpu_mask(pmap), &tmp_mask);

sys/dev/hyperv/vmbus/hyperv_mmu.c

165

CPU_COPY(pmap_invalidate_cpu_mask(pmap), &mask);

sys/dev/hyperv/vmbus/hyperv_mmu.c

181

cr3 = pmap->pm_cr3;

sys/dev/hyperv/vmbus/hyperv_mmu.c

232

curcpu_cb(pmap, addr1, addr2);

sys/dev/hyperv/vmbus/hyperv_mmu.c

237

status = hv_flush_tlb_others_ex(pmap, addr1, addr2, mask, op, sc);

sys/dev/hyperv/vmbus/hyperv_mmu.c

245

return smp_targeted_tlb_shootdown_native(pmap, addr1,

sys/dev/hyperv/vmbus/hyperv_mmu.c

250

hv_flush_tlb_others_ex(pmap_t pmap, vm_offset_t addr1, vm_offset_t addr2,

sys/dev/hyperv/vmbus/hyperv_mmu.c

264

cr3 = pmap->pm_cr3;

sys/dev/hyperv/vmbus/vmbus.c

807

hyperv_vm_tlb_flush(pmap_t pmap, vm_offset_t addr1, vm_offset_t addr2,

sys/dev/hyperv/vmbus/vmbus.c

811

return hv_vm_tlb_flush(pmap, addr1, addr2, op, sc, curcpu_cb);

sys/dev/ice/ice_flex_pipe.c

2894

struct ice_prof_map *pmap;

sys/dev/ice/ice_flex_pipe.c

2899

pmap = ice_search_prof_id(hw, blk, id);

sys/dev/ice/ice_flex_pipe.c

2900

if (!pmap) {

sys/dev/ice/ice_flex_pipe.c

2906

status = ice_rem_flow_all(hw, blk, pmap->profile_cookie);

sys/dev/ice/ice_flex_pipe.c

2911

ice_prof_dec_ref(hw, blk, pmap->prof_id);

sys/dev/ice/ice_flex_pipe.c

2913

LIST_DEL(&pmap->list);

sys/dev/ice/ice_flex_pipe.c

2914

ice_free(hw, pmap);

sys/dev/iommu/busdma_iommu.c

763

bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,

sys/dev/iommu/busdma_iommu.c

784

if (pmap == kernel_pmap)

sys/dev/iommu/busdma_iommu.c

787

paddr = pmap_extract(pmap, pstart);

sys/dev/pci/pci_user.c

968

!pmap_is_valid_memattr(map->pmap, pbm->pbm_memattr))

sys/dev/proto/proto_busdma.c

282

pmap_t pmap;

sys/dev/proto/proto_busdma.c

304

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/dev/proto/proto_busdma.c

305

md->physaddr = pmap_extract(pmap, ioc->u.md.virt_addr);

sys/dev/qlnx/qlnxe/ecore_rdma.c

451

u64 *pmap;

sys/dev/qlnx/qlnxe/ecore_rdma.c

467

pmap = (u64 *)bmap->bitmap;

sys/dev/qlnx/qlnxe/ecore_rdma.c

473

if (OSAL_BITMAP_WEIGHT((unsigned long *)&pmap[item], 64*8))

sys/dev/qlnx/qlnxe/ecore_rdma.c

476

line, (unsigned long long)pmap[item],

sys/dev/qlnx/qlnxe/ecore_rdma.c

477

(unsigned long long)pmap[item+1],

sys/dev/qlnx/qlnxe/ecore_rdma.c

478

(unsigned long long)pmap[item+2],

sys/dev/qlnx/qlnxe/ecore_rdma.c

479

(unsigned long long)pmap[item+3],

sys/dev/qlnx/qlnxe/ecore_rdma.c

480

(unsigned long long)pmap[item+4],

sys/dev/qlnx/qlnxe/ecore_rdma.c

481

(unsigned long long)pmap[item+5],

sys/dev/qlnx/qlnxe/ecore_rdma.c

482

(unsigned long long)pmap[item+6],

sys/dev/qlnx/qlnxe/ecore_rdma.c

483

(unsigned long long)pmap[item+7]);

sys/dev/qlnx/qlnxe/ecore_rdma.c

488

(OSAL_BITMAP_WEIGHT((unsigned long *)&pmap[item],

sys/dev/qlnx/qlnxe/ecore_rdma.c

497

(unsigned long long)pmap[item]);

sys/dev/sound/pci/envy24.c

165

bus_dmamap_t pmap, rmap;

sys/dev/sound/pci/envy24.c

2167

printf("%p -> %lx\n", sc->pmap, sc->paddr);

sys/dev/sound/pci/envy24.c

2205

bus_dmamap_unload(sc->dmat, sc->pmap);

sys/dev/sound/pci/envy24.c

2209

bus_dmamem_free(sc->dmat, sc->pbuf, sc->pmap);

sys/dev/sound/pci/envy24.c

2212

bus_dmamap_unload(sc->dmat, sc->pmap);

sys/dev/sound/pci/envy24.c

2214

bus_dmamem_free(sc->dmat, sc->pbuf, sc->pmap);

sys/dev/sound/pci/envy24.c

2243

if (bus_dmamem_alloc(sc->dmat, (void **)&sc->pbuf, BUS_DMA_NOWAIT, &sc->pmap))

sys/dev/sound/pci/envy24.c

2253

if (bus_dmamap_load(sc->dmat, sc->pmap, sc->pbuf, sc->psize,

sys/dev/sound/pci/envy24ht.c

164

bus_dmamap_t pmap, rmap;

sys/dev/sound/pci/envy24ht.c

2121

bus_dmamap_unload(sc->dmat, sc->pmap);

sys/dev/sound/pci/envy24ht.c

2125

bus_dmamem_free(sc->dmat, sc->pbuf, sc->pmap);

sys/dev/sound/pci/envy24ht.c

2128

bus_dmamap_unload(sc->dmat, sc->pmap);

sys/dev/sound/pci/envy24ht.c

2130

bus_dmamem_free(sc->dmat, sc->pbuf, sc->pmap);

sys/dev/sound/pci/envy24ht.c

2159

if (bus_dmamem_alloc(sc->dmat, (void **)&sc->pbuf, BUS_DMA_NOWAIT, &sc->pmap))

sys/dev/sound/pci/envy24ht.c

2169

if (bus_dmamap_load(sc->dmat, sc->pmap, sc->pbuf, sc->psize, envy24ht_dmapsetmap, sc, BUS_DMA_NOWAIT))

sys/dev/sound/pci/hdsp.c

192

&sc->pmap)) {

sys/dev/sound/pci/hdsp.c

197

if (bus_dmamap_load(sc->dmat, sc->pmap, sc->pbuf, sc->bufsize,

sys/dev/sound/pci/hdsp.c

970

bus_dmamap_unload(sc->dmat, sc->pmap);

sys/dev/sound/pci/hdsp.c

972

bus_dmamem_free(sc->dmat, sc->pbuf, sc->pmap);

sys/dev/sound/pci/hdsp.h

246

bus_dmamap_t pmap;

sys/dev/sound/pci/hdspe.c

202

&sc->pmap)) {

sys/dev/sound/pci/hdspe.c

207

if (bus_dmamap_load(sc->dmat, sc->pmap, sc->pbuf, sc->bufsize,

sys/dev/sound/pci/hdspe.c

862

bus_dmamap_unload(sc->dmat, sc->pmap);

sys/dev/sound/pci/hdspe.c

864

bus_dmamem_free(sc->dmat, sc->pbuf, sc->pmap);

sys/dev/sound/pci/hdspe.h

238

bus_dmamap_t pmap;

sys/dev/xdma/xdma.h

131

struct pmap p;

sys/fs/tmpfs/tmpfs_vfsops.c

176

pmap_protect(map->pmap, entry->start, entry->end,

sys/fs/udf/udf_vfsops.c

763

uint8_t *pmap = (uint8_t *) &lvd->maps[0];

sys/fs/udf/udf_vfsops.c

766

ptype = pmap[0];

sys/fs/udf/udf_vfsops.c

767

psize = pmap[1];

sys/fs/udf/udf_vfsops.c

777

pmap += UDF_PMAP_TYPE1_SIZE;

sys/fs/udf/udf_vfsops.c

782

pmap_id = (struct regid *)&pmap[4];

sys/fs/udf/udf_vfsops.c

792

pms = (struct part_map_spare *)pmap;

sys/fs/udf/udf_vfsops.c

793

pmap += UDF_PMAP_TYPE2_SIZE;

sys/i386/i386/genassym.c

79

ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));

sys/i386/i386/mp_machdep.c

547

smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap,

sys/i386/i386/mp_machdep.c

560

curcpu_cb(pmap, addr1, addr2);

sys/i386/i386/mp_machdep.c

583

smp_tlb_pmap = pmap;

sys/i386/i386/mp_machdep.c

593

curcpu_cb(pmap, addr1, addr2);

sys/i386/i386/mp_machdep.c

604

curcpu_cb(pmap, addr1, addr2);

sys/i386/i386/mp_machdep.c

609

smp_masked_invltlb(cpuset_t mask, pmap_t pmap, smp_invl_cb_t curcpu_cb)

sys/i386/i386/mp_machdep.c

611

smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0, curcpu_cb);

sys/i386/i386/mp_machdep.c

618

smp_masked_invlpg(cpuset_t mask, vm_offset_t addr, pmap_t pmap,

sys/i386/i386/mp_machdep.c

621

smp_targeted_tlb_shootdown(mask, IPI_INVLPG, pmap, addr, 0, curcpu_cb);

sys/i386/i386/mp_machdep.c

629

pmap_t pmap, smp_invl_cb_t curcpu_cb)

sys/i386/i386/mp_machdep.c

631

smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, pmap, addr1, addr2,

sys/i386/i386/pmap.c

1077

__CONCAT(PMTYPE, is_valid_memattr)(pmap_t pmap __unused, vm_memattr_t mode)

sys/i386/i386/pmap.c

1089

__CONCAT(PMTYPE, cache_bits)(pmap_t pmap, int mode, bool is_pde)

sys/i386/i386/pmap.c

1093

if (!pmap_is_valid_memattr(pmap, mode))

sys/i386/i386/pmap.c

1114

pmap_pat_index(pmap_t pmap, pt_entry_t pte, bool is_pde)

sys/i386/i386/pmap.c

1146

__CONCAT(PMTYPE, ps_enabled)(pmap_t pmap __unused)

sys/i386/i386/pmap.c

1189

pmap_curcpu_cb_dummy(pmap_t pmap __unused, vm_offset_t addr1 __unused,

sys/i386/i386/pmap.c

1214

pmap_invalidate_page_int(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1220

if (pmap == kernel_pmap) {

sys/i386/i386/pmap.c

1223

} else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {

sys/i386/i386/pmap.c

1229

CPU_AND(&other_cpus, &other_cpus, &pmap->pm_active);

sys/i386/i386/pmap.c

1232

smp_masked_invlpg(*mask, va, pmap, pmap_curcpu_cb_dummy);

sys/i386/i386/pmap.c

1240

pmap_invalidate_range_int(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap.c

1247

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

1252

if (pmap == kernel_pmap) {

sys/i386/i386/pmap.c

1256

} else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {

sys/i386/i386/pmap.c

1262

CPU_AND(&other_cpus, &other_cpus, &pmap->pm_active);

sys/i386/i386/pmap.c

1265

smp_masked_invlpg_range(*mask, sva, eva, pmap, pmap_curcpu_cb_dummy);

sys/i386/i386/pmap.c

1270

pmap_invalidate_all_int(pmap_t pmap)

sys/i386/i386/pmap.c

1276

if (pmap == kernel_pmap) {

sys/i386/i386/pmap.c

1279

} else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {

sys/i386/i386/pmap.c

1285

CPU_AND(&other_cpus, &other_cpus, &pmap->pm_active);

sys/i386/i386/pmap.c

1288

smp_masked_invltlb(*mask, pmap, pmap_curcpu_cb_dummy);

sys/i386/i386/pmap.c

1293

pmap_invalidate_cache_curcpu_cb(pmap_t pmap __unused,

sys/i386/i386/pmap.c

1352

pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)

sys/i386/i386/pmap.c

1362

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

1365

active = pmap->pm_active;

sys/i386/i386/pmap.c

1374

smp_no_rendezvous_barrier, pmap == kernel_pmap ?

sys/i386/i386/pmap.c

1378

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

1393

pmap_invalidate_page_int(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1396

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

1401

pmap_invalidate_range_int(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap.c

1405

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

1411

pmap_invalidate_all_int(pmap_t pmap)

sys/i386/i386/pmap.c

1414

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

1426

pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)

sys/i386/i386/pmap.c

1429

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

1433

if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))

sys/i386/i386/pmap.c

1439

__CONCAT(PMTYPE, invalidate_page)(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1442

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

1446

__CONCAT(PMTYPE, invalidate_range)(pmap_t pmap, vm_offset_t sva,

sys/i386/i386/pmap.c

1450

pmap_invalidate_range_int(pmap, sva, eva);

sys/i386/i386/pmap.c

1454

__CONCAT(PMTYPE, invalidate_all)(pmap_t pmap)

sys/i386/i386/pmap.c

1457

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

1461

pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde)

sys/i386/i386/pmap.c

1476

pmap_invalidate_range_int(pmap, va, va + NBPDR - 1);

sys/i386/i386/pmap.c

1478

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

1485

pmap_is_current(pmap_t pmap)

sys/i386/i386/pmap.c

1488

return (pmap == kernel_pmap);

sys/i386/i386/pmap.c

1496

__CONCAT(PMTYPE, pte)(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1501

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

1506

if (pmap_is_current(pmap))

sys/i386/i386/pmap.c

1557

pmap_pte_quick(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1562

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

1567

if (pmap_is_current(pmap))

sys/i386/i386/pmap.c

1594

pmap_pte_quick3(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1599

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

1628

pmap_pte_ufast(pmap_t pmap, vm_offset_t va, pd_entry_t pde)

sys/i386/i386/pmap.c

1632

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

1681

__CONCAT(PMTYPE, extract)(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1688

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

1689

pde = pmap->pm_pdir[va >> PDRSHIFT];

sys/i386/i386/pmap.c

1694

pte = pmap_pte_ufast(pmap, va, pde);

sys/i386/i386/pmap.c

1698

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

1710

__CONCAT(PMTYPE, extract_and_hold)(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/i386/i386/pmap.c

1717

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

1718

pde = *pmap_pde(pmap, va);

sys/i386/i386/pmap.c

1725

pte = pmap_pte_ufast(pmap, va, pde);

sys/i386/i386/pmap.c

1733

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

1928

pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/i386/i386/pmap.c

1932

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

1936

return (vm_radix_insert(&pmap->pm_root, mpte));

sys/i386/i386/pmap.c

1946

pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

1949

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

1950

return (vm_radix_remove(&pmap->pm_root, va >> PDRSHIFT));

sys/i386/i386/pmap.c

1960

pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free)

sys/i386/i386/pmap.c

1965

_pmap_unwire_ptp(pmap, m, free);

sys/i386/i386/pmap.c

1972

_pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free)

sys/i386/i386/pmap.c

1978

pmap->pm_pdir[m->pindex] = 0;

sys/i386/i386/pmap.c

1979

--pmap->pm_stats.resident_count;

sys/i386/i386/pmap.c

1988

MPASS(pmap != kernel_pmap);

sys/i386/i386/pmap.c

1997

pmap_unuse_pt(pmap_t pmap, vm_offset_t va, struct spglist *free)

sys/i386/i386/pmap.c

2002

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

2004

ptepde = *pmap_pde(pmap, va);

sys/i386/i386/pmap.c

2006

return (pmap_unwire_ptp(pmap, mpte, free));

sys/i386/i386/pmap.c

2014

pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte)

sys/i386/i386/pmap.c

2019

if (pmap_unwire_ptp(pmap, mpte, &free)) {

sys/i386/i386/pmap.c

2025

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

2034

__CONCAT(PMTYPE, pinit0)(pmap_t pmap)

sys/i386/i386/pmap.c

2037

PMAP_LOCK_INIT(pmap);

sys/i386/i386/pmap.c

2038

pmap->pm_pdir = IdlePTD;

sys/i386/i386/pmap.c

2040

pmap->pm_pdpt = IdlePDPT;

sys/i386/i386/pmap.c

2042

vm_radix_init(&pmap->pm_root);

sys/i386/i386/pmap.c

2043

CPU_ZERO(&pmap->pm_active);

sys/i386/i386/pmap.c

2044

TAILQ_INIT(&pmap->pm_pvchunk);

sys/i386/i386/pmap.c

2045

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/i386/i386/pmap.c

2046

pmap_activate_boot(pmap);

sys/i386/i386/pmap.c

2054

__CONCAT(PMTYPE, pinit)(pmap_t pmap)

sys/i386/i386/pmap.c

2062

if (pmap->pm_pdir == NULL) {

sys/i386/i386/pmap.c

2063

pmap->pm_pdir = kva_alloc(NBPTD);

sys/i386/i386/pmap.c

2064

if (pmap->pm_pdir == NULL)

sys/i386/i386/pmap.c

2067

pmap->pm_pdpt = uma_zalloc(pdptzone, M_WAITOK | M_ZERO);

sys/i386/i386/pmap.c

2068

KASSERT(((vm_offset_t)pmap->pm_pdpt &

sys/i386/i386/pmap.c

2071

KASSERT(pmap_kextract((vm_offset_t)pmap->pm_pdpt) < (4ULL<<30),

sys/i386/i386/pmap.c

2074

vm_radix_init(&pmap->pm_root);

sys/i386/i386/pmap.c

2076

KASSERT(vm_radix_is_empty(&pmap->pm_root),

sys/i386/i386/pmap.c

2083

pmap->pm_ptdpg[i] = vm_page_alloc_noobj(VM_ALLOC_WIRED |

sys/i386/i386/pmap.c

2086

pmap->pm_pdpt[i] = VM_PAGE_TO_PHYS(pmap->pm_ptdpg[i]) | PG_V;

sys/i386/i386/pmap.c

2090

pmap_qenter(pmap->pm_pdir, pmap->pm_ptdpg, NPGPTD);

sys/i386/i386/pmap.c

2094

trunc_page((vm_offset_t)pmap->pm_pdpt),

sys/i386/i386/pmap.c

2095

round_page((vm_offset_t)pmap->pm_pdpt +

sys/i386/i386/pmap.c

2101

pmap->pm_pdir[TRPTDI] = PTD[TRPTDI];

sys/i386/i386/pmap.c

2103

CPU_ZERO(&pmap->pm_active);

sys/i386/i386/pmap.c

2104

TAILQ_INIT(&pmap->pm_pvchunk);

sys/i386/i386/pmap.c

2105

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/i386/i386/pmap.c

2115

_pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags)

sys/i386/i386/pmap.c

2125

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

2129

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

2145

pmap->pm_stats.resident_count++;

sys/i386/i386/pmap.c

2148

KASSERT((pmap->pm_pdir[ptepindex] & PG_V) == 0,

sys/i386/i386/pmap.c

2150

__func__, (uintmax_t)pmap->pm_pdir[ptepindex]));

sys/i386/i386/pmap.c

2151

pmap->pm_pdir[ptepindex] =

sys/i386/i386/pmap.c

2158

pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags)

sys/i386/i386/pmap.c

2172

ptepa = pmap->pm_pdir[ptepindex];

sys/i386/i386/pmap.c

2179

(void)pmap_demote_pde(pmap, &pmap->pm_pdir[ptepindex], va);

sys/i386/i386/pmap.c

2180

ptepa = pmap->pm_pdir[ptepindex];

sys/i386/i386/pmap.c

2195

m = _pmap_allocpte(pmap, ptepindex, flags);

sys/i386/i386/pmap.c

2212

__CONCAT(PMTYPE, release)(pmap_t pmap)

sys/i386/i386/pmap.c

2217

KASSERT(pmap->pm_stats.resident_count == 0,

sys/i386/i386/pmap.c

2219

pmap->pm_stats.resident_count));

sys/i386/i386/pmap.c

2220

KASSERT(vm_radix_is_empty(&pmap->pm_root),

sys/i386/i386/pmap.c

2222

KASSERT(CPU_EMPTY(&pmap->pm_active),

sys/i386/i386/pmap.c

2223

("releasing active pmap %p", pmap));

sys/i386/i386/pmap.c

2225

pmap_qremove(pmap->pm_pdir, NPGPTD);

sys/i386/i386/pmap.c

2228

m = pmap->pm_ptdpg[i];

sys/i386/i386/pmap.c

2230

KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME),

sys/i386/i386/pmap.c

2311

pmap_t pmap;

sys/i386/i386/pmap.c

2321

pmap = NULL;

sys/i386/i386/pmap.c

2328

if (pmap != pc->pc_pmap) {

sys/i386/i386/pmap.c

2329

if (pmap != NULL) {

sys/i386/i386/pmap.c

2330

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

2331

if (pmap != locked_pmap)

sys/i386/i386/pmap.c

2332

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

2334

pmap = pc->pc_pmap;

sys/i386/i386/pmap.c

2336

if (pmap > locked_pmap)

sys/i386/i386/pmap.c

2337

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

2338

else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {

sys/i386/i386/pmap.c

2339

pmap = NULL;

sys/i386/i386/pmap.c

2355

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

2358

pte = __CONCAT(PMTYPE, pte)(pmap, va);

sys/i386/i386/pmap.c

2367

pmap, va));

sys/i386/i386/pmap.c

2369

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

2385

pmap_unuse_pt(pmap, va, &free);

sys/i386/i386/pmap.c

2394

pmap->pm_stats.resident_count -= freed;

sys/i386/i386/pmap.c

2398

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

2401

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,

sys/i386/i386/pmap.c

2409

if (pmap == locked_pmap)

sys/i386/i386/pmap.c

2426

if (pmap != NULL) {

sys/i386/i386/pmap.c

2427

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

2428

if (pmap != locked_pmap)

sys/i386/i386/pmap.c

2429

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

2445

free_pv_entry(pmap_t pmap, pv_entry_t pv)

sys/i386/i386/pmap.c

2451

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2466

if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=

sys/i386/i386/pmap.c

2468

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

2469

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,

sys/i386/i386/pmap.c

2474

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

2500

get_pv_entry(pmap_t pmap, bool try)

sys/i386/i386/pmap.c

2510

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2519

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/i386/i386/pmap.c

2536

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

2537

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

2554

m = pmap_pv_reclaim(pmap);

sys/i386/i386/pmap.c

2562

pc->pc_pmap = pmap;

sys/i386/i386/pmap.c

2568

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

2574

pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

2580

if (pmap == PV_PMAP(pv) && va == pv->pv_va) {

sys/i386/i386/pmap.c

2589

pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)

sys/i386/i386/pmap.c

2606

pv = pmap_pvh_remove(pvh, pmap, va);

sys/i386/i386/pmap.c

2617

pmap_insert_entry(pmap, va, m);

sys/i386/i386/pmap.c

2623

pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)

sys/i386/i386/pmap.c

2643

pv = pmap_pvh_remove(&m->md, pmap, va);

sys/i386/i386/pmap.c

2652

pmap_pvh_free(&m->md, pmap, va);

sys/i386/i386/pmap.c

2658

pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap.c

2662

pv = pmap_pvh_remove(pvh, pmap, va);

sys/i386/i386/pmap.c

2664

free_pv_entry(pmap, pv);

sys/i386/i386/pmap.c

2668

pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)

sys/i386/i386/pmap.c

2673

pmap_pvh_free(&m->md, pmap, va);

sys/i386/i386/pmap.c

2686

pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/i386/i386/pmap.c

2691

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2692

pv = get_pv_entry(pmap, false);

sys/i386/i386/pmap.c

2701

pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/i386/i386/pmap.c

2706

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2708

(pv = get_pv_entry(pmap, true)) != NULL) {

sys/i386/i386/pmap.c

2720

pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, u_int flags)

sys/i386/i386/pmap.c

2729

(pv = get_pv_entry(pmap, noreclaim)) == NULL)

sys/i386/i386/pmap.c

2756

pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)

sys/i386/i386/pmap.c

2765

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2769

if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==

sys/i386/i386/pmap.c

2784

pmap_remove_pde(pmap, pde, sva, &free);

sys/i386/i386/pmap.c

2786

pmap_invalidate_pde_page(pmap, sva, oldpde);

sys/i386/i386/pmap.c

2789

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

2793

if (pmap != kernel_pmap) {

sys/i386/i386/pmap.c

2795

pmap->pm_stats.resident_count++;

sys/i386/i386/pmap.c

2806

if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

287

static void free_pv_entry(pmap_t pmap, pv_entry_t pv);

sys/i386/i386/pmap.c

2870

pmap_update_pde(pmap, va, pde, newpde);

sys/i386/i386/pmap.c

2871

else if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

288

static pv_entry_t get_pv_entry(pmap_t pmap, bool try);

sys/i386/i386/pmap.c

2881

pmap_invalidate_page_int(pmap, (vm_offset_t)vtopte(va));

sys/i386/i386/pmap.c

289

static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);

sys/i386/i386/pmap.c

2893

pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME);

sys/i386/i386/pmap.c

2897

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

290

static bool pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde,

sys/i386/i386/pmap.c

2905

pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)

sys/i386/i386/pmap.c

2911

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2912

mpte = pmap_remove_pt_page(pmap, va);

sys/i386/i386/pmap.c

293

static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);

sys/i386/i386/pmap.c

2930

pmap_update_pde(pmap, va, pde, newpde);

sys/i386/i386/pmap.c

2937

pmap_invalidate_page_int(pmap, (vm_offset_t)vtopte(va));

sys/i386/i386/pmap.c

2944

pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,

sys/i386/i386/pmap.c

295

static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);

sys/i386/i386/pmap.c

2952

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

2957

pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;

sys/i386/i386/pmap.c

296

static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,

sys/i386/i386/pmap.c

2966

pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;

sys/i386/i386/pmap.c

2969

pmap_pvh_free(pvh, pmap, sva);

sys/i386/i386/pmap.c

2982

if (pmap == kernel_pmap) {

sys/i386/i386/pmap.c

2983

pmap_remove_kernel_pde(pmap, pdq, sva);

sys/i386/i386/pmap.c

2985

mpte = pmap_remove_pt_page(pmap, sva);

sys/i386/i386/pmap.c

2989

pmap->pm_stats.resident_count--;

sys/i386/i386/pmap.c

300

static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);

sys/i386/i386/pmap.c

3002

pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,

sys/i386/i386/pmap.c

3009

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

301

static bool pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);

sys/i386/i386/pmap.c

3012

("pmap_remove_pte: pmap %p va %x zero pte", pmap, va));

sys/i386/i386/pmap.c

3014

pmap->pm_stats.wired_count -= 1;

sys/i386/i386/pmap.c

302

static int pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/i386/i386/pmap.c

3021

pmap->pm_stats.resident_count -= 1;

sys/i386/i386/pmap.c

3028

pmap_remove_entry(pmap, m, va);

sys/i386/i386/pmap.c

3030

return (pmap_unuse_pt(pmap, va, free));

sys/i386/i386/pmap.c

3037

pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free)

sys/i386/i386/pmap.c

304

static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,

sys/i386/i386/pmap.c

3043

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

3044

if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0)

sys/i386/i386/pmap.c

3046

pmap_remove_pte(pmap, pte, va, free);

sys/i386/i386/pmap.c

3047

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

3054

pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/i386/i386/pmap.c

306

static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,

sys/i386/i386/pmap.c

3062

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

3064

for (pte = pmap_pte_quick(pmap, sva); sva != eva; pte++,

sys/i386/i386/pmap.c

3076

if (pmap_remove_pte(pmap, pte, sva, free))

sys/i386/i386/pmap.c

308

static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/i386/i386/pmap.c

3089

__CONCAT(PMTYPE, remove)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap.c

3099

if (pmap->pm_stats.resident_count == 0)

sys/i386/i386/pmap.c

310

static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,

sys/i386/i386/pmap.c

3107

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

3115

((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) {

sys/i386/i386/pmap.c

3116

pmap_remove_page(pmap, sva, &free);

sys/i386/i386/pmap.c

3129

if (pmap->pm_stats.resident_count == 0)

sys/i386/i386/pmap.c

3133

ptpaddr = pmap->pm_pdir[pdirindex];

sys/i386/i386/pmap.c

3157

pmap_remove_pde(pmap,

sys/i386/i386/pmap.c

3158

&pmap->pm_pdir[pdirindex], sva, &free);

sys/i386/i386/pmap.c

3160

} else if (!pmap_demote_pde(pmap,

sys/i386/i386/pmap.c

3161

&pmap->pm_pdir[pdirindex], sva)) {

sys/i386/i386/pmap.c

3175

if (pmap_remove_ptes(pmap, sva, pdnxt, &free))

sys/i386/i386/pmap.c

3181

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

3183

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

319

static bool pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,

sys/i386/i386/pmap.c

3205

pmap_t pmap;

sys/i386/i386/pmap.c

322

static bool pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,

sys/i386/i386/pmap.c

3221

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

3222

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

3223

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

3224

(void)pmap_demote_pde(pmap, pde, va);

sys/i386/i386/pmap.c

3225

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

3229

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

3230

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

3231

pmap->pm_stats.resident_count--;

sys/i386/i386/pmap.c

3232

pde = pmap_pde(pmap, pv->pv_va);

sys/i386/i386/pmap.c

3235

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

3238

pmap, pv->pv_va));

sys/i386/i386/pmap.c

3240

pmap->pm_stats.wired_count--;

sys/i386/i386/pmap.c

3249

pmap_unuse_pt(pmap, pv->pv_va, &free);

sys/i386/i386/pmap.c

325

static void pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,

sys/i386/i386/pmap.c

3250

pmap_invalidate_page_int(pmap, pv->pv_va);

sys/i386/i386/pmap.c

3252

free_pv_entry(pmap, pv);

sys/i386/i386/pmap.c

3253

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

3265

pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)

sys/i386/i386/pmap.c

327

static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,

sys/i386/i386/pmap.c

3271

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

329

static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);

sys/i386/i386/pmap.c

330

static void pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free);

sys/i386/i386/pmap.c

331

static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/i386/i386/pmap.c

3311

__CONCAT(PMTYPE, protect)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/i386/i386/pmap.c

3321

pmap_remove(pmap, sva, eva);

sys/i386/i386/pmap.c

333

static void pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va);

sys/i386/i386/pmap.c

3334

if (pmap_is_current(pmap))

sys/i386/i386/pmap.c

334

static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);

sys/i386/i386/pmap.c

3344

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

335

static bool pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,

sys/i386/i386/pmap.c

3354

ptpaddr = pmap->pm_pdir[pdirindex];

sys/i386/i386/pmap.c

337

static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,

sys/i386/i386/pmap.c

3376

if (pmap_protect_pde(pmap,

sys/i386/i386/pmap.c

3377

&pmap->pm_pdir[pdirindex], sva, prot))

sys/i386/i386/pmap.c

3386

pmap);

sys/i386/i386/pmap.c

3387

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

3392

if (!pmap_demote_pde(pmap,

sys/i386/i386/pmap.c

3393

&pmap->pm_pdir[pdirindex], sva)) {

sys/i386/i386/pmap.c

3406

for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,

sys/i386/i386/pmap.c

341

static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags);

sys/i386/i386/pmap.c

343

static vm_page_t _pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags);

sys/i386/i386/pmap.c

344

static void _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free);

sys/i386/i386/pmap.c

3443

pmap_invalidate_page_int(pmap, sva);

sys/i386/i386/pmap.c

345

static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va);

sys/i386/i386/pmap.c

3450

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

3455

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

3472

pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, vm_page_t mpte)

sys/i386/i386/pmap.c

3480

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

3489

firstpte = pmap_pte_quick(pmap, trunc_4mpage(va));

sys/i386/i386/pmap.c

3495

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

3498

if ((*firstpte & PG_MANAGED) != 0 && pmap == kernel_pmap) {

sys/i386/i386/pmap.c

3501

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

353

static void pmap_invalidate_all_int(pmap_t pmap);

sys/i386/i386/pmap.c

3531

" in pmap %p", va & ~PDRMASK, pmap);

sys/i386/i386/pmap.c

3547

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

3564

" in pmap %p", oldpteva, pmap);

sys/i386/i386/pmap.c

3569

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

3597

if (pmap_insert_pt_page(pmap, mpte, true, allpte_PG_A != 0)) {

sys/i386/i386/pmap.c

3601

pmap);

sys/i386/i386/pmap.c

3609

pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME);

sys/i386/i386/pmap.c

3621

pmap_update_pde(pmap, va, pde, PG_PS | newpde);

sys/i386/i386/pmap.c

3622

else if (pmap == kernel_pmap)

sys/i386/i386/pmap.c

3629

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

3647

__CONCAT(PMTYPE, enter)(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/i386/i386/pmap.c

3659

KASSERT((pmap == kernel_pmap && va < VM_MAX_KERNEL_ADDRESS) ||

sys/i386/i386/pmap.c

3660

(pmap != kernel_pmap && va < VM_MAXUSER_ADDRESS),

sys/i386/i386/pmap.c

3661

("pmap_enter: toobig k%d %#x", pmap == kernel_pmap, va));

sys/i386/i386/pmap.c

3665

KASSERT(pmap != kernel_pmap || (m->oflags & VPO_UNMANAGED) != 0 ||

sys/i386/i386/pmap.c

3686

if (pmap != kernel_pmap)

sys/i386/i386/pmap.c

3688

newpte |= pmap_cache_bits(pmap, m->md.pat_mode, psind > 0);

sys/i386/i386/pmap.c

3693

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

3699

rv = pmap_enter_pde(pmap, va, newpte | PG_PS, flags, m);

sys/i386/i386/pmap.c

3703

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

3704

if (pmap != kernel_pmap) {

sys/i386/i386/pmap.c

3711

mpte = pmap_allocpte(pmap, va, flags);

sys/i386/i386/pmap.c

3727

(uintmax_t)pmap->pm_pdir[PTDPTDI]));

sys/i386/i386/pmap.c

3729

pmap_demote_pde(pmap, pde, va);

sys/i386/i386/pmap.c

3731

pte = pmap_pte_quick(pmap, va);

sys/i386/i386/pmap.c

3740

(uintmax_t)pmap->pm_pdir[PTDPTDI], va);

sys/i386/i386/pmap.c

3757

pmap->pm_stats.wired_count++;

sys/i386/i386/pmap.c

3759

pmap->pm_stats.wired_count--;

sys/i386/i386/pmap.c

3811

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

3814

pv = pmap_pvh_remove(&om->md, pmap, va);

sys/i386/i386/pmap.c

3818

free_pv_entry(pmap, pv);

sys/i386/i386/pmap.c

3829

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

3837

pmap->pm_stats.wired_count++;

sys/i386/i386/pmap.c

3838

pmap->pm_stats.resident_count++;

sys/i386/i386/pmap.c

3846

pv = get_pv_entry(pmap, false);

sys/i386/i386/pmap.c

3882

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

3896

(void)pmap_promote_pde(pmap, pde, va, mpte);

sys/i386/i386/pmap.c

3903

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

3914

pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/i386/i386/pmap.c

3918

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

3920

pmap_cache_bits(pmap, m->md.pat_mode, true) | PG_PS | PG_V;

sys/i386/i386/pmap.c

3927

if (pmap != kernel_pmap)

sys/i386/i386/pmap.c

3929

return (pmap_enter_pde(pmap, va, newpde, PMAP_ENTER_NOSLEEP |

sys/i386/i386/pmap.c

3965

pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags,

sys/i386/i386/pmap.c

3976

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

3977

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

3984

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

3986

} else if (pmap != kernel_pmap ||

sys/i386/i386/pmap.c

3990

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

4001

(void)pmap_remove_pde(pmap, pde, va, &free);

sys/i386/i386/pmap.c

4003

pmap_invalidate_pde_page(pmap, va, oldpde);

sys/i386/i386/pmap.c

4005

if (pmap_remove_ptes(pmap, va, va + NBPDR, &free))

sys/i386/i386/pmap.c

4006

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

4008

if (pmap != kernel_pmap) {

sys/i386/i386/pmap.c

4021

if (pmap_insert_pt_page(pmap, mt, false, false))

sys/i386/i386/pmap.c

4030

if ((newpde & PG_W) != 0 && pmap != kernel_pmap) {

sys/i386/i386/pmap.c

4036

if (pmap_insert_pt_page(pmap, uwptpg, true, false)) {

sys/i386/i386/pmap.c

4041

pmap->pm_stats.resident_count++;

sys/i386/i386/pmap.c

4048

if (!pmap_pv_insert_pde(pmap, va, newpde, flags)) {

sys/i386/i386/pmap.c

4050

mt = pmap_remove_pt_page(pmap, va);

sys/i386/i386/pmap.c

4054

pmap->pm_stats.resident_count--;

sys/i386/i386/pmap.c

4060

" in pmap %p", va, pmap);

sys/i386/i386/pmap.c

4073

pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE;

sys/i386/i386/pmap.c

4074

pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE;

sys/i386/i386/pmap.c

4084

va, pmap);

sys/i386/i386/pmap.c

4101

__CONCAT(PMTYPE, enter_object)(pmap_t pmap, vm_offset_t start, vm_offset_t end,

sys/i386/i386/pmap.c

4116

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

4121

((rv = pmap_enter_4mpage(pmap, va, m, prot)) ==

sys/i386/i386/pmap.c

4125

mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte);

sys/i386/i386/pmap.c

4130

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

4143

__CONCAT(PMTYPE, enter_quick)(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/i386/i386/pmap.c

4148

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

4149

(void)pmap_enter_quick_locked(pmap, va, m, prot, NULL);

sys/i386/i386/pmap.c

4151

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

4155

pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/i386/i386/pmap.c

4161

KASSERT(pmap != kernel_pmap || !VA_IS_CLEANMAP(va) ||

sys/i386/i386/pmap.c

4165

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/i386/i386/pmap.c

4172

if (pmap != kernel_pmap) {

sys/i386/i386/pmap.c

4186

pde = &pmap->pm_pdir[ptepindex];

sys/i386/i386/pmap.c

4199

mpte = _pmap_allocpte(pmap, ptepindex,

sys/i386/i386/pmap.c

4210

pte = pmap_pte_quick(pmap, va);

sys/i386/i386/pmap.c

4222

!pmap_try_insert_pv_entry(pmap, va, m)) {

sys/i386/i386/pmap.c

4224

pmap_abort_ptp(pmap, va, mpte);

sys/i386/i386/pmap.c

4232

pmap->pm_stats.resident_count++;

sys/i386/i386/pmap.c

4235

pmap_cache_bits(pmap, m->md.pat_mode, false);

sys/i386/i386/pmap.c

4242

if (pmap != kernel_pmap)

sys/i386/i386/pmap.c

4256

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

4262

if (pmap_promote_pde(pmap, pde, va, mpte))

sys/i386/i386/pmap.c

4292

__CONCAT(PMTYPE, object_init_pt)(pmap_t pmap, vm_offset_t addr,

sys/i386/i386/pmap.c

4342

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

4343

for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, true);

sys/i386/i386/pmap.c

4345

pde = pmap_pde(pmap, addr);

sys/i386/i386/pmap.c

4349

pmap->pm_stats.resident_count += NBPDR /

sys/i386/i386/pmap.c

4356

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

4370

__CONCAT(PMTYPE, unwire)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap.c

4377

if (pmap_is_current(pmap))

sys/i386/i386/pmap.c

4385

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

4390

pde = pmap_pde(pmap, sva);

sys/i386/i386/pmap.c

4409

pmap->pm_stats.wired_count -= NBPDR /

sys/i386/i386/pmap.c

4416

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

4422

if (!pmap_demote_pde(pmap, pde, sva))

sys/i386/i386/pmap.c

4428

for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,

sys/i386/i386/pmap.c

4444

pmap->pm_stats.wired_count--;

sys/i386/i386/pmap.c

4451

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

4728

__CONCAT(PMTYPE, page_exists_quick)(pmap_t pmap, vm_page_t m)

sys/i386/i386/pmap.c

4740

if (PV_PMAP(pv) == pmap) {

sys/i386/i386/pmap.c

4751

if (PV_PMAP(pv) == pmap) {

sys/i386/i386/pmap.c

4796

pmap_t pmap;

sys/i386/i386/pmap.c

4803

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

4804

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

4805

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

4808

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

4842

__CONCAT(PMTYPE, remove_pages)(pmap_t pmap)

sys/i386/i386/pmap.c

4855

if (pmap != PCPU_GET(curpmap)) {

sys/i386/i386/pmap.c

4861

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

4863

TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {

sys/i386/i386/pmap.c

4864

KASSERT(pc->pc_pmap == pmap, ("Wrong pmap %p %p", pmap,

sys/i386/i386/pmap.c

4876

pte = pmap_pde(pmap, pv->pv_va);

sys/i386/i386/pmap.c

4879

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

4928

pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;

sys/i386/i386/pmap.c

4936

mpte = pmap_remove_pt_page(pmap, pv->pv_va);

sys/i386/i386/pmap.c

4940

pmap->pm_stats.resident_count--;

sys/i386/i386/pmap.c

4947

pmap->pm_stats.resident_count--;

sys/i386/i386/pmap.c

4955

pmap_unuse_pt(pmap, pv->pv_va, &free);

sys/i386/i386/pmap.c

4960

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/i386/i386/pmap.c

4965

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

4967

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5008

pmap_t pmap;

sys/i386/i386/pmap.c

5015

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5016

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5017

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5019

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5034

__CONCAT(PMTYPE, is_prefaultable)(pmap_t pmap, vm_offset_t addr)

sys/i386/i386/pmap.c

5040

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5041

pde = *pmap_pde(pmap, addr);

sys/i386/i386/pmap.c

5043

rv = pmap_pte_ufast(pmap, addr, pde) == 0;

sys/i386/i386/pmap.c

5044

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5078

pmap_t pmap;

sys/i386/i386/pmap.c

5085

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5086

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5087

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5089

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5105

pmap_t pmap;

sys/i386/i386/pmap.c

5123

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5124

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5125

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

5127

(void)pmap_demote_pde(pmap, pde, va);

sys/i386/i386/pmap.c

5128

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5132

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5133

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5134

pde = pmap_pde(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5137

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5151

pmap_invalidate_page_int(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5153

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5181

pmap_t pmap;

sys/i386/i386/pmap.c

5198

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5199

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5200

pde = pmap_pde(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5229

(uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&

sys/i386/i386/pmap.c

5232

pmap_invalidate_page_int(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5236

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5250

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5251

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5252

pde = pmap_pde(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5256

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5261

pmap_invalidate_page_int(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5264

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5284

__CONCAT(PMTYPE, advise)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/i386/i386/pmap.c

5295

if (pmap_is_current(pmap))

sys/i386/i386/pmap.c

5304

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5309

pde = pmap_pde(pmap, sva);

sys/i386/i386/pmap.c

5320

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

5321

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5326

if (!pmap_demote_pde(pmap, pde, sva)) {

sys/i386/i386/pmap.c

5349

pte = pmap_pte_quick(pmap, va);

sys/i386/i386/pmap.c

5352

pmap_remove_pte(pmap, pte, va, NULL);

sys/i386/i386/pmap.c

5359

for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,

sys/i386/i386/pmap.c

5386

pmap_invalidate_range_int(pmap, va, sva);

sys/i386/i386/pmap.c

5391

pmap_invalidate_range_int(pmap, va, sva);

sys/i386/i386/pmap.c

5394

pmap_invalidate_all_int(pmap);

sys/i386/i386/pmap.c

5399

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5410

pmap_t pmap;

sys/i386/i386/pmap.c

5428

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5429

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5430

pde = pmap_pde(pmap, va);

sys/i386/i386/pmap.c

5434

pmap_demote_pde(pmap, pde, va) &&

sys/i386/i386/pmap.c

5441

pte = pmap_pte_quick(pmap, va);

sys/i386/i386/pmap.c

5449

pmap_invalidate_page_int(pmap, va);

sys/i386/i386/pmap.c

5451

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5455

pmap = PV_PMAP(pv);

sys/i386/i386/pmap.c

5456

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5457

pde = pmap_pde(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5460

pte = pmap_pte_quick(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5468

pmap_invalidate_page_int(pmap, pv->pv_va);

sys/i386/i386/pmap.c

5470

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5823

__CONCAT(PMTYPE, mincore)(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/i386/i386/pmap.c

5830

PMAP_LOCK(pmap);

sys/i386/i386/pmap.c

5831

pde = *pmap_pde(pmap, addr);

sys/i386/i386/pmap.c

5840

pte = pmap_pte_ufast(pmap, addr, pde);

sys/i386/i386/pmap.c

5861

PMAP_UNLOCK(pmap);

sys/i386/i386/pmap.c

5868

pmap_t pmap, oldpmap;

sys/i386/i386/pmap.c

5873

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/i386/i386/pmap.c

5878

CPU_SET_ATOMIC(cpuid, &pmap->pm_active);

sys/i386/i386/pmap.c

5881

CPU_SET(cpuid, &pmap->pm_active);

sys/i386/i386/pmap.c

5884

cr3 = vtophys(pmap->pm_pdpt);

sys/i386/i386/pmap.c

5886

cr3 = vtophys(pmap->pm_pdir);

sys/i386/i386/pmap.c

5892

PCPU_SET(curpmap, pmap);

sys/i386/i386/pmap.c

5897

__CONCAT(PMTYPE, activate_boot)(pmap_t pmap)

sys/i386/i386/pmap.c

5903

CPU_SET_ATOMIC(cpuid, &pmap->pm_active);

sys/i386/i386/pmap.c

5905

CPU_SET(cpuid, &pmap->pm_active);

sys/i386/i386/pmap.c

5907

PCPU_SET(curpmap, pmap);

sys/i386/i386/pmap.c

6137

__CONCAT(PMTYPE, get_cr3)(pmap_t pmap)

sys/i386/i386/pmap.c

6141

return ((u_int)vtophys(pmap->pm_pdpt));

sys/i386/i386/pmap.c

6143

return ((u_int)vtophys(pmap->pm_pdir));

sys/i386/i386/pmap_base.c

106

static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,

sys/i386/i386/pmap_base.c

261

struct pmap kernel_pmap_store;

sys/i386/i386/pmap_base.c

518

pmap_get_cr3(pmap_t pmap)

sys/i386/i386/pmap_base.c

521

return (pmap_methods_ptr->pm_get_cr3(pmap));

sys/i386/i386/pmap_base.c

567

pmap_is_valid_memattr(pmap_t pmap, vm_memattr_t mode)

sys/i386/i386/pmap_base.c

570

return (pmap_methods_ptr->pm_is_valid_memattr(pmap, mode));

sys/i386/i386/pmap_base.c

574

pmap_cache_bits(pmap_t pmap, int mode, bool is_pde)

sys/i386/i386/pmap_base.c

577

return (pmap_methods_ptr->pm_cache_bits(pmap, mode, is_pde));

sys/i386/i386/pmap_base.c

581

pmap_ps_enabled(pmap_t pmap)

sys/i386/i386/pmap_base.c

584

return (pmap_methods_ptr->pm_ps_enabled(pmap));

sys/i386/i386/pmap_base.c

588

pmap_pinit0(pmap_t pmap)

sys/i386/i386/pmap_base.c

591

pmap_methods_ptr->pm_pinit0(pmap);

sys/i386/i386/pmap_base.c

595

pmap_pinit(pmap_t pmap)

sys/i386/i386/pmap_base.c

598

return (pmap_methods_ptr->pm_pinit(pmap));

sys/i386/i386/pmap_base.c

609

pmap_activate_boot(pmap_t pmap)

sys/i386/i386/pmap_base.c

612

pmap_methods_ptr->pm_activate_boot(pmap);

sys/i386/i386/pmap_base.c

616

pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)

sys/i386/i386/pmap_base.c

619

pmap_methods_ptr->pm_advise(pmap, sva, eva, advice);

sys/i386/i386/pmap_base.c

637

pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/i386/i386/pmap_base.c

640

return (pmap_methods_ptr->pm_mincore(pmap, addr, pap));

sys/i386/i386/pmap_base.c

681

pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/i386/i386/pmap_base.c

685

return (pmap_methods_ptr->pm_enter(pmap, va, m, prot, flags, psind));

sys/i386/i386/pmap_base.c

689

pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,

sys/i386/i386/pmap_base.c

693

pmap_methods_ptr->pm_enter_object(pmap, start, end, m_start, prot);

sys/i386/i386/pmap_base.c

697

pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/i386/i386/pmap_base.c

700

pmap_methods_ptr->pm_enter_quick(pmap, va, m, prot);

sys/i386/i386/pmap_base.c

711

pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,

sys/i386/i386/pmap_base.c

715

pmap_methods_ptr->pm_object_init_pt(pmap, addr, object, pindex, size);

sys/i386/i386/pmap_base.c

719

pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap_base.c

722

pmap_methods_ptr->pm_unwire(pmap, sva, eva);

sys/i386/i386/pmap_base.c

726

pmap_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/i386/i386/pmap_base.c

729

return (pmap_methods_ptr->pm_page_exists_quick(pmap, m));

sys/i386/i386/pmap_base.c

747

pmap_remove_pages(pmap_t pmap)

sys/i386/i386/pmap_base.c

750

pmap_methods_ptr->pm_remove_pages(pmap);

sys/i386/i386/pmap_base.c

761

pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/i386/i386/pmap_base.c

764

return (pmap_methods_ptr->pm_is_prefaultable(pmap, addr));

sys/i386/i386/pmap_base.c

826

pmap_extract(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap_base.c

829

return (pmap_methods_ptr->pm_extract(pmap, va));

sys/i386/i386/pmap_base.c

833

pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/i386/i386/pmap_base.c

836

return (pmap_methods_ptr->pm_extract_and_hold(pmap, va, prot));

sys/i386/i386/pmap_base.c

861

pmap_release(pmap_t pmap)

sys/i386/i386/pmap_base.c

864

pmap_methods_ptr->pm_release(pmap);

sys/i386/i386/pmap_base.c

868

pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap_base.c

871

pmap_methods_ptr->pm_remove(pmap, sva, eva);

sys/i386/i386/pmap_base.c

875

pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)

sys/i386/i386/pmap_base.c

878

pmap_methods_ptr->pm_protect(pmap, sva, eva, prot);

sys/i386/i386/pmap_base.c

914

pmap_invalidate_page(pmap_t pmap, vm_offset_t va)

sys/i386/i386/pmap_base.c

917

pmap_methods_ptr->pm_invalidate_page(pmap, va);

sys/i386/i386/pmap_base.c

921

pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/i386/i386/pmap_base.c

924

pmap_methods_ptr->pm_invalidate_range(pmap, sva, eva);

sys/i386/i386/pmap_base.c

928

pmap_invalidate_all(pmap_t pmap)

sys/i386/i386/pmap_base.c

931

pmap_methods_ptr->pm_invalidate_all(pmap);

sys/i386/i386/pmap_base.c

956

pmap_active_cpus(pmap_t pmap, cpuset_t *res)

sys/i386/i386/pmap_base.c

958

*res = pmap->pm_active;

sys/i386/i386/vm86.c

605

if (vmc->pmap[i].pte_num == pagenum)

sys/i386/i386/vm86.c

606

return (vmc->pmap[i].kva);

sys/i386/i386/vm86.c

616

if (vmc->pmap[i].pte_num == pagenum)

sys/i386/i386/vm86.c

628

vmc->pmap[i].flags = flags;

sys/i386/i386/vm86.c

629

vmc->pmap[i].kva = kva;

sys/i386/i386/vm86.c

630

vmc->pmap[i].pte_num = pagenum;

sys/i386/i386/vm86.c

736

page = vtophys(vmc->pmap[i].kva & PG_FRAME_PAE);

sys/i386/i386/vm86.c

737

entry = vmc->pmap[i].pte_num;

sys/i386/i386/vm86.c

738

vmc->pmap[i].old_pte = pte_pae[entry];

sys/i386/i386/vm86.c

740

pmap_invalidate_page(kernel_pmap, vmc->pmap[i].kva);

sys/i386/i386/vm86.c

745

page = vtophys(vmc->pmap[i].kva & PG_FRAME_NOPAE);

sys/i386/i386/vm86.c

746

entry = vmc->pmap[i].pte_num;

sys/i386/i386/vm86.c

747

vmc->pmap[i].old_pte = pte_nopae[entry];

sys/i386/i386/vm86.c

749

pmap_invalidate_page(kernel_pmap, vmc->pmap[i].kva);

sys/i386/i386/vm86.c

762

entry = vmc->pmap[i].pte_num;

sys/i386/i386/vm86.c

763

pte_pae[entry] = vmc->pmap[i].old_pte;

sys/i386/i386/vm86.c

764

pmap_invalidate_page(kernel_pmap, vmc->pmap[i].kva);

sys/i386/i386/vm86.c

768

entry = vmc->pmap[i].pte_num;

sys/i386/i386/vm86.c

769

pte_nopae[entry] = vmc->pmap[i].old_pte;

sys/i386/i386/vm86.c

770

pmap_invalidate_page(kernel_pmap, vmc->pmap[i].kva);

sys/i386/i386/vm86.c

787

if (page == vmc->pmap[i].pte_num)

sys/i386/i386/vm86.c

788

return (vmc->pmap[i].kva + (addr & PAGE_MASK));

sys/i386/i386/vm86.c

799

if (kva >= vmc->pmap[i].kva &&

sys/i386/i386/vm86.c

800

kva < vmc->pmap[i].kva + PAGE_SIZE) {

sys/i386/i386/vm86.c

801

*off = kva - vmc->pmap[i].kva;

sys/i386/i386/vm86.c

802

*sel = vmc->pmap[i].pte_num << 8;

sys/i386/i386/vm_machdep.c

571

sf_buf_shootdown_curcpu_cb(pmap_t pmap __unused,

sys/i386/include/pcpu.h

56

struct pmap *pc_curpmap; \

sys/i386/include/pmap.h

169

LIST_ENTRY(pmap) pm_list; /* List of all pmaps */

sys/i386/include/pmap.h

175

typedef struct pmap *pmap_t;

sys/i386/include/pmap.h

178

extern struct pmap kernel_pmap_store;

sys/i386/include/pmap.h

181

#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)

sys/i386/include/pmap.h

182

#define PMAP_LOCK_ASSERT(pmap, type) \

sys/i386/include/pmap.h

183

mtx_assert(&(pmap)->pm_mtx, (type))

sys/i386/include/pmap.h

184

#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)

sys/i386/include/pmap.h

185

#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \

sys/i386/include/pmap.h

187

#define PMAP_LOCKED(pmap) mtx_owned(&(pmap)->pm_mtx)

sys/i386/include/pmap.h

188

#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)

sys/i386/include/pmap.h

189

#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)

sys/i386/include/pmap.h

190

#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)

sys/i386/include/pmap.h

216

void pmap_activate_boot(pmap_t pmap);

sys/i386/include/pmap.h

242

bool pmap_ps_enabled(pmap_t pmap);

sys/i386/include/pmap.h

257

#define pmap_map_delete(pmap, sva, eva) pmap_remove(pmap, sva, eva)

sys/i386/include/pmap_nopae.h

94

struct pmap;

sys/i386/include/pmap_nopae.h

95

pt_entry_t *__CONCAT(PMTYPE, pmap_pte)(struct pmap *, vm_offset_t) __pure2;

sys/i386/include/vm86.h

113

} pmap[VM86_PMAPSIZE];

sys/kern/kern_proc.c

2470

if (pmap_extract(map->pmap, addr))

sys/kern/kern_proc.c

2610

pmap_mincore(map->pmap, addr, &pa) & MINCORE_SUPER) != 0) {

sys/kern/subr_bus_dma.c

103

bus_dma_segment_t *list, int sglist_cnt, struct pmap *pmap, int *nsegs,

sys/kern/subr_bus_dma.c

122

error = _bus_dmamap_load_buffer(dmat, map, addr, ds_len, pmap,

sys/kern/subr_bus_dma.c

289

pmap_t pmap;

sys/kern/subr_bus_dma.c

296

pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);

sys/kern/subr_bus_dma.c

298

pmap = kernel_pmap;

sys/kern/subr_bus_dma.c

313

minlen, pmap, flags, NULL, nsegs);

sys/kern/subr_sglist.c

108

_sglist_append_buf(struct sglist *sg, void *buf, size_t len, pmap_t pmap,

sys/kern/subr_sglist.c

125

if (pmap != NULL)

sys/kern/subr_sglist.c

126

paddr = pmap_extract(pmap, vaddr);

sys/kern/subr_sglist.c

148

if (pmap != NULL)

sys/kern/subr_sglist.c

149

paddr = pmap_extract(pmap, vaddr);

sys/kern/subr_sglist.c

601

pmap_t pmap;

sys/kern/subr_sglist.c

613

pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);

sys/kern/subr_sglist.c

615

pmap = NULL;

sys/kern/subr_sglist.c

627

pmap, NULL);

sys/kern/subr_sglist.c

648

pmap_t pmap;

sys/kern/subr_sglist.c

657

pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);

sys/kern/subr_sglist.c

659

pmap = NULL;

sys/kern/subr_sglist.c

677

error = _sglist_append_buf(sg, iov->iov_base, len, pmap, &done);

sys/powerpc/aim/mmu_oea.c

1126

moea_ps_enabled(pmap_t pmap __unused)

sys/powerpc/aim/mmu_oea.c

1137

moea_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/powerpc/aim/mmu_oea.c

1144

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1145

error = moea_enter_locked(pmap, va, m, prot, flags, psind);

sys/powerpc/aim/mmu_oea.c

1147

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1165

moea_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/powerpc/aim/mmu_oea.c

1175

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_oea.c

1206

error = moea_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m),

sys/powerpc/aim/mmu_oea.c

1215

if (pmap != kernel_pmap && error == ENOENT &&

sys/powerpc/aim/mmu_oea.c

1295

moea_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/powerpc/aim/mmu_oea.c

1301

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1302

pvo = moea_pvo_find_va(pmap, va & ~ADDR_POFF, NULL);

sys/powerpc/aim/mmu_oea.c

1310

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1361

moea_is_prefaultable(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_oea.c

1366

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1367

pvo = moea_pvo_find_va(pmap, va & ~ADDR_POFF, NULL);

sys/powerpc/aim/mmu_oea.c

1369

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1396

pmap_t pmap;

sys/powerpc/aim/mmu_oea.c

1409

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea.c

1410

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1424

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1468

pmap_t pmap;

sys/powerpc/aim/mmu_oea.c

1484

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea.c

1485

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1496

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1667

moea_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/powerpc/aim/mmu_oea.c

1679

if (pvo->pvo_pmap == pmap) {

sys/powerpc/aim/mmu_oea.c

1723

moea_pinit(pmap_t pmap)

sys/powerpc/aim/mmu_oea.c

1728

RB_INIT(&pmap->pmap_pvo);

sys/powerpc/aim/mmu_oea.c

1733

if ((pmap->pmap_phys = (pmap_t)moea_kextract((vm_offset_t)pmap))

sys/powerpc/aim/mmu_oea.c

1735

pmap->pmap_phys = pmap;

sys/powerpc/aim/mmu_oea.c

1774

pmap->pm_sr[i] = VSID_MAKE(i, hash);

sys/powerpc/aim/mmu_oea.c

1878

moea_release(pmap_t pmap)

sys/powerpc/aim/mmu_oea.c

1885

if (pmap->pm_sr[0] == 0)

sys/powerpc/aim/mmu_oea.c

1889

idx = VSID_TO_HASH(pmap->pm_sr[0]) & (NPMAPS-1);

sys/powerpc/aim/mmu_oea.c

1925

pmap_t pmap;

sys/powerpc/aim/mmu_oea.c

1932

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea.c

1933

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea.c

1935

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea.c

329

bool moea_ps_enabled(pmap_t pmap);

sys/powerpc/aim/mmu_oea64.c

1362

moea64_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/powerpc/aim/mmu_oea64.c

1370

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1372

pvo = moea64_pvo_find_va(pmap, addr);

sys/powerpc/aim/mmu_oea64.c

1382

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1386

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1604

moea64_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_oea64.c

1621

return (moea64_sp_enter(pmap, va, m, prot, flags, psind));

sys/powerpc/aim/mmu_oea64.c

1644

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1646

init_pvo_entry(pvo, pmap, va);

sys/powerpc/aim/mmu_oea64.c

1648

if (moea64_ps_enabled(pmap) &&

sys/powerpc/aim/mmu_oea64.c

1649

(tpvo = moea64_pvo_find_va(pmap, va & ~HPT_SP_MASK)) != NULL &&

sys/powerpc/aim/mmu_oea64.c

1677

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1684

KASSERT(oldpvo->pvo_pmap == pmap, ("pmap of old "

sys/powerpc/aim/mmu_oea64.c

1690

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1704

if (pmap != kernel_pmap && (m->a.flags & PGA_EXECUTABLE) == 0 &&

sys/powerpc/aim/mmu_oea64.c

1707

moea64_syncicache(pmap, va, pa, PAGE_SIZE);

sys/powerpc/aim/mmu_oea64.c

1720

if (moea64_ps_enabled(pmap) && pmap != kernel_pmap && pvo != NULL &&

sys/powerpc/aim/mmu_oea64.c

1727

moea64_sp_promote(pmap, va, m);

sys/powerpc/aim/mmu_oea64.c

1734

moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/powerpc/aim/mmu_oea64.c

1753

} else if (pmap == kernel_pmap) {

sys/powerpc/aim/mmu_oea64.c

1846

moea64_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/powerpc/aim/mmu_oea64.c

1852

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1853

pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);

sys/powerpc/aim/mmu_oea64.c

1859

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1978

moea64_is_prefaultable(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_oea64.c

1983

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

1984

pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);

sys/powerpc/aim/mmu_oea64.c

1987

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

2012

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

2025

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

2026

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

2042

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

2079

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

2097

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

2098

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

2123

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

2344

moea64_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/powerpc/aim/mmu_oea64.c

2356

if (!(pvo->pvo_vaddr & PVO_DEAD) && pvo->pvo_pmap == pmap) {

sys/powerpc/aim/mmu_oea64.c

2453

moea64_pinit(pmap_t pmap)

sys/powerpc/aim/mmu_oea64.c

2456

RB_INIT(&pmap->pmap_pvo);

sys/powerpc/aim/mmu_oea64.c

2458

pmap->pm_slb_tree_root = slb_alloc_tree();

sys/powerpc/aim/mmu_oea64.c

2459

pmap->pm_slb = slb_alloc_user_cache();

sys/powerpc/aim/mmu_oea64.c

2460

pmap->pm_slb_len = 0;

sys/powerpc/aim/mmu_oea64.c

2466

moea64_pinit(pmap_t pmap)

sys/powerpc/aim/mmu_oea64.c

2471

RB_INIT(&pmap->pmap_pvo);

sys/powerpc/aim/mmu_oea64.c

2474

pmap->pmap_phys = (pmap_t)moea64_kextract((vm_offset_t)pmap);

sys/powerpc/aim/mmu_oea64.c

2476

pmap->pmap_phys = pmap;

sys/powerpc/aim/mmu_oea64.c

2484

pmap->pm_sr[i] = VSID_MAKE(i, hash);

sys/powerpc/aim/mmu_oea64.c

2486

KASSERT(pmap->pm_sr[0] != 0, ("moea64_pinit: pm_sr[0] = 0"));

sys/powerpc/aim/mmu_oea64.c

263

static void moea64_syncicache(pmap_t pmap, vm_offset_t va,

sys/powerpc/aim/mmu_oea64.c

2640

moea64_release(pmap_t pmap)

sys/powerpc/aim/mmu_oea64.c

2647

slb_free_tree(pmap);

sys/powerpc/aim/mmu_oea64.c

2648

slb_free_user_cache(pmap->pm_slb);

sys/powerpc/aim/mmu_oea64.c

2650

KASSERT(pmap->pm_sr[0] != 0, ("moea64_release: pm_sr[0] = 0"));

sys/powerpc/aim/mmu_oea64.c

2652

moea64_release_vsid(VSID_TO_HASH(pmap->pm_sr[0]));

sys/powerpc/aim/mmu_oea64.c

2761

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

2767

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

2768

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

2781

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

331

static int moea64_sp_enter(pmap_t pmap, vm_offset_t va,

sys/powerpc/aim/mmu_oea64.c

337

static void moea64_sp_promote(pmap_t pmap, vm_offset_t va, vm_page_t m);

sys/powerpc/aim/mmu_oea64.c

3536

moea64_ps_enabled(pmap_t pmap)

sys/powerpc/aim/mmu_oea64.c

3645

moea64_sp_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_oea64.c

3663

KASSERT(pmap != kernel_pmap,

sys/powerpc/aim/mmu_oea64.c

3696

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

3699

moea64_remove_locked(pmap, va, va + HPT_SP_SIZE, &tofree);

sys/powerpc/aim/mmu_oea64.c

3721

init_pvo_entry(pvo, pmap, va);

sys/powerpc/aim/mmu_oea64.c

3733

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

3752

moea64_syncicache(pmap, sva, spa, HPT_SP_SIZE);

sys/powerpc/aim/mmu_oea64.c

3756

__func__, (uintmax_t)sva, pmap);

sys/powerpc/aim/mmu_oea64.c

3764

moea64_sp_promote(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/powerpc/aim/mmu_oea64.c

3780

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

3798

first = pvo = moea64_pvo_find_va(pmap, sva);

sys/powerpc/aim/mmu_oea64.c

3804

__func__, pmap, (uintmax_t)va);

sys/powerpc/aim/mmu_oea64.c

3810

__func__, pmap, (uintmax_t)va,

sys/powerpc/aim/mmu_oea64.c

3819

__func__, pmap, (uintmax_t)va,

sys/powerpc/aim/mmu_oea64.c

3828

__func__, pmap, (uintmax_t)va,

sys/powerpc/aim/mmu_oea64.c

3837

__func__, pmap, (uintmax_t)va,

sys/powerpc/aim/mmu_oea64.c

3844

pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo);

sys/powerpc/aim/mmu_oea64.c

3864

pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {

sys/powerpc/aim/mmu_oea64.c

3877

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

3882

__func__, (uintmax_t)sva, pmap);

sys/powerpc/aim/mmu_oea64.c

3887

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

3898

pmap_t pmap __diagused;

sys/powerpc/aim/mmu_oea64.c

3903

pmap = sp->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

3904

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_oea64.c

3916

pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo),

sys/powerpc/aim/mmu_oea64.c

3946

__func__, (uintmax_t)PVO_VADDR(sp), pmap);

sys/powerpc/aim/mmu_oea64.c

4096

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

4101

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

4102

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_oea64.c

4106

sp = moea64_pvo_find_va(pmap, PVO_VADDR(pvo) & ~HPT_SP_MASK);

sys/powerpc/aim/mmu_oea64.c

4115

pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {

sys/powerpc/aim/mmu_oea64.c

4141

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

4145

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

4146

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4154

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4159

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4172

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

4177

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

4178

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4186

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4192

sp = moea64_pvo_find_va(pmap, PVO_VADDR(pvo) & ~HPT_SP_MASK);

sys/powerpc/aim/mmu_oea64.c

4201

pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {

sys/powerpc/aim/mmu_oea64.c

4209

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4222

pmap_t pmap;

sys/powerpc/aim/mmu_oea64.c

4225

pmap = pvo->pvo_pmap;

sys/powerpc/aim/mmu_oea64.c

4242

(PVO_VADDR(pvo) >> HPT_SP_SHIFT) ^ (uintptr_t)pmap) &

sys/powerpc/aim/mmu_oea64.c

4267

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4276

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4284

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

4296

moea64_pvo_protect(pmap, pvo,

sys/powerpc/aim/mmu_oea64.c

4299

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_oea64.c

537

init_pvo_entry(struct pvo_entry *pvo, pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_oea64.c

543

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_oea64.c

545

pvo->pvo_pmap = pmap;

sys/powerpc/aim/mmu_oea64.c

548

vsid = va_to_vsid(pmap, va);

sys/powerpc/aim/mmu_oea64.c

552

if (pmap == kernel_pmap && (pvo->pvo_vaddr & PVO_LARGE) != 0)

sys/powerpc/aim/mmu_radix.c

1020

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

1025

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

1073

pmap_invalidate_page(pmap_t pmap, vm_offset_t start)

sys/powerpc/aim/mmu_radix.c

1076

if (pmap == kernel_pmap)

sys/powerpc/aim/mmu_radix.c

1079

radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);

sys/powerpc/aim/mmu_radix.c

1084

pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start)

sys/powerpc/aim/mmu_radix.c

1087

if (pmap == kernel_pmap)

sys/powerpc/aim/mmu_radix.c

1090

radix_tlbie_invlpg_user_2m(pmap->pm_pid, start);

sys/powerpc/aim/mmu_radix.c

1095

pmap_invalidate_pwc(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

1098

if (pmap == kernel_pmap)

sys/powerpc/aim/mmu_radix.c

1101

radix_tlbie_invlpwc_user(pmap->pm_pid);

sys/powerpc/aim/mmu_radix.c

1106

pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)

sys/powerpc/aim/mmu_radix.c

1109

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

1113

if (pmap == kernel_pmap) {

sys/powerpc/aim/mmu_radix.c

1120

radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);

sys/powerpc/aim/mmu_radix.c

1128

pmap_invalidate_all(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

1131

if (pmap == kernel_pmap)

sys/powerpc/aim/mmu_radix.c

1134

radix_tlbie_flush_user(pmap->pm_pid);

sys/powerpc/aim/mmu_radix.c

1139

pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e)

sys/powerpc/aim/mmu_radix.c

1155

pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1);

sys/powerpc/aim/mmu_radix.c

1157

pmap_invalidate_page_2m(pmap, va);

sys/powerpc/aim/mmu_radix.c

1159

pmap_invalidate_pwc(pmap);

sys/powerpc/aim/mmu_radix.c

1183

reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)

sys/powerpc/aim/mmu_radix.c

1191

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

1203

TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {

sys/powerpc/aim/mmu_radix.c

1219

m = reclaim_pv_chunk(pmap, lockp);

sys/powerpc/aim/mmu_radix.c

1228

pc->pc_pmap = pmap;

sys/powerpc/aim/mmu_radix.c

1231

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1257

pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

1270

if (pmap == PV_PMAP(pv) && va == pv->pv_va) {

sys/powerpc/aim/mmu_radix.c

1285

pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/powerpc/aim/mmu_radix.c

1295

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

1307

pv = pmap_pvh_remove(pvh, pmap, va);

sys/powerpc/aim/mmu_radix.c

1317

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/powerpc/aim/mmu_radix.c

1337

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1338

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1342

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1343

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1350

reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap)

sys/powerpc/aim/mmu_radix.c

1353

if (pmap == NULL)

sys/powerpc/aim/mmu_radix.c

1355

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

1356

if (pmap != locked_pmap)

sys/powerpc/aim/mmu_radix.c

1357

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

1379

pmap_t next_pmap, pmap;

sys/powerpc/aim/mmu_radix.c

1390

pmap = NULL;

sys/powerpc/aim/mmu_radix.c

1421

if (pmap != next_pmap) {

sys/powerpc/aim/mmu_radix.c

1422

reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);

sys/powerpc/aim/mmu_radix.c

1423

pmap = next_pmap;

sys/powerpc/aim/mmu_radix.c

1425

if (pmap > locked_pmap) {

sys/powerpc/aim/mmu_radix.c

1427

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

1430

} else if (pmap != locked_pmap) {

sys/powerpc/aim/mmu_radix.c

1431

if (PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

1435

pmap = NULL; /* pmap is not locked */

sys/powerpc/aim/mmu_radix.c

1456

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

1481

pmap_unuse_pt(pmap, va, be64toh(*l3e), &free);

sys/powerpc/aim/mmu_radix.c

1490

pmap_resident_count_dec(pmap, freed);

sys/powerpc/aim/mmu_radix.c

1494

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1506

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1509

if (pmap == locked_pmap)

sys/powerpc/aim/mmu_radix.c

1514

if (active_reclaims == 1 && pmap != NULL) {

sys/powerpc/aim/mmu_radix.c

1533

reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);

sys/powerpc/aim/mmu_radix.c

1548

free_pv_entry(pmap_t pmap, pv_entry_t pv)

sys/powerpc/aim/mmu_radix.c

1554

if (pmap != kernel_pmap)

sys/powerpc/aim/mmu_radix.c

1555

printf("%s(%p, %p)\n", __func__, pmap, pv);

sys/powerpc/aim/mmu_radix.c

1557

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

1568

if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {

sys/powerpc/aim/mmu_radix.c

1569

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1570

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1574

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1605

get_pv_entry(pmap_t pmap, struct rwlock **lockp)

sys/powerpc/aim/mmu_radix.c

1612

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

1615

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/powerpc/aim/mmu_radix.c

1628

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1629

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,

sys/powerpc/aim/mmu_radix.c

1645

m = reclaim_pv_chunk(pmap, lockp);

sys/powerpc/aim/mmu_radix.c

1653

pc->pc_pmap = pmap;

sys/powerpc/aim/mmu_radix.c

1660

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

1674

pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/powerpc/aim/mmu_radix.c

1695

pv = pmap_pvh_remove(&m->md, pmap, va);

sys/powerpc/aim/mmu_radix.c

1705

pmap_pvh_free(&m->md, pmap, va);

sys/powerpc/aim/mmu_radix.c

1716

pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

1720

pv = pmap_pvh_remove(pvh, pmap, va);

sys/powerpc/aim/mmu_radix.c

1722

free_pv_entry(pmap, pv);

sys/powerpc/aim/mmu_radix.c

1730

pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

1735

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

1737

if ((pv = get_pv_entry(pmap, NULL)) != NULL) {

sys/powerpc/aim/mmu_radix.c

1844

bzero(kernel_pmap, sizeof(struct pmap));

sys/powerpc/aim/mmu_radix.c

2217

mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/powerpc/aim/mmu_radix.c

2232

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2234

l1e = pmap_pml1e(pmap, sva);

sys/powerpc/aim/mmu_radix.c

2259

if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) {

sys/powerpc/aim/mmu_radix.c

2288

pmap_remove_pte(pmap, pte, va, be64toh(*l3e), NULL,

sys/powerpc/aim/mmu_radix.c

2300

MPASS(pte == pmap_pte(pmap, sva));

sys/powerpc/aim/mmu_radix.c

2331

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

2332

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2427

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

2453

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

2454

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

2457

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2460

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2465

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

2468

pmap_demote_l3e_locked(pmap, l3e, va, &lock) &&

sys/powerpc/aim/mmu_radix.c

2484

pmap_invalidate_page(pmap, va);

sys/powerpc/aim/mmu_radix.c

2486

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2489

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

2490

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

2494

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2497

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2501

l3e = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

2507

pmap_invalidate_page(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

2509

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2718

pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va,

sys/powerpc/aim/mmu_radix.c

2725

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

2737

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

2761

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

2774

(va & ~L3_PAGE_MASK), pmap);

sys/powerpc/aim/mmu_radix.c

2778

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

2795

if (pmap_insert_pt_page(pmap, mpte)) {

sys/powerpc/aim/mmu_radix.c

2798

pmap);

sys/powerpc/aim/mmu_radix.c

2806

pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp);

sys/powerpc/aim/mmu_radix.c

2812

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

2821

mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

2837

CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va,

sys/powerpc/aim/mmu_radix.c

2879

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

2884

rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock);

sys/powerpc/aim/mmu_radix.c

2894

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

2896

pmap_demote_l3e_locked(pmap, l3e, va, &lock))) {

sys/powerpc/aim/mmu_radix.c

2908

mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va),

sys/powerpc/aim/mmu_radix.c

2932

pmap, va, m, prot, flags, psind, pmap->pm_pid,

sys/powerpc/aim/mmu_radix.c

2935

pmap_pte_walk(pmap->pm_pml1, va);

sys/powerpc/aim/mmu_radix.c

2946

pmap->pm_stats.wired_count++;

sys/powerpc/aim/mmu_radix.c

2948

pmap->pm_stats.wired_count--;

sys/powerpc/aim/mmu_radix.c

3014

pv = pmap_pvh_remove(&om->md, pmap, va);

sys/powerpc/aim/mmu_radix.c

3016

free_pv_entry(pmap, pv);

sys/powerpc/aim/mmu_radix.c

3037

if (pmap != kernel_pmap) {

sys/powerpc/aim/mmu_radix.c

3041

pmap, va, m, prot, flags, psind,

sys/powerpc/aim/mmu_radix.c

3042

pmap->pm_pid, curproc->p_pid,

sys/powerpc/aim/mmu_radix.c

3051

pmap->pm_stats.wired_count++;

sys/powerpc/aim/mmu_radix.c

3052

pmap_resident_count_inc(pmap, 1);

sys/powerpc/aim/mmu_radix.c

3060

pv = get_pv_entry(pmap, &lock);

sys/powerpc/aim/mmu_radix.c

3066

pv, pmap);

sys/powerpc/aim/mmu_radix.c

3115

mmu_radix_ps_enabled(pmap) &&

sys/powerpc/aim/mmu_radix.c

3118

pmap_promote_l3e(pmap, l3e, va, &lock) == 0)

sys/powerpc/aim/mmu_radix.c

3122

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

3124

pmap_invalidate_page(pmap, va);

sys/powerpc/aim/mmu_radix.c

3130

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3140

pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t pdpg)

sys/powerpc/aim/mmu_radix.c

3145

if (pmap_unwire_ptp(pmap, va, pdpg, &free)) {

sys/powerpc/aim/mmu_radix.c

3152

pmap_invalidate_page(pmap, va);

sys/powerpc/aim/mmu_radix.c

3165

pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/powerpc/aim/mmu_radix.c

3170

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

3181

return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP |

sys/powerpc/aim/mmu_radix.c

3198

pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags,

sys/powerpc/aim/mmu_radix.c

3208

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

3210

if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?

sys/powerpc/aim/mmu_radix.c

3213

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

3225

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

3237

(void)pmap_remove_l3e(pmap, l3e, va, &free, lockp);

sys/powerpc/aim/mmu_radix.c

3238

pmap_invalidate_l3e_page(pmap, va, oldl3e);

sys/powerpc/aim/mmu_radix.c

3240

if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e,

sys/powerpc/aim/mmu_radix.c

3242

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

3247

if (pmap_insert_pt_page(pmap, mt)) {

sys/powerpc/aim/mmu_radix.c

3264

if ((newpde & PG_W) != 0 && pmap != kernel_pmap) {

sys/powerpc/aim/mmu_radix.c

3267

pmap_abort_ptp(pmap, va, pdpg);

sys/powerpc/aim/mmu_radix.c

3271

if (pmap_insert_pt_page(pmap, uwptpg)) {

sys/powerpc/aim/mmu_radix.c

3274

pmap_abort_ptp(pmap, va, pdpg);

sys/powerpc/aim/mmu_radix.c

3277

pmap_resident_count_inc(pmap, 1);

sys/powerpc/aim/mmu_radix.c

3285

if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) {

sys/powerpc/aim/mmu_radix.c

3286

pmap_abort_ptp(pmap, va, pdpg);

sys/powerpc/aim/mmu_radix.c

3288

mt = pmap_remove_pt_page(pmap, va);

sys/powerpc/aim/mmu_radix.c

3292

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

3298

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

3311

pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE;

sys/powerpc/aim/mmu_radix.c

3312

pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);

sys/powerpc/aim/mmu_radix.c

3323

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

3328

mmu_radix_enter_object(pmap_t pmap, vm_offset_t start,

sys/powerpc/aim/mmu_radix.c

3339

CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start,

sys/powerpc/aim/mmu_radix.c

3347

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3351

m->psind == 1 && mmu_radix_ps_enabled(pmap) &&

sys/powerpc/aim/mmu_radix.c

3352

pmap_enter_2mpage(pmap, va, m, prot, &lock)) {

sys/powerpc/aim/mmu_radix.c

3355

mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot,

sys/powerpc/aim/mmu_radix.c

3364

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

3365

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3369

mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

3379

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

3399

ptepa = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

3417

mpte = _pmap_allocpte(pmap, ptepindex, NULL);

sys/powerpc/aim/mmu_radix.c

3426

pte = pmap_pte(pmap, va);

sys/powerpc/aim/mmu_radix.c

3440

!pmap_try_insert_pv_entry(pmap, va, m, lockp)) {

sys/powerpc/aim/mmu_radix.c

3443

if (pmap_unwire_ptp(pmap, va, mpte, &free)) {

sys/powerpc/aim/mmu_radix.c

3461

pmap_resident_count_inc(pmap, 1);

sys/powerpc/aim/mmu_radix.c

3476

mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

3484

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3485

mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock,

sys/powerpc/aim/mmu_radix.c

3491

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

3492

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3496

mmu_radix_extract(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

3502

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

3528

mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/powerpc/aim/mmu_radix.c

3535

CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot);

sys/powerpc/aim/mmu_radix.c

3536

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3537

l3ep = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

3553

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

364

pmap_pml1e(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

367

return (&pmap->pm_pml1[pmap_pml1e_index(va)]);

sys/powerpc/aim/mmu_radix.c

371

pmap_pml2e(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

375

l1e = pmap_pml1e(pmap, va);

sys/powerpc/aim/mmu_radix.c

3756

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

3765

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

3766

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

3769

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3772

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3776

pte = pmap_pte(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

3783

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3790

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

3791

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

3795

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3799

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3803

pte = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

3810

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

382

pmap_pml3e(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

3843

mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/powerpc/aim/mmu_radix.c

3849

CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);

sys/powerpc/aim/mmu_radix.c

3851

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3852

l3e = pmap_pml3e(pmap, addr);

sys/powerpc/aim/mmu_radix.c

3857

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

386

l2e = pmap_pml2e(pmap, va);

sys/powerpc/aim/mmu_radix.c

3895

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

3920

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

3921

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

3924

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3927

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

393

pmap_pte(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

3931

l3e = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

3961

(uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&

sys/powerpc/aim/mmu_radix.c

3964

pmap_invalidate_page(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

397

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

3972

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3989

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

3990

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

3994

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

3997

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4001

l3e = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

4010

pmap_invalidate_page(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

4013

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4039

mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr,

sys/powerpc/aim/mmu_radix.c

4048

CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr,

sys/powerpc/aim/mmu_radix.c

4055

if (!mmu_radix_ps_enabled(pmap))

sys/powerpc/aim/mmu_radix.c

4089

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4092

pdpg = pmap_allocl3e(pmap, addr, NULL);

sys/powerpc/aim/mmu_radix.c

4109

pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);

sys/powerpc/aim/mmu_radix.c

4121

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4126

mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/powerpc/aim/mmu_radix.c

4136

CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m);

sys/powerpc/aim/mmu_radix.c

4141

if (PV_PMAP(pv) == pmap) {

sys/powerpc/aim/mmu_radix.c

4152

if (PV_PMAP(pv) == pmap) {

sys/powerpc/aim/mmu_radix.c

4179

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

4192

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

4193

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

4196

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4199

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4203

pte = pmap_pte(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

4206

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4211

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

4212

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

4216

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4220

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4224

pte = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

4227

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4241

mmu_radix_pinit(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

4246

CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);

sys/powerpc/aim/mmu_radix.c

4251

pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK);

sys/powerpc/aim/mmu_radix.c

4254

pagezero((char *)pmap->pm_pml1 + j * PAGE_SIZE);

sys/powerpc/aim/mmu_radix.c

4255

vm_radix_init(&pmap->pm_radix);

sys/powerpc/aim/mmu_radix.c

4256

TAILQ_INIT(&pmap->pm_pvchunk);

sys/powerpc/aim/mmu_radix.c

4257

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/powerpc/aim/mmu_radix.c

4258

pmap->pm_flags = PMAP_PDE_SUPERPAGE;

sys/powerpc/aim/mmu_radix.c

4261

pmap->pm_pid = pid;

sys/powerpc/aim/mmu_radix.c

4262

l1pa = DMAP_TO_PHYS(pmap->pm_pml1);

sys/powerpc/aim/mmu_radix.c

4281

_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)

sys/powerpc/aim/mmu_radix.c

4285

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4293

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4295

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4316

l1e = &pmap->pm_pml1[pml1index];

sys/powerpc/aim/mmu_radix.c

4330

l1e = &pmap->pm_pml1[pml1index];

sys/powerpc/aim/mmu_radix.c

4333

if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index,

sys/powerpc/aim/mmu_radix.c

4363

l1e = &pmap->pm_pml1[pml1index];

sys/powerpc/aim/mmu_radix.c

4366

if (_pmap_allocpte(pmap, NUPDE + pdpindex,

sys/powerpc/aim/mmu_radix.c

437

vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va);

sys/powerpc/aim/mmu_radix.c

4379

if (_pmap_allocpte(pmap, NUPDE + pdpindex,

sys/powerpc/aim/mmu_radix.c

4400

pmap_resident_count_inc(pmap, 1);

sys/powerpc/aim/mmu_radix.c

4404

pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)

sys/powerpc/aim/mmu_radix.c

4411

pdpe = pmap_pml2e(pmap, va);

sys/powerpc/aim/mmu_radix.c

4420

pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp);

sys/powerpc/aim/mmu_radix.c

4428

pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)

sys/powerpc/aim/mmu_radix.c

4442

pd = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

4449

if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) {

sys/powerpc/aim/mmu_radix.c

4470

m = _pmap_allocpte(pmap, ptepindex, lockp);

sys/powerpc/aim/mmu_radix.c

4478

mmu_radix_pinit0(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

4481

CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);

sys/powerpc/aim/mmu_radix.c

4482

PMAP_LOCK_INIT(pmap);

sys/powerpc/aim/mmu_radix.c

4483

pmap->pm_pml1 = kernel_pmap->pm_pml1;

sys/powerpc/aim/mmu_radix.c

4484

pmap->pm_pid = kernel_pmap->pm_pid;

sys/powerpc/aim/mmu_radix.c

4486

vm_radix_init(&pmap->pm_radix);

sys/powerpc/aim/mmu_radix.c

4487

TAILQ_INIT(&pmap->pm_pvchunk);

sys/powerpc/aim/mmu_radix.c

4488

bzero(&pmap->pm_stats, sizeof pmap->pm_stats);

sys/powerpc/aim/mmu_radix.c

4490

pmap->pm_flags = PMAP_PDE_SUPERPAGE;

sys/powerpc/aim/mmu_radix.c

4496

pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot)

sys/powerpc/aim/mmu_radix.c

4503

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4536

mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/powerpc/aim/mmu_radix.c

4546

CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva,

sys/powerpc/aim/mmu_radix.c

4551

mmu_radix_remove(pmap, sva, eva);

sys/powerpc/aim/mmu_radix.c

4562

pmap, sva, eva, prot, pmap->pm_pid);

sys/powerpc/aim/mmu_radix.c

4566

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4568

l1e = pmap_pml1e(pmap, sva);

sys/powerpc/aim/mmu_radix.c

4606

if (pmap_protect_l3e(pmap, l3e, sva, prot))

sys/powerpc/aim/mmu_radix.c

4609

} else if (!pmap_demote_l3e(pmap, l3e, sva)) {

sys/powerpc/aim/mmu_radix.c

4626

MPASS(pte == pmap_pte(pmap, sva));

sys/powerpc/aim/mmu_radix.c

4658

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

4659

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

4750

pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)

sys/powerpc/aim/mmu_radix.c

4753

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4754

return (vm_radix_insert(&pmap->pm_radix, mpte));

sys/powerpc/aim/mmu_radix.c

4764

pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

4767

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4768

return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va)));

sys/powerpc/aim/mmu_radix.c

4778

pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/powerpc/aim/mmu_radix.c

4783

_pmap_unwire_ptp(pmap, va, m, free);

sys/powerpc/aim/mmu_radix.c

4790

_pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/powerpc/aim/mmu_radix.c

4793

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4800

pml1 = pmap_pml1e(pmap, va);

sys/powerpc/aim/mmu_radix.c

4805

l2e = pmap_pml2e(pmap, va);

sys/powerpc/aim/mmu_radix.c

4810

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

4813

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

4818

pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME);

sys/powerpc/aim/mmu_radix.c

4819

pmap_unwire_ptp(pmap, va, pdpg, free);

sys/powerpc/aim/mmu_radix.c

4825

pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME);

sys/powerpc/aim/mmu_radix.c

4826

pmap_unwire_ptp(pmap, va, pdppg, free);

sys/powerpc/aim/mmu_radix.c

4841

pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde,

sys/powerpc/aim/mmu_radix.c

4850

return (pmap_unwire_ptp(pmap, va, mpte, free));

sys/powerpc/aim/mmu_radix.c

4854

mmu_radix_release(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

4857

CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);

sys/powerpc/aim/mmu_radix.c

4858

KASSERT(pmap->pm_stats.resident_count == 0,

sys/powerpc/aim/mmu_radix.c

4860

pmap->pm_stats.resident_count));

sys/powerpc/aim/mmu_radix.c

4861

KASSERT(vm_radix_is_empty(&pmap->pm_radix),

sys/powerpc/aim/mmu_radix.c

4864

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

4865

isa3_proctab[pmap->pm_pid].proctab0 = 0;

sys/powerpc/aim/mmu_radix.c

4866

uma_zfree(zone_radix_pgd, pmap->pm_pml1);

sys/powerpc/aim/mmu_radix.c

4867

vmem_free(asid_arena, pmap->pm_pid, 1);

sys/powerpc/aim/mmu_radix.c

4876

pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags,

sys/powerpc/aim/mmu_radix.c

4883

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4885

if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?

sys/powerpc/aim/mmu_radix.c

4912

pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

4918

rv = pmap_demote_l3e_locked(pmap, pde, va, &lock);

sys/powerpc/aim/mmu_radix.c

4925

pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,

sys/powerpc/aim/mmu_radix.c

4935

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

4940

if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==

sys/powerpc/aim/mmu_radix.c

4963

pmap_remove_l3e(pmap, l3e, sva, &free, lockp);

sys/powerpc/aim/mmu_radix.c

4964

pmap_invalidate_l3e_page(pmap, sva, oldpde);

sys/powerpc/aim/mmu_radix.c

4967

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

4972

pmap_resident_count_inc(pmap, 1);

sys/powerpc/aim/mmu_radix.c

5009

reserve_pv_entries(pmap, NPTEPG - 1, lockp);

sys/powerpc/aim/mmu_radix.c

5019

pmap_invalidate_l3e_page(pmap, trunc_2mpage(va), oldpde);

sys/powerpc/aim/mmu_radix.c

5024

pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp);

sys/powerpc/aim/mmu_radix.c

5028

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

5036

pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

5041

KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));

sys/powerpc/aim/mmu_radix.c

5042

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

5043

mpte = pmap_remove_pt_page(pmap, va);

sys/powerpc/aim/mmu_radix.c

5065

pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,

sys/powerpc/aim/mmu_radix.c

5073

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

5078

pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE);

sys/powerpc/aim/mmu_radix.c

5079

pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);

sys/powerpc/aim/mmu_radix.c

5083

pmap_pvh_free(pvh, pmap, sva);

sys/powerpc/aim/mmu_radix.c

5096

if (pmap == kernel_pmap) {

sys/powerpc/aim/mmu_radix.c

5097

pmap_remove_kernel_l3e(pmap, pdq, sva);

sys/powerpc/aim/mmu_radix.c

5099

mpte = pmap_remove_pt_page(pmap, sva);

sys/powerpc/aim/mmu_radix.c

5101

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

5108

return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free));

sys/powerpc/aim/mmu_radix.c

5115

pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,

sys/powerpc/aim/mmu_radix.c

5122

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

5125

pmap->pm_stats.wired_count -= 1;

sys/powerpc/aim/mmu_radix.c

5126

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

5134

pmap_pvh_free(&m->md, pmap, va);

sys/powerpc/aim/mmu_radix.c

5142

return (pmap_unuse_pt(pmap, va, ptepde, free));

sys/powerpc/aim/mmu_radix.c

5149

pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e,

sys/powerpc/aim/mmu_radix.c

5156

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

5166

invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock);

sys/powerpc/aim/mmu_radix.c

5170

pmap_invalidate_page(pmap, va);

sys/powerpc/aim/mmu_radix.c

5178

pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/powerpc/aim/mmu_radix.c

5185

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

5190

MPASS(pte == pmap_pte(pmap, sva));

sys/powerpc/aim/mmu_radix.c

5200

if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) {

sys/powerpc/aim/mmu_radix.c

5207

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

5209

pmap_invalidate_range(pmap, va, sva);

sys/powerpc/aim/mmu_radix.c

5214

mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/powerpc/aim/mmu_radix.c

5224

CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);

sys/powerpc/aim/mmu_radix.c

5229

if (pmap->pm_stats.resident_count == 0)

sys/powerpc/aim/mmu_radix.c

5239

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5247

l3e = pmap_pml3e(pmap, sva);

sys/powerpc/aim/mmu_radix.c

5249

anyvalid = pmap_remove_page(pmap, sva, l3e, &free);

sys/powerpc/aim/mmu_radix.c

5256

if (pmap->pm_stats.resident_count == 0)

sys/powerpc/aim/mmu_radix.c

5258

l1e = pmap_pml1e(pmap, sva);

sys/powerpc/aim/mmu_radix.c

5299

pmap_remove_l3e(pmap, l3e, sva, &free, &lock);

sys/powerpc/aim/mmu_radix.c

5302

} else if (!pmap_demote_l3e_locked(pmap, l3e, sva,

sys/powerpc/aim/mmu_radix.c

5318

if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock))

sys/powerpc/aim/mmu_radix.c

5325

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

5326

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5335

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

5353

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

5354

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

5357

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5361

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5366

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

5367

(void)pmap_demote_l3e_locked(pmap, l3e, va, &lock);

sys/powerpc/aim/mmu_radix.c

5368

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5371

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

5372

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

5376

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5380

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5384

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

5385

l3e = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

5391

pmap->pm_stats.wired_count--;

sys/powerpc/aim/mmu_radix.c

5400

pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free);

sys/powerpc/aim/mmu_radix.c

5401

pmap_invalidate_page(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

5404

free_pv_entry(pmap, pv);

sys/powerpc/aim/mmu_radix.c

5405

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5439

mmu_radix_remove_pages(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

5442

CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);

sys/powerpc/aim/mmu_radix.c

5465

KASSERT(pmap->pm_pid == mfspr(SPR_PID),

sys/powerpc/aim/mmu_radix.c

5466

("non-current asid %lu - expected %lu", pmap->pm_pid,

sys/powerpc/aim/mmu_radix.c

5472

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5473

TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {

sys/powerpc/aim/mmu_radix.c

5487

pte = pmap_pml2e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

5558

pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);

sys/powerpc/aim/mmu_radix.c

5568

mpte = pmap_remove_pt_page(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

5570

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

5577

pmap_resident_count_dec(pmap, 1);

sys/powerpc/aim/mmu_radix.c

5580

pmap, pv);

sys/powerpc/aim/mmu_radix.c

5592

pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free);

sys/powerpc/aim/mmu_radix.c

5602

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/powerpc/aim/mmu_radix.c

5608

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

5609

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5617

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

5637

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

5638

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

5641

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5644

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5649

l3e = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

5651

(void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock);

sys/powerpc/aim/mmu_radix.c

5655

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5658

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

5659

if (!PMAP_TRYLOCK(pmap)) {

sys/powerpc/aim/mmu_radix.c

566

static bool pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,

sys/powerpc/aim/mmu_radix.c

5663

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5667

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5672

l3e = pmap_pml3e(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

568

static bool pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va);

sys/powerpc/aim/mmu_radix.c

5685

pmap_invalidate_page(pmap, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

5687

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

570

static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,

sys/powerpc/aim/mmu_radix.c

5706

mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/powerpc/aim/mmu_radix.c

5714

CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);

sys/powerpc/aim/mmu_radix.c

5715

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5717

l1e = pmap_pml1e(pmap, sva);

sys/powerpc/aim/mmu_radix.c

572

static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,

sys/powerpc/aim/mmu_radix.c

574

static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);

sys/powerpc/aim/mmu_radix.c

5748

pmap->pm_stats.wired_count -= L3_PAGE_SIZE /

sys/powerpc/aim/mmu_radix.c

575

static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde,

sys/powerpc/aim/mmu_radix.c

5751

} else if (!pmap_demote_l3e(pmap, l3e, sva))

sys/powerpc/aim/mmu_radix.c

5758

MPASS(pte == pmap_pte(pmap, sva));

sys/powerpc/aim/mmu_radix.c

577

static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/powerpc/aim/mmu_radix.c

5771

pmap->pm_stats.wired_count--;

sys/powerpc/aim/mmu_radix.c

5774

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5799

mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)

sys/powerpc/aim/mmu_radix.c

580

static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e,

sys/powerpc/aim/mmu_radix.c

5806

CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);

sys/powerpc/aim/mmu_radix.c

5807

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5809

l3ep = pmap_pml3e(pmap, addr);

sys/powerpc/aim/mmu_radix.c

583

static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/powerpc/aim/mmu_radix.c

5840

PMAP_UNLOCK(pmap);

sys/powerpc/aim/mmu_radix.c

5847

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

5852

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/powerpc/aim/mmu_radix.c

5854

if (pmap->pm_pid > isa3_base_pid &&

sys/powerpc/aim/mmu_radix.c

5855

curpid != pmap->pm_pid) {

sys/powerpc/aim/mmu_radix.c

5856

mmu_radix_pid_set(pmap);

sys/powerpc/aim/mmu_radix.c

586

static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);

sys/powerpc/aim/mmu_radix.c

587

static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);

sys/powerpc/aim/mmu_radix.c

588

static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

591

static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

593

static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde,

sys/powerpc/aim/mmu_radix.c

598

static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp);

sys/powerpc/aim/mmu_radix.c

599

static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va,

sys/powerpc/aim/mmu_radix.c

601

static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,

sys/powerpc/aim/mmu_radix.c

6014

pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va)

sys/powerpc/aim/mmu_radix.c

6021

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

6028

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

603

static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/aim/mmu_radix.c

605

static bool pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free);

sys/powerpc/aim/mmu_radix.c

6056

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

6060

" in pmap %p", va, pmap);

sys/powerpc/aim/mmu_radix.c

607

static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start);

sys/powerpc/aim/mmu_radix.c

608

static void pmap_invalidate_all(pmap_t pmap);

sys/powerpc/aim/mmu_radix.c

6505

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

6513

pmap = PV_PMAP(pv);

sys/powerpc/aim/mmu_radix.c

6514

db_printf("pmap %p ", pmap);

sys/powerpc/aim/mmu_radix.c

6515

if (pmap != NULL) {

sys/powerpc/aim/mmu_radix.c

6516

db_printf("asid: %lu\n", pmap->pm_pid);

sys/powerpc/aim/mmu_radix.c

6517

pmap_pte_walk(pmap->pm_pml1, pv->pv_va);

sys/powerpc/aim/mmu_radix.c

6525

pmap_t pmap;

sys/powerpc/aim/mmu_radix.c

6534

pmap = kernel_pmap;

sys/powerpc/aim/mmu_radix.c

6536

pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace);

sys/powerpc/aim/mmu_radix.c

6538

pmap = vmspace_pmap(curthread->td_proc->p_vmspace);

sys/powerpc/aim/mmu_radix.c

6540

pmap_pte_walk(pmap->pm_pml1, va);

sys/powerpc/aim/mmu_radix.c

819

mmu_radix_pid_set(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

822

mtspr(SPR_PID, pmap->pm_pid);

sys/powerpc/aim/mmu_radix.c

867

pmap_resident_count_inc(pmap_t pmap, int count)

sys/powerpc/aim/mmu_radix.c

870

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

871

pmap->pm_stats.resident_count += count;

sys/powerpc/aim/mmu_radix.c

875

pmap_resident_count_dec(pmap_t pmap, int count)

sys/powerpc/aim/mmu_radix.c

878

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/aim/mmu_radix.c

879

KASSERT(pmap->pm_stats.resident_count >= count,

sys/powerpc/aim/mmu_radix.c

880

("pmap %p resident count underflow %ld %d", pmap,

sys/powerpc/aim/mmu_radix.c

881

pmap->pm_stats.resident_count, count));

sys/powerpc/aim/mmu_radix.c

882

pmap->pm_stats.resident_count -= count;

sys/powerpc/aim/mmu_radix.c

927

mmu_radix_ps_enabled(pmap_t pmap)

sys/powerpc/aim/mmu_radix.c

929

return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0);

sys/powerpc/aim/mmu_radix.c

933

pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e)

sys/powerpc/aim/mmu_radix.c

939

l3e = pmap_pml3e(pmap, va);

sys/powerpc/aim/mmu_radix.c

956

pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags)

sys/powerpc/aim/mmu_radix.c

965

if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL)

sys/powerpc/aim/mmu_radix.c

972

pmap_invalidate_all(pmap);

sys/powerpc/aim/mmu_radix.c

976

__func__, pmap, va, flags, origpte);

sys/powerpc/aim/mmu_radix.c

983

printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va,

sys/powerpc/aim/mmu_radix.c

986

PMAP_LOCK(pmap);

sys/powerpc/aim/mmu_radix.c

987

if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL ||

sys/powerpc/aim/mmu_radix.c

989

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1037

mmu_booke_extract(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap.c

1041

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1042

pa = pte_vatopa(pmap, va);

sys/powerpc/booke/pmap.c

1043

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1263

mmu_booke_ps_enabled(pmap_t pmap __unused)

sys/powerpc/booke/pmap.c

1272

mmu_booke_pinit0(pmap_t pmap)

sys/powerpc/booke/pmap.c

1275

PMAP_LOCK_INIT(pmap);

sys/powerpc/booke/pmap.c

1276

mmu_booke_pinit(pmap);

sys/powerpc/booke/pmap.c

1277

PCPU_SET(curpmap, pmap);

sys/powerpc/booke/pmap.c

1286

mmu_booke_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/booke/pmap.c

1292

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1293

error = mmu_booke_enter_locked(pmap, va, m, prot, flags, psind);

sys/powerpc/booke/pmap.c

1294

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1300

mmu_booke_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/booke/pmap.c

1309

su = (pmap == kernel_pmap);

sys/powerpc/booke/pmap.c

1332

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/booke/pmap.c

1338

if (((pte = pte_find(pmap, va)) != NULL) &&

sys/powerpc/booke/pmap.c

1352

pmap->pm_stats.wired_count++;

sys/powerpc/booke/pmap.c

1357

pmap->pm_stats.wired_count--;

sys/powerpc/booke/pmap.c

1449

error = pte_enter(pmap, m, va, flags,

sys/powerpc/booke/pmap.c

1455

pmap->pm_stats.wired_count++;

sys/powerpc/booke/pmap.c

1462

if (sync && (su || pmap == PCPU_GET(curpmap))) {

sys/powerpc/booke/pmap.c

1483

mmu_booke_enter_object(pmap_t pmap, vm_offset_t start,

sys/powerpc/booke/pmap.c

1496

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1499

mmu_booke_enter_locked(pmap, va, m,

sys/powerpc/booke/pmap.c

1504

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1509

mmu_booke_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/powerpc/booke/pmap.c

1514

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1515

mmu_booke_enter_locked(pmap, va, m,

sys/powerpc/booke/pmap.c

1518

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1528

mmu_booke_remove(pmap_t pmap, vm_offset_t va, vm_offset_t endva)

sys/powerpc/booke/pmap.c

1533

int su = (pmap == kernel_pmap);

sys/powerpc/booke/pmap.c

1547

if (PMAP_REMOVE_DONE(pmap)) {

sys/powerpc/booke/pmap.c

1552

hold_flag = PTBL_HOLD_FLAG(pmap);

sys/powerpc/booke/pmap.c

1556

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1558

pte = pte_find_next(pmap, &va);

sys/powerpc/booke/pmap.c

1563

pte_remove(pmap, va, hold_flag);

sys/powerpc/booke/pmap.c

1565

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1624

pmap_t pmap;

sys/powerpc/booke/pmap.c

1627

pmap = &td->td_proc->p_vmspace->vm_pmap;

sys/powerpc/booke/pmap.c

1630

__func__, td, td->td_proc->p_comm, td->td_proc->p_pid, pmap);

sys/powerpc/booke/pmap.c

1632

KASSERT((pmap != kernel_pmap), ("mmu_booke_activate: kernel_pmap!"));

sys/powerpc/booke/pmap.c

1637

CPU_SET_ATOMIC(cpuid, &pmap->pm_active);

sys/powerpc/booke/pmap.c

1638

PCPU_SET(curpmap, pmap);

sys/powerpc/booke/pmap.c

1640

if (pmap->pm_tid[cpuid] == TID_NONE)

sys/powerpc/booke/pmap.c

1641

tid_alloc(pmap);

sys/powerpc/booke/pmap.c

1644

mtspr(SPR_PID0, pmap->pm_tid[cpuid]);

sys/powerpc/booke/pmap.c

1652

pmap->pm_tid[PCPU_GET(cpuid)], td->td_proc->p_comm);

sys/powerpc/booke/pmap.c

1661

pmap_t pmap;

sys/powerpc/booke/pmap.c

1663

pmap = &td->td_proc->p_vmspace->vm_pmap;

sys/powerpc/booke/pmap.c

1666

__func__, td, td->td_proc->p_comm, td->td_proc->p_pid, pmap);

sys/powerpc/booke/pmap.c

1670

CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmap->pm_active);

sys/powerpc/booke/pmap.c

1692

mmu_booke_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/powerpc/booke/pmap.c

1700

mmu_booke_remove(pmap, sva, eva);

sys/powerpc/booke/pmap.c

1707

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1709

if ((pte = pte_find(pmap, va)) != NULL) {

sys/powerpc/booke/pmap.c

1728

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1779

mmu_booke_extract_and_hold(pmap_t pmap, vm_offset_t va,

sys/powerpc/booke/pmap.c

1787

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

1788

pte = pte_find(pmap, va);

sys/powerpc/booke/pmap.c

1790

if (pmap == kernel_pmap)

sys/powerpc/booke/pmap.c

1801

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

1858

mmu_booke_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/powerpc/booke/pmap.c

197

#define PMAP_REMOVE_DONE(pmap) \

sys/powerpc/booke/pmap.c

198

((pmap) != kernel_pmap && (pmap)->pm_stats.resident_count == 0)

sys/powerpc/booke/pmap.c

1995

mmu_booke_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/powerpc/booke/pmap.c

2000

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

2002

if ((pte = pte_find(pmap, va)) != NULL &&

sys/powerpc/booke/pmap.c

2008

pmap->pm_stats.wired_count--;

sys/powerpc/booke/pmap.c

2011

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

2022

mmu_booke_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/powerpc/booke/pmap.c

2034

if (pv->pv_pmap == pmap) {

sys/powerpc/booke/pmap.c

209

static __inline void tid_set_busy(int cpu, int tid, pmap_t pmap);

sys/powerpc/booke/pmap.c

228

static tlbtid_t tid_alloc(struct pmap *);

sys/powerpc/booke/pmap.c

2376

mmu_booke_object_init_pt(pmap_t pmap, vm_offset_t addr,

sys/powerpc/booke/pmap.c

2389

mmu_booke_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/powerpc/booke/pmap.c

2485

tid_set_busy(int cpu, int tid, pmap_t pmap)

sys/powerpc/booke/pmap.c

2489

if (pmap == NULL) {

sys/powerpc/booke/pmap.c

2493

pmap->pm_tid[cpu] = tid;

sys/powerpc/booke/pmap.c

2494

*pm = pmap;

sys/powerpc/booke/pmap.c

2508

tid_alloc(pmap_t pmap)

sys/powerpc/booke/pmap.c

2513

KASSERT((pmap != kernel_pmap), ("tid_alloc: kernel pmap"));

sys/powerpc/booke/pmap.c

2515

CTR2(KTR_PMAP, "%s: s (pmap = %p)", __func__, pmap);

sys/powerpc/booke/pmap.c

2535

tid_set_busy(thiscpu, tid, pmap);

sys/powerpc/booke/pmap.c

2536

pmap->pm_tid[thiscpu] = tid;

sys/powerpc/booke/pmap.c

2986

pmap_track_page(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap.c

2997

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap.c

3000

if ((pmap == pve->pv_pmap) && (va == pve->pv_va)) {

sys/powerpc/booke/pmap.c

3005

pv_insert(pmap, va, page);

sys/powerpc/booke/pmap.c

3007

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap.c

361

static bool mmu_booke_ps_enabled(pmap_t pmap);

sys/powerpc/booke/pmap.c

571

pv_insert(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/powerpc/booke/pmap.c

583

pve->pv_pmap = pmap;

sys/powerpc/booke/pmap.c

587

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/booke/pmap.c

597

pv_remove(pmap_t pmap, vm_offset_t va, vm_page_t m)

sys/powerpc/booke/pmap.c

604

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/booke/pmap.c

609

if ((pmap == pve->pv_pmap) && (va == pve->pv_va)) {

sys/powerpc/booke/pmap_32.c

216

ptbl_free_pmap_ptbl(pmap_t pmap, pte_t *ptbl)

sys/powerpc/booke/pmap_32.c

222

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/powerpc/booke/pmap_32.c

224

TAILQ_FOREACH(pbuf, &pmap->pm_ptbl_list, link)

sys/powerpc/booke/pmap_32.c

227

TAILQ_REMOVE(&pmap->pm_ptbl_list, pbuf, link);

sys/powerpc/booke/pmap_32.c

237

ptbl_alloc(pmap_t pmap, unsigned int pdir_idx, bool nosleep)

sys/powerpc/booke/pmap_32.c

246

CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,

sys/powerpc/booke/pmap_32.c

247

(pmap == kernel_pmap), pdir_idx);

sys/powerpc/booke/pmap_32.c

251

KASSERT((pmap->pm_pdir[pdir_idx] == NULL),

sys/powerpc/booke/pmap_32.c

266

ptbl_free_pmap_ptbl(pmap, ptbl);

sys/powerpc/booke/pmap_32.c

272

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap_32.c

276

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap_32.c

289

TAILQ_INSERT_TAIL(&pmap->pm_ptbl_list, pbuf, link);

sys/powerpc/booke/pmap_32.c

296

ptbl_free(pmap_t pmap, unsigned int pdir_idx)

sys/powerpc/booke/pmap_32.c

304

CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,

sys/powerpc/booke/pmap_32.c

305

(pmap == kernel_pmap), pdir_idx);

sys/powerpc/booke/pmap_32.c

310

ptbl = pmap->pm_pdir[pdir_idx];

sys/powerpc/booke/pmap_32.c

323

pmap->pm_pdir[pdir_idx] = NULL;

sys/powerpc/booke/pmap_32.c

337

ptbl_free_pmap_ptbl(pmap, ptbl);

sys/powerpc/booke/pmap_32.c

347

ptbl_unhold(pmap_t pmap, unsigned int pdir_idx)

sys/powerpc/booke/pmap_32.c

354

CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,

sys/powerpc/booke/pmap_32.c

355

(pmap == kernel_pmap), pdir_idx);

sys/powerpc/booke/pmap_32.c

359

KASSERT((pmap != kernel_pmap),

sys/powerpc/booke/pmap_32.c

362

ptbl = pmap->pm_pdir[pdir_idx];

sys/powerpc/booke/pmap_32.c

382

ptbl_free(pmap, pdir_idx);

sys/powerpc/booke/pmap_32.c

396

ptbl_hold(pmap_t pmap, unsigned int pdir_idx)

sys/powerpc/booke/pmap_32.c

403

CTR3(KTR_PMAP, "%s: pmap = %p pdir_idx = %d", __func__, pmap,

sys/powerpc/booke/pmap_32.c

408

KASSERT((pmap != kernel_pmap),

sys/powerpc/booke/pmap_32.c

411

ptbl = pmap->pm_pdir[pdir_idx];

sys/powerpc/booke/pmap_32.c

429

pte_remove(pmap_t pmap, vm_offset_t va, uint8_t flags)

sys/powerpc/booke/pmap_32.c

441

ptbl = pmap->pm_pdir[pdir_idx];

sys/powerpc/booke/pmap_32.c

450

pmap->pm_stats.wired_count--;

sys/powerpc/booke/pmap_32.c

463

pv_remove(pmap, va, m);

sys/powerpc/booke/pmap_32.c

464

} else if (pmap == kernel_pmap && m && m->md.pv_tracked) {

sys/powerpc/booke/pmap_32.c

470

pv_remove(pmap, va, m);

sys/powerpc/booke/pmap_32.c

484

pmap->pm_stats.resident_count--;

sys/powerpc/booke/pmap_32.c

488

return (ptbl_unhold(pmap, pdir_idx));

sys/powerpc/booke/pmap_32.c

499

pte_enter(pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags,

sys/powerpc/booke/pmap_32.c

507

pmap == kernel_pmap, pmap, va);

sys/powerpc/booke/pmap_32.c

510

ptbl = pmap->pm_pdir[pdir_idx];

sys/powerpc/booke/pmap_32.c

514

ptbl = ptbl_alloc(pmap, pdir_idx, nosleep);

sys/powerpc/booke/pmap_32.c

519

pmap->pm_pdir[pdir_idx] = ptbl;

sys/powerpc/booke/pmap_32.c

526

pte = &pmap->pm_pdir[pdir_idx][ptbl_idx];

sys/powerpc/booke/pmap_32.c

528

pte_remove(pmap, va, PTBL_HOLD);

sys/powerpc/booke/pmap_32.c

534

if (pmap != kernel_pmap)

sys/powerpc/booke/pmap_32.c

535

ptbl_hold(pmap, pdir_idx);

sys/powerpc/booke/pmap_32.c

547

pv_insert(pmap, va, m);

sys/powerpc/booke/pmap_32.c

550

pmap->pm_stats.resident_count++;

sys/powerpc/booke/pmap_32.c

568

pte_vatopa(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap_32.c

573

pte = pte_find(pmap, va);

sys/powerpc/booke/pmap_32.c

581

pte_find(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap_32.c

586

KASSERT((pmap != NULL), ("pte_find: invalid pmap"));

sys/powerpc/booke/pmap_32.c

588

if (pmap->pm_pdir[pdir_idx])

sys/powerpc/booke/pmap_32.c

589

return (&(pmap->pm_pdir[pdir_idx][ptbl_idx]));

sys/powerpc/booke/pmap_32.c

596

pte_find_next(pmap_t pmap, vm_offset_t *pva)

sys/powerpc/booke/pmap_32.c

603

KASSERT((pmap != NULL), ("pte_find: invalid pmap"));

sys/powerpc/booke/pmap_32.c

608

pdir = pmap->pm_pdir;

sys/powerpc/booke/pmap_32.c

688

mmu_booke_pinit(pmap_t pmap)

sys/powerpc/booke/pmap_32.c

692

CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap,

sys/powerpc/booke/pmap_32.c

695

KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap"));

sys/powerpc/booke/pmap_32.c

698

pmap->pm_tid[i] = TID_NONE;

sys/powerpc/booke/pmap_32.c

700

bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));

sys/powerpc/booke/pmap_32.c

701

pmap->pm_pdir = uma_zalloc(ptbl_root_zone, M_WAITOK);

sys/powerpc/booke/pmap_32.c

702

bzero(pmap->pm_pdir, sizeof(pte_t *) * PDIR_NENTRIES);

sys/powerpc/booke/pmap_32.c

703

TAILQ_INIT(&pmap->pm_ptbl_list);

sys/powerpc/booke/pmap_32.c

714

mmu_booke_release(pmap_t pmap)

sys/powerpc/booke/pmap_32.c

717

KASSERT(pmap->pm_stats.resident_count == 0,

sys/powerpc/booke/pmap_32.c

719

pmap->pm_stats.resident_count));

sys/powerpc/booke/pmap_32.c

720

uma_zfree(ptbl_root_zone, pmap->pm_pdir);

sys/powerpc/booke/pmap_32.c

729

pmap_t pmap;

sys/powerpc/booke/pmap_32.c

735

pmap = PCPU_GET(curpmap);

sys/powerpc/booke/pmap_32.c

736

active = (pm == kernel_pmap || pm == pmap) ? 1 : 0;

sys/powerpc/booke/pmap_32.c

760

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap_32.c

761

pte_enter(pmap, m, addr,

sys/powerpc/booke/pmap_32.c

765

pte_remove(pmap, addr, PTBL_UNHOLD);

sys/powerpc/booke/pmap_32.c

766

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap_64.c

134

static pte_t *ptbl_alloc(pmap_t pmap, vm_offset_t va,

sys/powerpc/booke/pmap_64.c

152

mmu_booke_alloc_page(pmap_t pmap, unsigned int idx, bool nosleep)

sys/powerpc/booke/pmap_64.c

162

PMAP_UNLOCK(pmap);

sys/powerpc/booke/pmap_64.c

166

PMAP_LOCK(pmap);

sys/powerpc/booke/pmap_64.c

181

pte_find(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap_64.c

187

KASSERT((pmap != NULL), ("pte_find: invalid pmap"));

sys/powerpc/booke/pmap_64.c

189

pdir_l1 = pmap->pm_root[PG_ROOT_IDX(va)];

sys/powerpc/booke/pmap_64.c

202

pte_find_next(pmap_t pmap, vm_offset_t *pva)

sys/powerpc/booke/pmap_64.c

209

KASSERT((pmap != NULL), ("pte_find: invalid pmap"));

sys/powerpc/booke/pmap_64.c

216

pm_root = pmap->pm_root;

sys/powerpc/booke/pmap_64.c

246

unhold_free_page(pmap_t pmap, vm_page_t m)

sys/powerpc/booke/pmap_64.c

258

get_pgtbl_page(pmap_t pmap, void **ptr_tbl, uint32_t index,

sys/powerpc/booke/pmap_64.c

265

KASSERT(page != 0 || pmap != kernel_pmap,

sys/powerpc/booke/pmap_64.c

268

page = mmu_booke_alloc_page(pmap, index, nosleep);

sys/powerpc/booke/pmap_64.c

291

ptbl_alloc(pmap_t pmap, vm_offset_t va, bool nosleep, bool *is_new)

sys/powerpc/booke/pmap_64.c

299

pdir_l1 = get_pgtbl_page(pmap, (void **)pmap->pm_root,

sys/powerpc/booke/pmap_64.c

303

pdir = get_pgtbl_page(pmap, pdir_l1, pdir_l1_idx,

sys/powerpc/booke/pmap_64.c

307

ptbl = get_pgtbl_page(pmap, pdir, pdir_idx,

sys/powerpc/booke/pmap_64.c

320

ptbl_unhold(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap_64.c

334

KASSERT((pmap != kernel_pmap),

sys/powerpc/booke/pmap_64.c

337

pdir_l1 = pmap->pm_root[pg_root_idx];

sys/powerpc/booke/pmap_64.c

344

if (!unhold_free_page(pmap, m))

sys/powerpc/booke/pmap_64.c

350

if (!unhold_free_page(pmap, m))

sys/powerpc/booke/pmap_64.c

356

if (!unhold_free_page(pmap, m))

sys/powerpc/booke/pmap_64.c

358

pmap->pm_root[pg_root_idx] = NULL;

sys/powerpc/booke/pmap_64.c

368

ptbl_hold(pmap_t pmap, pte_t *ptbl)

sys/powerpc/booke/pmap_64.c

372

KASSERT((pmap != kernel_pmap),

sys/powerpc/booke/pmap_64.c

385

pte_remove(pmap_t pmap, vm_offset_t va, u_int8_t flags)

sys/powerpc/booke/pmap_64.c

390

pte = pte_find(pmap, va);

sys/powerpc/booke/pmap_64.c

392

__func__, (uintmax_t)va, pmap));

sys/powerpc/booke/pmap_64.c

401

pmap->pm_stats.wired_count--;

sys/powerpc/booke/pmap_64.c

414

pv_remove(pmap, va, m);

sys/powerpc/booke/pmap_64.c

415

} else if (pmap == kernel_pmap && m && m->md.pv_tracked) {

sys/powerpc/booke/pmap_64.c

416

pv_remove(pmap, va, m);

sys/powerpc/booke/pmap_64.c

429

pmap->pm_stats.resident_count--;

sys/powerpc/booke/pmap_64.c

432

return (ptbl_unhold(pmap, va));

sys/powerpc/booke/pmap_64.c

441

pte_enter(pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags,

sys/powerpc/booke/pmap_64.c

449

ptbl = ptbl_alloc(pmap, va, nosleep, &is_new);

sys/powerpc/booke/pmap_64.c

463

pte_remove(pmap, va, PTBL_HOLD);

sys/powerpc/booke/pmap_64.c

469

if (pmap != kernel_pmap)

sys/powerpc/booke/pmap_64.c

470

ptbl_hold(pmap, ptbl);

sys/powerpc/booke/pmap_64.c

482

pv_insert(pmap, va, m);

sys/powerpc/booke/pmap_64.c

485

pmap->pm_stats.resident_count++;

sys/powerpc/booke/pmap_64.c

504

pte_vatopa(pmap_t pmap, vm_offset_t va)

sys/powerpc/booke/pmap_64.c

509

pte = pte_find(pmap, va);

sys/powerpc/booke/pmap_64.c

590

mmu_booke_pinit(pmap_t pmap)

sys/powerpc/booke/pmap_64.c

594

CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap,

sys/powerpc/booke/pmap_64.c

597

KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap"));

sys/powerpc/booke/pmap_64.c

600

pmap->pm_tid[i] = TID_NONE;

sys/powerpc/booke/pmap_64.c

602

bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));

sys/powerpc/booke/pmap_64.c

603

pmap->pm_root = uma_zalloc(ptbl_root_zone, M_WAITOK);

sys/powerpc/booke/pmap_64.c

604

bzero(pmap->pm_root, sizeof(pte_t **) * PG_ROOT_NENTRIES);

sys/powerpc/booke/pmap_64.c

615

mmu_booke_release(pmap_t pmap)

sys/powerpc/booke/pmap_64.c

618

KASSERT(pmap->pm_stats.resident_count == 0,

sys/powerpc/booke/pmap_64.c

620

pmap->pm_stats.resident_count));

sys/powerpc/booke/pmap_64.c

627

KASSERT(pmap->pm_root[i] == 0,

sys/powerpc/booke/pmap_64.c

628

("Index %d on root page %p is non-zero!\n", i, pmap->pm_root));

sys/powerpc/booke/pmap_64.c

630

uma_zfree(ptbl_root_zone, pmap->pm_root);

sys/powerpc/include/bus_dma.h

138

bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,

sys/powerpc/include/bus_dma.h

144

return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs,

sys/powerpc/include/bus_dma_impl.h

68

void *buf, bus_size_t buflen, struct pmap *pmap, int flags,

sys/powerpc/include/pcb.h

122

extern struct pmap *curpm;

sys/powerpc/include/pcb.h

53

struct pmap *pcb_pm; /* pmap of our vmspace */

sys/powerpc/include/pcpu.h

37

struct pmap;

sys/powerpc/include/pcpu.h

43

struct pmap *pc_curpmap; /* current pmap */ \

sys/powerpc/include/pmap.h

100

typedef struct pmap *pmap_t;

sys/powerpc/include/pmap.h

174

struct pmap *pmap_phys;

sys/powerpc/include/pmap.h

284

extern struct pmap kernel_pmap_store;

sys/powerpc/include/pmap.h

289

#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)

sys/powerpc/include/pmap.h

290

#define PMAP_LOCK_ASSERT(pmap, type) \

sys/powerpc/include/pmap.h

291

mtx_assert(&(pmap)->pm_mtx, (type))

sys/powerpc/include/pmap.h

292

#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)

sys/powerpc/include/pmap.h

293

#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, \

sys/powerpc/include/pmap.h

294

(pmap == kernel_pmap) ? "kernelpmap" : \

sys/powerpc/include/pmap.h

296

#define PMAP_LOCKED(pmap) mtx_owned(&(pmap)->pm_mtx)

sys/powerpc/include/pmap.h

297

#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)

sys/powerpc/include/pmap.h

298

#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)

sys/powerpc/include/pmap.h

299

#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)

sys/powerpc/include/pmap.h

324

bool pmap_ps_enabled(pmap_t pmap);

sys/powerpc/include/pmap.h

325

int pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags);

sys/powerpc/include/pmap.h

327

#define pmap_map_delete(pmap, sva, eva) pmap_remove(pmap, sva, eva)

sys/powerpc/include/pmap.h

348

void pmap_track_page(pmap_t pmap, vm_offset_t va);

sys/powerpc/include/pmap.h

99

struct pmap;

sys/powerpc/include/pte.h

230

#define PTBL_HOLD_FLAG(pmap) (((pmap) == kernel_pmap) ? PTBL_HOLD : PTBL_UNHOLD)

sys/powerpc/include/slb.h

88

struct pmap;

sys/powerpc/include/slb.h

90

int handle_user_slb_spill(struct pmap *pm, vm_offset_t addr);

sys/powerpc/include/tlb.h

172

struct pmap;

sys/powerpc/powerpc/busdma_bounce.c

494

_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,

sys/powerpc/powerpc/busdma_bounce.c

518

if (pmap == kernel_pmap)

sys/powerpc/powerpc/busdma_bounce.c

521

paddr = pmap_extract(pmap, vaddr);

sys/powerpc/powerpc/busdma_bounce.c

599

pmap_t pmap,

sys/powerpc/powerpc/busdma_bounce.c

613

_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);

sys/powerpc/powerpc/busdma_bounce.c

627

if (pmap == kernel_pmap) {

sys/powerpc/powerpc/busdma_bounce.c

631

curaddr = pmap_extract(pmap, (vm_offset_t)vaddr);

sys/powerpc/powerpc/genassym.c

119

ASSYM(PM_SR, offsetof(struct pmap, pm_sr));

sys/powerpc/powerpc/genassym.c

124

ASSYM(PM_ROOT, offsetof(struct pmap, pm_root));

sys/powerpc/powerpc/genassym.c

126

ASSYM(PM_PDIR, offsetof(struct pmap, pm_pdir));

sys/powerpc/powerpc/pmap_dispatch.c

246

pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)

sys/powerpc/powerpc/pmap_dispatch.c

264

pmap_active_cpus(pmap_t pmap, cpuset_t *res)

sys/powerpc/powerpc/pmap_dispatch.c

266

*res = pmap->pm_active;

sys/powerpc/powerpc/pmap_dispatch.c

70

struct pmap kernel_pmap_store;

sys/powerpc/powerpc/pmap_dispatch.c

83

SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");

sys/riscv/include/bus_dma.h

110

bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,

sys/riscv/include/bus_dma.h

116

return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs,

sys/riscv/include/bus_dma_impl.h

68

void *buf, bus_size_t buflen, struct pmap *pmap, int flags,

sys/riscv/include/pcpu.h

46

struct pmap *pc_curpmap; /* Currently active pmap */ \

sys/riscv/include/pmap.h

100

#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)

sys/riscv/include/pmap.h

101

#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \

sys/riscv/include/pmap.h

103

#define PMAP_OWNED(pmap) mtx_owned(&(pmap)->pm_mtx)

sys/riscv/include/pmap.h

104

#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)

sys/riscv/include/pmap.h

105

#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)

sys/riscv/include/pmap.h

106

#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)

sys/riscv/include/pmap.h

58

#define pmap_map_delete(pmap, sva, eva) pmap_remove(pmap, sva, eva)

sys/riscv/include/pmap.h

83

LIST_ENTRY(pmap) pm_list; /* List of all pmaps */

sys/riscv/include/pmap.h

88

typedef struct pmap *pmap_t;

sys/riscv/include/pmap.h

91

extern struct pmap kernel_pmap_store;

sys/riscv/include/pmap.h

95

#define PMAP_ASSERT_LOCKED(pmap) \

sys/riscv/include/pmap.h

96

mtx_assert(&(pmap)->pm_mtx, MA_OWNED)

sys/riscv/include/pmap.h

97

#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)

sys/riscv/include/pmap.h

98

#define PMAP_LOCK_ASSERT(pmap, type) \

sys/riscv/include/pmap.h

99

mtx_assert(&(pmap)->pm_mtx, (type))

sys/riscv/include/vmm.h

125

struct pmap;

sys/riscv/include/vmm.h

140

DECLARE_VMMOPS_FUNC(void *, init, (struct vm *vm, struct pmap *pmap));

sys/riscv/include/vmm.h

143

DECLARE_VMMOPS_FUNC(int, run, (void *vcpui, register_t pc, struct pmap *pmap,

sys/riscv/iommu/iommu_pmap.c

108

pmap_l1(struct riscv_iommu_pmap *pmap, vm_offset_t va)

sys/riscv/iommu/iommu_pmap.c

114

if (pmap->pm_mode == PMAP_MODE_SV39) {

sys/riscv/iommu/iommu_pmap.c

115

return (&pmap->pm_top[pmap_l1_index(va)]);

sys/riscv/iommu/iommu_pmap.c

117

l0 = pmap_l0(pmap, va);

sys/riscv/iommu/iommu_pmap.c

122

return (pmap_l0_to_l1(pmap, l0, va));

sys/riscv/iommu/iommu_pmap.c

139

pmap_l2(struct riscv_iommu_pmap *pmap, vm_offset_t va)

sys/riscv/iommu/iommu_pmap.c

143

l1 = pmap_l1(pmap, va);

sys/riscv/iommu/iommu_pmap.c

167

pmap_l3(struct riscv_iommu_pmap *pmap, vm_offset_t va)

sys/riscv/iommu/iommu_pmap.c

171

l2 = pmap_l2(pmap, va);

sys/riscv/iommu/iommu_pmap.c

184

pmap_resident_count_inc(struct riscv_iommu_pmap *pmap, int count)

sys/riscv/iommu/iommu_pmap.c

187

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/iommu/iommu_pmap.c

188

pmap->sp_resident_count += count;

sys/riscv/iommu/iommu_pmap.c

192

pmap_resident_count_dec(struct riscv_iommu_pmap *pmap, int count)

sys/riscv/iommu/iommu_pmap.c

195

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/iommu/iommu_pmap.c

196

KASSERT(pmap->sp_resident_count >= count,

sys/riscv/iommu/iommu_pmap.c

197

("pmap %p resident count underflow %ld %d", pmap,

sys/riscv/iommu/iommu_pmap.c

198

pmap->sp_resident_count, count));

sys/riscv/iommu/iommu_pmap.c

199

pmap->sp_resident_count -= count;

sys/riscv/iommu/iommu_pmap.c

203

pmap_resident_count_inc(struct riscv_iommu_pmap *pmap, int count)

sys/riscv/iommu/iommu_pmap.c

208

pmap_resident_count_dec(struct riscv_iommu_pmap *pmap, int count)

sys/riscv/iommu/iommu_pmap.c

218

iommu_pmap_pinit(struct riscv_iommu_pmap *pmap, enum pmap_mode pm_mode)

sys/riscv/iommu/iommu_pmap.c

226

pmap->pm_top = (pd_entry_t *)PHYS_TO_DMAP(topphys);

sys/riscv/iommu/iommu_pmap.c

227

pmap->pm_mode = pm_mode;

sys/riscv/iommu/iommu_pmap.c

231

pmap->pm_satp = SATP_MODE_SV39;

sys/riscv/iommu/iommu_pmap.c

234

pmap->pm_satp = SATP_MODE_SV48;

sys/riscv/iommu/iommu_pmap.c

240

pmap->pm_satp |= (topphys >> PAGE_SHIFT);

sys/riscv/iommu/iommu_pmap.c

243

pmap->sp_resident_count = 0;

sys/riscv/iommu/iommu_pmap.c

246

mtx_init(&pmap->pm_mtx, "iommu pmap", NULL, MTX_DEF);

sys/riscv/iommu/iommu_pmap.c

257

iommu_pmap_release(struct riscv_iommu_pmap *pmap)

sys/riscv/iommu/iommu_pmap.c

261

KASSERT(pmap->sp_resident_count == 0,

sys/riscv/iommu/iommu_pmap.c

263

pmap->sp_resident_count));

sys/riscv/iommu/iommu_pmap.c

265

m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_top));

sys/riscv/iommu/iommu_pmap.c

268

mtx_destroy(&pmap->pm_mtx);

sys/riscv/iommu/iommu_pmap.c

283

_pmap_alloc_l3(struct riscv_iommu_pmap *pmap, vm_pindex_t ptepindex)

sys/riscv/iommu/iommu_pmap.c

290

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/iommu/iommu_pmap.c

314

KASSERT(pmap->pm_mode != PMAP_MODE_SV39,

sys/riscv/iommu/iommu_pmap.c

320

l0 = &pmap->pm_top[l0index];

sys/riscv/iommu/iommu_pmap.c

331

if (pmap->pm_mode == PMAP_MODE_SV39) {

sys/riscv/iommu/iommu_pmap.c

332

l1 = &pmap->pm_top[l1index];

sys/riscv/iommu/iommu_pmap.c

335

l0 = &pmap->pm_top[l0index];

sys/riscv/iommu/iommu_pmap.c

338

if (_pmap_alloc_l3(pmap,

sys/riscv/iommu/iommu_pmap.c

360

if (pmap->pm_mode == PMAP_MODE_SV39) {

sys/riscv/iommu/iommu_pmap.c

361

l1 = &pmap->pm_top[l1index];

sys/riscv/iommu/iommu_pmap.c

364

if (_pmap_alloc_l3(pmap, NUL2E + l1index)

sys/riscv/iommu/iommu_pmap.c

373

l0 = &pmap->pm_top[l0index];

sys/riscv/iommu/iommu_pmap.c

376

if (_pmap_alloc_l3(pmap, NUL2E + l1index)

sys/riscv/iommu/iommu_pmap.c

388

if (_pmap_alloc_l3(pmap,

sys/riscv/iommu/iommu_pmap.c

408

pmap_resident_count_inc(pmap, 1);

sys/riscv/iommu/iommu_pmap.c

422

iommu_pmap_remove(struct riscv_iommu_pmap *pmap, vm_offset_t va)

sys/riscv/iommu/iommu_pmap.c

427

PMAP_LOCK(pmap);

sys/riscv/iommu/iommu_pmap.c

429

l3 = pmap_l3(pmap, va);

sys/riscv/iommu/iommu_pmap.c

431

pmap_resident_count_dec(pmap, 1);

sys/riscv/iommu/iommu_pmap.c

437

PMAP_UNLOCK(pmap);

sys/riscv/iommu/iommu_pmap.c

444

iommu_pmap_enter(struct riscv_iommu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,

sys/riscv/iommu/iommu_pmap.c

480

PMAP_LOCK(pmap);

sys/riscv/iommu/iommu_pmap.c

482

l2 = pmap_l2(pmap, va);

sys/riscv/iommu/iommu_pmap.c

487

mpte = _pmap_alloc_l3(pmap, pmap_l2_pindex(va));

sys/riscv/iommu/iommu_pmap.c

493

l3 = pmap_l3(pmap, va);

sys/riscv/iommu/iommu_pmap.c

500

pmap_resident_count_inc(pmap, 1);

sys/riscv/iommu/iommu_pmap.c

504

PMAP_UNLOCK(pmap);

sys/riscv/iommu/iommu_pmap.c

510

iommu_pmap_remove_pages_sv48(struct riscv_iommu_pmap *pmap)

sys/riscv/iommu/iommu_pmap.c

517

PMAP_LOCK(pmap);

sys/riscv/iommu/iommu_pmap.c

520

l0e = pmap->pm_top[i];

sys/riscv/iommu/iommu_pmap.c

553

pmap_resident_count_dec(pmap, 1);

sys/riscv/iommu/iommu_pmap.c

559

pmap_resident_count_dec(pmap, 1);

sys/riscv/iommu/iommu_pmap.c

564

pmap_resident_count_dec(pmap, 1);

sys/riscv/iommu/iommu_pmap.c

566

pmap_clear(&pmap->pm_top[i]);

sys/riscv/iommu/iommu_pmap.c

569

KASSERT(pmap->sp_resident_count == 0,

sys/riscv/iommu/iommu_pmap.c

570

("Invalid resident count %jd", pmap->sp_resident_count));

sys/riscv/iommu/iommu_pmap.c

572

PMAP_UNLOCK(pmap);

sys/riscv/iommu/iommu_pmap.c

576

iommu_pmap_remove_pages_sv39(struct riscv_iommu_pmap *pmap)

sys/riscv/iommu/iommu_pmap.c

583

PMAP_LOCK(pmap);

sys/riscv/iommu/iommu_pmap.c

586

l1e = pmap->pm_top[j];

sys/riscv/iommu/iommu_pmap.c

611

pmap_resident_count_dec(pmap, 1);

sys/riscv/iommu/iommu_pmap.c

616

pmap_resident_count_dec(pmap, 1);

sys/riscv/iommu/iommu_pmap.c

618

pmap_clear(&pmap->pm_top[j]);

sys/riscv/iommu/iommu_pmap.c

621

KASSERT(pmap->sp_resident_count == 0,

sys/riscv/iommu/iommu_pmap.c

622

("Invalid resident count %jd", pmap->sp_resident_count));

sys/riscv/iommu/iommu_pmap.c

624

PMAP_UNLOCK(pmap);

sys/riscv/iommu/iommu_pmap.c

628

iommu_pmap_remove_pages(struct riscv_iommu_pmap *pmap)

sys/riscv/iommu/iommu_pmap.c

631

switch (pmap->pm_mode) {

sys/riscv/iommu/iommu_pmap.c

633

iommu_pmap_remove_pages_sv39(pmap);

sys/riscv/iommu/iommu_pmap.c

636

iommu_pmap_remove_pages_sv48(pmap);

sys/riscv/iommu/iommu_pmap.c

84

pmap_l0(struct riscv_iommu_pmap *pmap, vm_offset_t va)

sys/riscv/iommu/iommu_pmap.c

86

KASSERT(pmap->pm_mode != PMAP_MODE_SV39,

sys/riscv/iommu/iommu_pmap.c

90

return (&pmap->pm_top[pmap_l0_index(va)]);

sys/riscv/iommu/iommu_pmap.c

94

pmap_l0_to_l1(struct riscv_iommu_pmap *pmap, pd_entry_t *l0, vm_offset_t va)

sys/riscv/iommu/iommu_pmap.c

99

KASSERT(pmap->pm_mode != PMAP_MODE_SV39,

sys/riscv/iommu/iommu_pmap.h

41

int iommu_pmap_enter(struct riscv_iommu_pmap *pmap, vm_offset_t va,

sys/riscv/iommu/iommu_pmap.h

43

int iommu_pmap_remove(struct riscv_iommu_pmap *pmap, vm_offset_t va);

sys/riscv/iommu/iommu_pmap.h

44

void iommu_pmap_remove_pages(struct riscv_iommu_pmap *pmap);

sys/riscv/iommu/iommu_pmap.h

45

int iommu_pmap_pinit(struct riscv_iommu_pmap *pmap, enum pmap_mode pm_mode);

sys/riscv/iommu/iommu_pmap.h

46

void iommu_pmap_release(struct riscv_iommu_pmap *pmap);

sys/riscv/iommu/iommu_pmap.h

47

void iommu_pmap_remove_pages(struct riscv_iommu_pmap *pmap);

sys/riscv/riscv/busdma_bounce.c

107

pmap_t pmap, void *buf, bus_size_t buflen, int flags);

sys/riscv/riscv/busdma_bounce.c

514

_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,

sys/riscv/riscv/busdma_bounce.c

539

if (pmap == kernel_pmap)

sys/riscv/riscv/busdma_bounce.c

542

paddr = pmap_extract(pmap, vaddr);

sys/riscv/riscv/busdma_bounce.c

629

bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,

sys/riscv/riscv/busdma_bounce.c

642

_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);

sys/riscv/riscv/busdma_bounce.c

659

if (pmap == kernel_pmap) {

sys/riscv/riscv/busdma_bounce.c

663

curaddr = pmap_extract(pmap, (vm_offset_t)vaddr);

sys/riscv/riscv/genassym.c

63

ASSYM(PM_SATP, offsetof(struct pmap, pm_satp));

sys/riscv/riscv/pmap.c

1080

pmap_invalidate_page(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

1085

mask = pmap->pm_active;

sys/riscv/riscv/pmap.c

1095

pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/riscv/riscv/pmap.c

1100

mask = pmap->pm_active;

sys/riscv/riscv/pmap.c

1115

pmap_invalidate_all(pmap_t pmap)

sys/riscv/riscv/pmap.c

1120

mask = pmap->pm_active;

sys/riscv/riscv/pmap.c

1140

pmap_invalidate_page(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

1147

pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/riscv/riscv/pmap.c

1158

pmap_invalidate_all(pmap_t pmap)

sys/riscv/riscv/pmap.c

1172

pmap_extract(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

1183

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

1184

l2p = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

1196

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

1208

pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)

sys/riscv/riscv/pmap.c

1215

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

1216

l2p = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

1233

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

1443

pmap_ps_enabled(pmap_t pmap __unused)

sys/riscv/riscv/pmap.c

1486

pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,

sys/riscv/riscv/pmap.c

1490

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

1494

return (vm_radix_insert(&pmap->pm_root, mpte));

sys/riscv/riscv/pmap.c

1504

pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

1507

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

1508

return (vm_radix_remove(&pmap->pm_root, pmap_l2_pindex(va)));

sys/riscv/riscv/pmap.c

1518

pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/riscv/riscv/pmap.c

1525

_pmap_unwire_ptp(pmap, va, m, free);

sys/riscv/riscv/pmap.c

1533

_pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)

sys/riscv/riscv/pmap.c

1536

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

1539

l0 = pmap_l0(pmap, va);

sys/riscv/riscv/pmap.c

1543

l1 = pmap_l1(pmap, va);

sys/riscv/riscv/pmap.c

1545

pmap_distribute_l1(pmap, pmap_l1_index(va), 0);

sys/riscv/riscv/pmap.c

1548

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

1551

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

1556

l1 = pmap_l1(pmap, va);

sys/riscv/riscv/pmap.c

1558

pmap_unwire_ptp(pmap, va, pdpg, free);

sys/riscv/riscv/pmap.c

1563

l0 = pmap_l0(pmap, va);

sys/riscv/riscv/pmap.c

1565

pmap_unwire_ptp(pmap, va, pdpg, free);

sys/riscv/riscv/pmap.c

1567

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

1583

pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde,

sys/riscv/riscv/pmap.c

1592

return (pmap_unwire_ptp(pmap, va, mpte, free));

sys/riscv/riscv/pmap.c

1602

pmap_pinit0(pmap_t pmap)

sys/riscv/riscv/pmap.c

1604

PMAP_LOCK_INIT(pmap);

sys/riscv/riscv/pmap.c

1605

bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));

sys/riscv/riscv/pmap.c

1606

pmap->pm_stage = PM_STAGE1;

sys/riscv/riscv/pmap.c

1607

pmap->pm_top = kernel_pmap->pm_top;

sys/riscv/riscv/pmap.c

1608

pmap->pm_satp = pmap_satp_mode() |

sys/riscv/riscv/pmap.c

1609

(vtophys(pmap->pm_top) >> PAGE_SHIFT);

sys/riscv/riscv/pmap.c

1610

CPU_ZERO(&pmap->pm_active);

sys/riscv/riscv/pmap.c

1611

TAILQ_INIT(&pmap->pm_pvchunk);

sys/riscv/riscv/pmap.c

1612

vm_radix_init(&pmap->pm_root);

sys/riscv/riscv/pmap.c

1613

pmap_activate_boot(pmap);

sys/riscv/riscv/pmap.c

1617

pmap_pinit_stage(pmap_t pmap, enum pmap_stage stage)

sys/riscv/riscv/pmap.c

1635

pmap->pm_top = PHYS_TO_DMAP(topphys);

sys/riscv/riscv/pmap.c

1636

pmap->pm_satp = pmap_satp_mode() | (topphys >> PAGE_SHIFT);

sys/riscv/riscv/pmap.c

1637

pmap->pm_stage = stage;

sys/riscv/riscv/pmap.c

1639

bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));

sys/riscv/riscv/pmap.c

1641

CPU_ZERO(&pmap->pm_active);

sys/riscv/riscv/pmap.c

1653

LIST_INSERT_HEAD(&allpmaps, pmap, pm_list);

sys/riscv/riscv/pmap.c

1656

pmap->pm_top[i] = kernel_pmap->pm_top[i];

sys/riscv/riscv/pmap.c

1659

pmap->pm_top[i] = kernel_pmap->pm_top[i];

sys/riscv/riscv/pmap.c

1663

pmap->pm_top[i] = kernel_pmap->pm_top[i];

sys/riscv/riscv/pmap.c

1667

TAILQ_INIT(&pmap->pm_pvchunk);

sys/riscv/riscv/pmap.c

1668

vm_radix_init(&pmap->pm_root);

sys/riscv/riscv/pmap.c

1674

pmap_pinit(pmap_t pmap)

sys/riscv/riscv/pmap.c

1677

return (pmap_pinit_stage(pmap, PM_STAGE1));

sys/riscv/riscv/pmap.c

1692

_pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)

sys/riscv/riscv/pmap.c

1699

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

1708

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

1712

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

1738

l0 = &pmap->pm_top[l0index];

sys/riscv/riscv/pmap.c

1750

l1 = &pmap->pm_top[l1index];

sys/riscv/riscv/pmap.c

1753

l0 = &pmap->pm_top[l0index];

sys/riscv/riscv/pmap.c

1756

if (_pmap_alloc_l3(pmap,

sys/riscv/riscv/pmap.c

1773

pmap_distribute_l1(pmap, l1index, entry);

sys/riscv/riscv/pmap.c

1780

l1 = &pmap->pm_top[l1index];

sys/riscv/riscv/pmap.c

1783

if (_pmap_alloc_l3(pmap, NUL2E + l1index,

sys/riscv/riscv/pmap.c

1792

l0 = &pmap->pm_top[l0index];

sys/riscv/riscv/pmap.c

1795

if (_pmap_alloc_l3(pmap, NUL2E + l1index,

sys/riscv/riscv/pmap.c

1807

if (_pmap_alloc_l3(pmap,

sys/riscv/riscv/pmap.c

1827

pmap_resident_count_inc(pmap, 1);

sys/riscv/riscv/pmap.c

1838

pmap_alloc_l2(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)

sys/riscv/riscv/pmap.c

1845

l1 = pmap_l1(pmap, va);

sys/riscv/riscv/pmap.c

1856

l2pg = _pmap_alloc_l3(pmap, pindex, lockp);

sys/riscv/riscv/pmap.c

1864

pmap_alloc_l3(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)

sys/riscv/riscv/pmap.c

1878

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

1892

m = _pmap_alloc_l3(pmap, ptepindex, lockp);

sys/riscv/riscv/pmap.c

1909

pmap_release(pmap_t pmap)

sys/riscv/riscv/pmap.c

1915

KASSERT(pmap->pm_stats.resident_count == 0,

sys/riscv/riscv/pmap.c

1917

pmap->pm_stats.resident_count));

sys/riscv/riscv/pmap.c

1918

KASSERT(CPU_EMPTY(&pmap->pm_active),

sys/riscv/riscv/pmap.c

1919

("releasing active pmap %p", pmap));

sys/riscv/riscv/pmap.c

1921

if (pmap->pm_stage == PM_STAGE2)

sys/riscv/riscv/pmap.c

1926

LIST_REMOVE(pmap, pm_list);

sys/riscv/riscv/pmap.c

1931

npages = pmap->pm_stage == PM_STAGE2 ? 4 : 1;

sys/riscv/riscv/pmap.c

1932

m = DMAP_TO_VM_PAGE(pmap->pm_top);

sys/riscv/riscv/pmap.c

2101

free_pv_entry(pmap_t pmap, pv_entry_t pv)

sys/riscv/riscv/pmap.c

2107

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2118

if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {

sys/riscv/riscv/pmap.c

2119

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2120

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2124

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2155

get_pv_entry(pmap_t pmap, struct rwlock **lockp)

sys/riscv/riscv/pmap.c

2163

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2166

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/riscv/riscv/pmap.c

2179

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2180

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,

sys/riscv/riscv/pmap.c

2195

m = reclaim_pv_chunk(pmap, lockp);

sys/riscv/riscv/pmap.c

2203

pc->pc_pmap = pmap;

sys/riscv/riscv/pmap.c

2211

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2224

reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)

sys/riscv/riscv/pmap.c

2233

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2245

TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {

sys/riscv/riscv/pmap.c

2257

m = reclaim_pv_chunk(pmap, lockp);

sys/riscv/riscv/pmap.c

2266

pc->pc_pmap = pmap;

sys/riscv/riscv/pmap.c

227

static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,

sys/riscv/riscv/pmap.c

2270

TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2296

pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

2302

if (pmap == PV_PMAP(pv) && va == pv->pv_va) {

sys/riscv/riscv/pmap.c

231

LIST_HEAD(pmaplist, pmap);

sys/riscv/riscv/pmap.c

2317

pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

2321

pv = pmap_pvh_remove(pvh, pmap, va);

sys/riscv/riscv/pmap.c

2324

free_pv_entry(pmap, pv);

sys/riscv/riscv/pmap.c

2332

pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/riscv/riscv/pmap.c

2338

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2340

if ((pv = get_pv_entry(pmap, NULL)) != NULL) {

sys/riscv/riscv/pmap.c

2356

pmap_pv_demote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/riscv/riscv/pmap.c

2367

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2377

pv = pmap_pvh_remove(pvh, pmap, va);

sys/riscv/riscv/pmap.c

2386

pc = TAILQ_FIRST(&pmap->pm_pvchunk);

sys/riscv/riscv/pmap.c

239

struct pmap kernel_pmap_store;

sys/riscv/riscv/pmap.c

2404

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2405

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2409

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2410

TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

2418

pmap_pv_promote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,

sys/riscv/riscv/pmap.c

2434

pv = pmap_pvh_remove(&m->md, pmap, va);

sys/riscv/riscv/pmap.c

2444

pmap_pvh_free(&m->md, pmap, va);

sys/riscv/riscv/pmap.c

2455

pmap_pv_insert_l2(pmap_t pmap, vm_offset_t va, pd_entry_t l2e, u_int flags,

sys/riscv/riscv/pmap.c

2462

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2464

if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?

sys/riscv/riscv/pmap.c

2477

pmap_remove_kernel_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va)

sys/riscv/riscv/pmap.c

2484

KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));

sys/riscv/riscv/pmap.c

2485

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2487

ml3 = pmap_remove_pt_page(pmap, va);

sys/riscv/riscv/pmap.c

2513

pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva,

sys/riscv/riscv/pmap.c

2521

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2534

pmap_invalidate_range(pmap, sva, sva + L2_SIZE);

sys/riscv/riscv/pmap.c

2536

pmap->pm_stats.wired_count -= L2_SIZE / PAGE_SIZE;

sys/riscv/riscv/pmap.c

2537

pmap_resident_count_dec(pmap, L2_SIZE / PAGE_SIZE);

sys/riscv/riscv/pmap.c

2541

pmap_pvh_free(pvh, pmap, sva);

sys/riscv/riscv/pmap.c

2554

if (pmap == kernel_pmap) {

sys/riscv/riscv/pmap.c

2555

pmap_remove_kernel_l2(pmap, l2, sva);

sys/riscv/riscv/pmap.c

2557

ml3 = pmap_remove_pt_page(pmap, sva);

sys/riscv/riscv/pmap.c

2561

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

2569

return (pmap_unuse_pt(pmap, sva, l1e, free));

sys/riscv/riscv/pmap.c

2576

pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va,

sys/riscv/riscv/pmap.c

2583

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

2585

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

2587

pmap->pm_stats.wired_count -= 1;

sys/riscv/riscv/pmap.c

2588

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

2596

pmap_pvh_free(&m->md, pmap, va);

sys/riscv/riscv/pmap.c

2605

return (pmap_unuse_pt(pmap, va, l2e, free));

sys/riscv/riscv/pmap.c

2615

pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/riscv/riscv/pmap.c

2626

if (pmap->pm_stats.resident_count == 0)

sys/riscv/riscv/pmap.c

2632

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

2636

if (pmap->pm_stats.resident_count == 0)

sys/riscv/riscv/pmap.c

2640

l0 = pmap_l0(pmap, sva);

sys/riscv/riscv/pmap.c

2649

l1 = pmap_l1(pmap, sva);

sys/riscv/riscv/pmap.c

2671

(void)pmap_remove_l2(pmap, l2, sva,

sys/riscv/riscv/pmap.c

2674

} else if (!pmap_demote_l2_locked(pmap, l2, sva,

sys/riscv/riscv/pmap.c

2697

pmap_invalidate_range(pmap, va, sva);

sys/riscv/riscv/pmap.c

2704

if (pmap_remove_l3(pmap, l3, sva, l2e, &free, &lock)) {

sys/riscv/riscv/pmap.c

2710

pmap_invalidate_range(pmap, va, sva);

sys/riscv/riscv/pmap.c

2715

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

2737

pmap_t pmap;

sys/riscv/riscv/pmap.c

2751

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

2752

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

2754

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

2755

(void)pmap_demote_l2(pmap, l2, va);

sys/riscv/riscv/pmap.c

2756

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

2759

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

2760

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

2761

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

2762

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

2771

pmap_invalidate_page(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

2773

pmap->pm_stats.wired_count--;

sys/riscv/riscv/pmap.c

2782

pmap_unuse_pt(pmap, pv->pv_va, pmap_load(l2), &free);

sys/riscv/riscv/pmap.c

2785

free_pv_entry(pmap, pv);

sys/riscv/riscv/pmap.c

2786

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

2798

pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)

sys/riscv/riscv/pmap.c

2807

pmap_remove(pmap, sva, eva);

sys/riscv/riscv/pmap.c

2823

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

2826

l0 = pmap_l0(pmap, sva);

sys/riscv/riscv/pmap.c

2835

l1 = pmap_l1(pmap, sva);

sys/riscv/riscv/pmap.c

2872

pmap);

sys/riscv/riscv/pmap.c

2873

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

2878

if (!pmap_demote_l2(pmap, l2, sva)) {

sys/riscv/riscv/pmap.c

2908

pmap_invalidate_all(pmap);

sys/riscv/riscv/pmap.c

2911

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

2915

pmap_fault_lookup(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

2919

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

2928

pmap_fault(pmap_t pmap, vm_offset_t va, vm_prot_t ftype)

sys/riscv/riscv/pmap.c

2935

if (pmap == kernel_pmap) {

sys/riscv/riscv/pmap.c

2944

pte = pmap_fault_lookup(pmap, va);

sys/riscv/riscv/pmap.c

2956

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

2957

pte = pmap_fault_lookup(pmap, va);

sys/riscv/riscv/pmap.c

2961

if ((pmap != kernel_pmap && (oldpte & PTE_U) == 0) ||

sys/riscv/riscv/pmap.c

2981

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

2990

pmap_demote_l1(pmap_t pmap, pd_entry_t *l1, vm_offset_t va)

sys/riscv/riscv/pmap.c

2997

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

3015

va, pmap);

sys/riscv/riscv/pmap.c

3042

va, pmap);

sys/riscv/riscv/pmap.c

3047

pmap_demote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va)

sys/riscv/riscv/pmap.c

3053

rv = pmap_demote_l2_locked(pmap, l2, va, &lock);

sys/riscv/riscv/pmap.c

3064

pmap_demote_l2_locked(pmap_t pmap, pd_entry_t *l2, vm_offset_t va,

sys/riscv/riscv/pmap.c

3074

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

3079

if ((oldl2 & PTE_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==

sys/riscv/riscv/pmap.c

3087

(void)pmap_remove_l2(pmap, l2, va & ~L2_OFFSET,

sys/riscv/riscv/pmap.c

3088

pmap_load(pmap_l1(pmap, va)), &free, lockp);

sys/riscv/riscv/pmap.c

3091

"failure for va %#lx in pmap %p", va, pmap);

sys/riscv/riscv/pmap.c

3097

pmap_resident_count_inc(pmap, 1);

sys/riscv/riscv/pmap.c

3137

reserve_pv_entries(pmap, Ln_ENTRIES - 1, lockp);

sys/riscv/riscv/pmap.c

3148

pmap_pv_demote_l2(pmap, va, PTE_TO_PHYS(oldl2), lockp);

sys/riscv/riscv/pmap.c

3152

va, pmap);

sys/riscv/riscv/pmap.c

3158

pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, vm_page_t ml3,

sys/riscv/riscv/pmap.c

3164

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

3165

if (!pmap_ps_enabled(pmap))

sys/riscv/riscv/pmap.c

3181

va, pmap);

sys/riscv/riscv/pmap.c

319

static void free_pv_entry(pmap_t pmap, pv_entry_t pv);

sys/riscv/riscv/pmap.c

320

static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp);

sys/riscv/riscv/pmap.c

3215

va, pmap);

sys/riscv/riscv/pmap.c

322

static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);

sys/riscv/riscv/pmap.c

3228

va, pmap);

sys/riscv/riscv/pmap.c

323

static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,

sys/riscv/riscv/pmap.c

325

static bool pmap_demote_l1(pmap_t pmap, pd_entry_t *l1, vm_offset_t va);

sys/riscv/riscv/pmap.c

3253

if (pmap_insert_pt_page(pmap, ml3, true, all_l3e_PTE_A != 0)) {

sys/riscv/riscv/pmap.c

3255

va, pmap);

sys/riscv/riscv/pmap.c

326

static bool pmap_demote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va);

sys/riscv/riscv/pmap.c

3261

pmap_pv_promote_l2(pmap, va, PTE_TO_PHYS(firstl3e), lockp);

sys/riscv/riscv/pmap.c

3267

pmap);

sys/riscv/riscv/pmap.c

327

static bool pmap_demote_l2_locked(pmap_t pmap, pd_entry_t *l2,

sys/riscv/riscv/pmap.c

3285

pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/riscv/riscv/pmap.c

329

static int pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2,

sys/riscv/riscv/pmap.c

331

static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,

sys/riscv/riscv/pmap.c

333

static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva,

sys/riscv/riscv/pmap.c

3336

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

3342

rv = pmap_enter_l2(pmap, va, new_l3, flags, m, &lock);

sys/riscv/riscv/pmap.c

3346

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

3348

((l2e & PTE_RWX) == 0 || pmap_demote_l2_locked(pmap, l2,

sys/riscv/riscv/pmap.c

335

static bool pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,

sys/riscv/riscv/pmap.c

3357

mpte = pmap_alloc_l3(pmap, va, nosleep ? NULL : &lock);

sys/riscv/riscv/pmap.c

3363

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

3366

l3 = pmap_l3(pmap, va);

sys/riscv/riscv/pmap.c

338

static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex,

sys/riscv/riscv/pmap.c

3387

pmap->pm_stats.wired_count++;

sys/riscv/riscv/pmap.c

3390

pmap->pm_stats.wired_count--;

sys/riscv/riscv/pmap.c

341

static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/riscv/riscv/pmap.c

3441

pv = pmap_pvh_remove(&om->md, pmap, va);

sys/riscv/riscv/pmap.c

3445

free_pv_entry(pmap, pv);

sys/riscv/riscv/pmap.c

3452

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

3459

pmap->pm_stats.wired_count++;

sys/riscv/riscv/pmap.c

3460

pmap_resident_count_inc(pmap, 1);

sys/riscv/riscv/pmap.c

3467

pv = get_pv_entry(pmap, &lock);

sys/riscv/riscv/pmap.c

3483

pmap_sync_icache(pmap, va, PAGE_SIZE);

sys/riscv/riscv/pmap.c

3490

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

3504

(void)pmap_promote_l2(pmap, l2, va, mpte, &lock);

sys/riscv/riscv/pmap.c

3512

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

3521

pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t l2pg)

sys/riscv/riscv/pmap.c

3526

if (pmap_unwire_ptp(pmap, va, l2pg, &free)) {

sys/riscv/riscv/pmap.c

3533

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

3545

pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,

sys/riscv/riscv/pmap.c

3551

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

3562

return (pmap_enter_l2(pmap, va, new_l2, PMAP_ENTER_NOSLEEP |

sys/riscv/riscv/pmap.c

3599

pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags,

sys/riscv/riscv/pmap.c

3608

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

3610

if ((l2pg = pmap_alloc_l2(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?

sys/riscv/riscv/pmap.c

3613

" for va %#lx in pmap %p", va, pmap);

sys/riscv/riscv/pmap.c

3627

" in pmap %p", va, pmap);

sys/riscv/riscv/pmap.c

3634

" for va %#lx in pmap %p", va, pmap);

sys/riscv/riscv/pmap.c

3640

(void)pmap_remove_l2(pmap, l2, va,

sys/riscv/riscv/pmap.c

3641

pmap_load(pmap_l1(pmap, va)), &free, lockp);

sys/riscv/riscv/pmap.c

3646

pmap_remove_l3(pmap, l3, sva, oldl2, &free,

sys/riscv/riscv/pmap.c

3657

if (pmap_insert_pt_page(pmap, mt, false, false))

sys/riscv/riscv/pmap.c

3668

if ((new_l2 & PTE_SW_WIRED) != 0 && pmap != kernel_pmap) {

sys/riscv/riscv/pmap.c

3671

pmap_abort_ptp(pmap, va, l2pg);

sys/riscv/riscv/pmap.c

3675

if (pmap_insert_pt_page(pmap, uwptpg, true, false)) {

sys/riscv/riscv/pmap.c

3678

pmap_abort_ptp(pmap, va, l2pg);

sys/riscv/riscv/pmap.c

3681

pmap_resident_count_inc(pmap, 1);

sys/riscv/riscv/pmap.c

3688

if (!pmap_pv_insert_l2(pmap, va, new_l2, flags, lockp)) {

sys/riscv/riscv/pmap.c

3689

pmap_abort_ptp(pmap, va, l2pg);

sys/riscv/riscv/pmap.c

3691

mt = pmap_remove_pt_page(pmap, va);

sys/riscv/riscv/pmap.c

3695

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

3702

" for va %#lx in pmap %p", va, pmap);

sys/riscv/riscv/pmap.c

3714

pmap->pm_stats.wired_count += L2_SIZE / PAGE_SIZE;

sys/riscv/riscv/pmap.c

3715

pmap->pm_stats.resident_count += L2_SIZE / PAGE_SIZE;

sys/riscv/riscv/pmap.c

3724

va, pmap);

sys/riscv/riscv/pmap.c

3742

pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,

sys/riscv/riscv/pmap.c

3759

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

3763

m->psind == 1 && pmap_ps_enabled(pmap) &&

sys/riscv/riscv/pmap.c

3764

((rv = pmap_enter_2mpage(pmap, va, m, prot, &lock)) ==

sys/riscv/riscv/pmap.c

3768

mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte,

sys/riscv/riscv/pmap.c

3776

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

3789

pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)

sys/riscv/riscv/pmap.c

3795

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

3796

(void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock);

sys/riscv/riscv/pmap.c

3800

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

3804

pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/riscv/riscv/pmap.c

3815

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

3818

CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va);

sys/riscv/riscv/pmap.c

3836

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

3854

mpte = _pmap_alloc_l3(pmap, l2pindex, NULL);

sys/riscv/riscv/pmap.c

3877

!pmap_try_insert_pv_entry(pmap, va, m, lockp)) {

sys/riscv/riscv/pmap.c

3880

if (pmap_unwire_ptp(pmap, va, mpte, &free))

sys/riscv/riscv/pmap.c

3889

pmap_resident_count_inc(pmap, 1);

sys/riscv/riscv/pmap.c

3905

pmap_sync_icache(pmap, va, PAGE_SIZE);

sys/riscv/riscv/pmap.c

3919

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

3925

if (pmap_promote_l2(pmap, l2, va, mpte, lockp))

sys/riscv/riscv/pmap.c

3939

pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,

sys/riscv/riscv/pmap.c

3958

pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

sys/riscv/riscv/pmap.c

3967

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

3970

l0 = pmap_l0(pmap, sva);

sys/riscv/riscv/pmap.c

3979

l1 = pmap_l1(pmap, sva);

sys/riscv/riscv/pmap.c

4007

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4013

if (!pmap_demote_l2(pmap, l2, sva))

sys/riscv/riscv/pmap.c

4034

pmap->pm_stats.wired_count--;

sys/riscv/riscv/pmap.c

4039

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

411

pmap_l0(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

4157

pmap_page_exists_quick(pmap_t pmap, vm_page_t m)

sys/riscv/riscv/pmap.c

416

return (&pmap->pm_top[pmap_l0_index(va)]);

sys/riscv/riscv/pmap.c

4172

if (PV_PMAP(pv) == pmap) {

sys/riscv/riscv/pmap.c

4183

if (PV_PMAP(pv) == pmap) {

sys/riscv/riscv/pmap.c

4208

pmap_t pmap;

sys/riscv/riscv/pmap.c

4222

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4223

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4226

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4229

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4233

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4239

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4244

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4245

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4249

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4253

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4257

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4260

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4290

pmap_remove_pages_pv(pmap_t pmap, vm_page_t m, pv_entry_t pv,

sys/riscv/riscv/pmap.c

4297

pmap_resident_count_dec(pmap, Ln_ENTRIES);

sys/riscv/riscv/pmap.c

4307

mpte = pmap_remove_pt_page(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4311

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

4318

pmap_resident_count_dec(pmap, 1);

sys/riscv/riscv/pmap.c

433

pmap_l1(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

4347

pmap_remove_pages(pmap_t pmap)

sys/riscv/riscv/pmap.c

4365

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4366

TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {

sys/riscv/riscv/pmap.c

4378

pte = pmap_l1(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

440

return (&pmap->pm_top[pmap_l1_index(va)]);

sys/riscv/riscv/pmap.c

442

l0 = pmap_l0(pmap, va);

sys/riscv/riscv/pmap.c

4430

pmap_remove_pages_pv(pmap, m, pv, &free,

sys/riscv/riscv/pmap.c

4432

pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);

sys/riscv/riscv/pmap.c

4440

TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);

sys/riscv/riscv/pmap.c

4446

pmap_invalidate_all(pmap);

sys/riscv/riscv/pmap.c

4448

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4460

pmap_t pmap;

sys/riscv/riscv/pmap.c

4476

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4477

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4480

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4483

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4487

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4492

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4499

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4500

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4504

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4508

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4512

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4514

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4553

pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)

sys/riscv/riscv/pmap.c

4563

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4564

l3 = pmap_l3(pmap, addr);

sys/riscv/riscv/pmap.c

4568

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4595

pmap_t pmap;

sys/riscv/riscv/pmap.c

4615

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4616

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4619

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4622

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4628

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

4630

(void)pmap_demote_l2_locked(pmap, l2, va, &lock);

sys/riscv/riscv/pmap.c

4634

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4637

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4638

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

464

pmap_l2(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

4642

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4645

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4650

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4662

pmap_invalidate_page(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4664

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

468

l1 = pmap_l1(pmap, va);

sys/riscv/riscv/pmap.c

4694

pmap_t pmap;

sys/riscv/riscv/pmap.c

4717

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4718

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4721

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4724

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4729

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

4759

(uintptr_t)pmap) & (Ln_ENTRIES - 1)) == 0 &&

sys/riscv/riscv/pmap.c

4762

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

4767

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4782

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4783

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4787

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4790

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4794

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4812

pmap_invalidate_page(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4817

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4839

pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)

sys/riscv/riscv/pmap.c

4851

pmap_t pmap;

sys/riscv/riscv/pmap.c

4872

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4873

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4876

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4879

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4884

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

4888

pmap_demote_l2_locked(pmap, l2, va, &lock) &&

sys/riscv/riscv/pmap.c

4898

pmap_invalidate_page(pmap, va);

sys/riscv/riscv/pmap.c

4900

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4903

pmap = PV_PMAP(pv);

sys/riscv/riscv/pmap.c

4904

if (!PMAP_TRYLOCK(pmap)) {

sys/riscv/riscv/pmap.c

4908

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

4911

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

4915

l2 = pmap_l2(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

492

pmap_l3(pmap_t pmap, vm_offset_t va)

sys/riscv/riscv/pmap.c

4921

pmap_invalidate_page(pmap, pv->pv_va);

sys/riscv/riscv/pmap.c

4923

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

496

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

508

pmap_resident_count_inc(pmap_t pmap, int count)

sys/riscv/riscv/pmap.c

511

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

512

pmap->pm_stats.resident_count += count;

sys/riscv/riscv/pmap.c

516

pmap_resident_count_dec(pmap_t pmap, int count)

sys/riscv/riscv/pmap.c

5165

pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)

sys/riscv/riscv/pmap.c

5172

PMAP_LOCK(pmap);

sys/riscv/riscv/pmap.c

5173

l2 = pmap_l2(pmap, addr);

sys/riscv/riscv/pmap.c

5182

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

519

PMAP_LOCK_ASSERT(pmap, MA_OWNED);

sys/riscv/riscv/pmap.c

520

KASSERT(pmap->pm_stats.resident_count >= count,

sys/riscv/riscv/pmap.c

5202

PMAP_UNLOCK(pmap);

sys/riscv/riscv/pmap.c

5209

pmap_t oldpmap, pmap;

sys/riscv/riscv/pmap.c

521

("pmap %p resident count underflow %ld %d", pmap,

sys/riscv/riscv/pmap.c

5213

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/riscv/riscv/pmap.c

5214

if (pmap == oldpmap)

sys/riscv/riscv/pmap.c

5216

csr_write(satp, pmap->pm_satp);

sys/riscv/riscv/pmap.c

522

pmap->pm_stats.resident_count, count));

sys/riscv/riscv/pmap.c

5220

CPU_SET_ATOMIC(hart, &pmap->pm_active);

sys/riscv/riscv/pmap.c

5223

CPU_SET(hart, &pmap->pm_active);

sys/riscv/riscv/pmap.c

5226

PCPU_SET(curpmap, pmap);

sys/riscv/riscv/pmap.c

523

pmap->pm_stats.resident_count -= count;

sys/riscv/riscv/pmap.c

5241

pmap_activate_boot(pmap_t pmap)

sys/riscv/riscv/pmap.c

5247

CPU_SET_ATOMIC(hart, &pmap->pm_active);

sys/riscv/riscv/pmap.c

5249

CPU_SET(hart, &pmap->pm_active);

sys/riscv/riscv/pmap.c

5251

PCPU_SET(curpmap, pmap);

sys/riscv/riscv/pmap.c

5255

pmap_active_cpus(pmap_t pmap, cpuset_t *res)

sys/riscv/riscv/pmap.c

5257

*res = pmap->pm_active;

sys/riscv/riscv/pmap.c

5261

pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz)

sys/riscv/riscv/pmap.c

527

pmap_distribute_l1(struct pmap *pmap, vm_pindex_t l1index,

sys/riscv/riscv/pmap.c

530

struct pmap *user_pmap;

sys/riscv/riscv/pmap.c

539

if (pmap != kernel_pmap || pmap_mode != PMAP_MODE_SV39)

sys/riscv/riscv/pmap.c

5392

pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)

sys/riscv/riscv/pmap.c

5399

pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t **l1, pd_entry_t **l2,

sys/riscv/riscv/pmap.c

5405

l1p = pmap_l1(pmap, va);

sys/riscv/riscv/pmap.c

5635

pmap_t pmap;

sys/riscv/riscv/pmap.c

5649

pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace);

sys/riscv/riscv/pmap.c

5651

pmap = PCPU_GET(curpmap);

sys/riscv/riscv/pmap.c

5655

l0 = pmap_l0(pmap, va);

sys/riscv/riscv/pmap.c

5659

l1 = pmap_l1(pmap, va);

sys/riscv/riscv/pmap.c

5667

l2 = pmap_l2(pmap, va);

sys/riscv/riscv/pmap.c

5675

l3 = pmap_l3(pmap, va);

sys/riscv/riscv/trap.c

266

if (VIRT_IS_VALID(va) && pmap_fault(map->pmap, va, ftype))

sys/riscv/vmm/vmm.c

582

pmap_t pmap;

sys/riscv/vmm/vmm.c

588

pmap = vmspace_pmap(vm_vmspace(vm));

sys/riscv/vmm/vmm.c

608

if (pmap_fault(pmap, addr, ftype))

sys/riscv/vmm/vmm.c

676

pmap_t pmap;

sys/riscv/vmm/vmm.c

693

pmap = vmspace_pmap(vm_vmspace(vm));

sys/riscv/vmm/vmm.c

704

error = vmmops_run(vcpu->cookie, vcpu->nextpc, pmap, &evinfo);

sys/riscv/vmm/vmm_riscv.c

124

vmmops_init(struct vm *vm, pmap_t pmap)

sys/riscv/vmm/vmm_riscv.c

241

riscv_vmm_pinit(pmap_t pmap)

sys/riscv/vmm/vmm_riscv.c

244

dprintf("%s: pmap %p\n", __func__, pmap);

sys/riscv/vmm/vmm_riscv.c

246

pmap_pinit_stage(pmap, PM_STAGE2);

sys/riscv/vmm/vmm_riscv.c

449

pmap_t pmap)

sys/riscv/vmm/vmm_riscv.c

603

vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)

sys/riscv/vmm/vmm_riscv.c

630

csr_write(hgatp, pmap->pm_satp);

sys/riscv/vmm/vmm_riscv.c

711

handled = riscv_handle_world_switch(hypctx, vme, pmap);

sys/security/mac/mac_process.c

364

pmap_protect(map->pmap, vme->start, vme->end,

sys/sys/_pv_entry.h

41

struct pmap;

sys/sys/_pv_entry.h

92

struct pmap *pc_pmap; \

sys/sys/bus_dma.h

117

struct pmap;

sys/sys/bus_dma_internal.h

43

void *buf, bus_size_t buflen, struct pmap *pmap,

sys/vm/pmap.h

121

void pmap_active_cpus(pmap_t pmap, cpuset_t *res);

sys/vm/pmap.h

122

void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,

sys/vm/pmap.h

131

int pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/vm/pmap.h

133

void pmap_enter_object(pmap_t pmap, vm_offset_t start,

sys/vm/pmap.h

135

void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,

sys/vm/pmap.h

137

vm_paddr_t pmap_extract(pmap_t pmap, vm_offset_t va);

sys/vm/pmap.h

138

vm_page_t pmap_extract_and_hold(pmap_t pmap, vm_offset_t va,

sys/vm/pmap.h

143

bool pmap_is_prefaultable(pmap_t pmap, vm_offset_t va);

sys/vm/pmap.h

147

int pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap);

sys/vm/pmap.h

148

void pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,

sys/vm/pmap.h

150

bool pmap_page_exists_quick(pmap_t pmap, vm_page_t m);

sys/vm/pmap.h

167

void pmap_unwire(pmap_t pmap, vm_offset_t start, vm_offset_t end);

sys/vm/vm_extern.h

114

typedef int (*pmap_pinit_t)(struct pmap *pmap);

sys/vm/vm_extern.h

35

struct pmap;

sys/vm/vm_fault.c

1933

pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot,

sys/vm/vm_fault.c

2027

pmap_advise(fs->map->pmap, start, end, MADV_DONTNEED);

sys/vm/vm_fault.c

2069

pmap_t pmap;

sys/vm/vm_fault.c

2078

pmap = fs->map->pmap;

sys/vm/vm_fault.c

2079

if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace))

sys/vm/vm_fault.c

2117

if (!pmap_is_prefaultable(pmap, addr))

sys/vm/vm_fault.c

2142

pmap_enter_quick(pmap, addr, m, prot);

sys/vm/vm_fault.c

2194

*mp = pmap_extract_and_hold(map->pmap, va, prot);

sys/vm/vm_fault.c

2469

pmap_enter(dst_map->pmap, vaddr, dst_m, prot,

sys/vm/vm_fault.c

421

!pmap_ps_enabled(fs->map->pmap)) {

sys/vm/vm_fault.c

440

if (pmap_enter(fs->map->pmap, vaddr, m_map, fs->prot, fs->fault_type |

sys/vm/vm_fault.c

606

rv = pmap_enter(fs->map->pmap, vaddr, m, fs->prot,

sys/vm/vm_fault.c

655

!pmap_ps_enabled(fs->map->pmap)))

sys/vm/vm_fault.c

677

rv = pmap_enter(fs->map->pmap, vaddr, m,

sys/vm/vm_fault.c

694

rv = pmap_enter(fs->map->pmap, vaddr + ptoa(i),

sys/vm/vm_glue.c

273

pmap_sync_icache(map->pmap, va, sz);

sys/vm/vm_kern.c

1021

pmap_active_cpus(pmap_t pmap, cpuset_t *res)

sys/vm/vm_kern.c

1037

if (pmap == vmspace_pmap(vm))

sys/vm/vm_map.c

129

static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min,

sys/vm/vm_map.c

2715

pmap_object_init_pt(map->pmap, addr, object, pindex,

sys/vm/vm_map.c

2773

pmap_enter_object(map->pmap, start, addr +

sys/vm/vm_map.c

2779

pmap_enter_object(map->pmap, start, addr + ptoa(psize),

sys/vm/vm_map.c

3024

pmap_protect(map->pmap, entry->start,

sys/vm/vm_map.c

3206

pmap_advise(map->pmap, useStart, useEnd,

sys/vm/vm_map.c

326

KASSERT(vm->vm_map.pmap == NULL, ("vm_map.pmap must be NULL"));

sys/vm/vm_map.c

3530

pmap_unwire(map->pmap, entry->start, failed_addr);

sys/vm/vm_map.c

381

vm->vm_map.pmap = NULL;

sys/vm/vm_map.c

3861

pmap_remove(map->pmap, start, end);

sys/vm/vm_map.c

3925

pmap_unwire(map->pmap, entry->start, entry->end);

sys/vm/vm_map.c

4091

pmap_map_delete(map->pmap, entry->start, entry->end);

sys/vm/vm_map.c

4280

pmap_protect(src_map->pmap,

sys/vm/vm_map.c

4328

pmap_copy(dst_map->pmap, src_map->pmap,

sys/vm/vm_map.c

4424

error = pmap_vmspace_copy(new_map->pmap, old_map->pmap);

sys/vm/vm_map.c

4540

pmap_copy(new_map->pmap, old_map->pmap,

sys/vm/vm_map.c

4843

if (ptoa(pmap_wired_count(map->pmap)) + grow_amount > lmemlim) {

sys/vm/vm_map.c

4851

ptoa(pmap_wired_count(map->pmap)) + grow_amount)) {

sys/vm/vm_map.c

4940

ptoa(pmap_wired_count(map->pmap)));

sys/vm/vm_map.c

5325

return (map->pmap);

sys/vm/vm_map.c

5414

(void *)map->pmap, map->nentries, map->timestamp);

sys/vm/vm_map.c

886

_vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max)

sys/vm/vm_map.c

890

map->pmap = pmap;

sys/vm/vm_map.c

905

vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max)

sys/vm/vm_map.c

907

_vm_map_init(map, pmap, min, max);

sys/vm/vm_map.c

912

vm_map_init_system(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max)

sys/vm/vm_map.c

914

_vm_map_init(map, pmap, min, max);

sys/vm/vm_map.h

214

pmap_t pmap; /* (c) Physical map */

sys/vm/vm_map.h

264

return (map->pmap);

sys/vm/vm_map.h

316

struct pmap vm_pmap; /* private physical map */

sys/vm/vm_mmap.c

1104

nsize = ptoa(npages + pmap_wired_count(map->pmap));

sys/vm/vm_mmap.c

1125

ptoa(pmap_wired_count(map->pmap)));

sys/vm/vm_mmap.c

1204

ptoa(pmap_wired_count(map->pmap)));

sys/vm/vm_mmap.c

1286

ptoa(pmap_wired_count(map->pmap)));

sys/vm/vm_mmap.c

1546

if (ptoa(pmap_wired_count(map->pmap)) + size >

sys/vm/vm_mmap.c

1553

ptoa(pmap_wired_count(map->pmap)) + size);

sys/vm/vm_mmap.c

833

pmap_t pmap;

sys/vm/vm_mmap.c

854

pmap = vmspace_pmap(td->td_proc->p_vmspace);

sys/vm/vm_mmap.c

910

mincoreinfo = pmap_mincore(pmap, addr, &pa);

sys/vm/vm_mmap.c

936

if (pa != pmap_extract(pmap, addr))

sys/vm/vm_swapout.c

139

vm_swapout_object_deactivate_page(pmap_t pmap, vm_page_t m, bool unmap)

sys/vm/vm_swapout.c

149

if (vm_page_wired(m) || !pmap_page_exists_quick(pmap, m)) {

sys/vm/vm_swapout.c

171

vm_swapout_object_deactivate(pmap_t pmap, vm_object_t first_object,

sys/vm/vm_swapout.c

183

if (pmap_resident_count(pmap) <= desired)

sys/vm/vm_swapout.c

199

if (pmap_resident_count(pmap) <= desired)

sys/vm/vm_swapout.c

203

vm_swapout_object_deactivate_page(pmap, m, unmap);

sys/vm/vm_swapout.c

253

vm_swapout_object_deactivate(map->pmap, bigobj, desired);

sys/vm/vm_swapout.c

267

vm_swapout_object_deactivate(map->pmap, obj,

sys/vm/vm_unix.c

130

if (ptoa(pmap_wired_count(map->pmap)) +

sys/vm/vm_unix.c

160

ptoa(pmap_wired_count(map->pmap)) +

sys/vm/vm_unix.c

199

ptoa(pmap_wired_count(map->pmap)));

sys/vm/vm_unix.c

222

ptoa(pmap_wired_count(map->pmap)));

sys/x86/include/bus_dma.h

160

bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,

sys/x86/include/bus_dma.h

166

return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs,

sys/x86/include/busdma_impl.h

70

void *buf, bus_size_t buflen, struct pmap *pmap, int flags,

sys/x86/include/x86_smp.h

114

void smp_masked_invlpg(cpuset_t mask, vm_offset_t addr, struct pmap *pmap,

sys/x86/include/x86_smp.h

117

vm_offset_t endva, struct pmap *pmap, smp_invl_cb_t curcpu_cb);

sys/x86/include/x86_smp.h

118

void smp_masked_invltlb(cpuset_t mask, struct pmap *pmap,

sys/x86/include/x86_smp.h

121

void smp_masked_invlpg(vm_offset_t addr, struct pmap *pmap,

sys/x86/include/x86_smp.h

124

struct pmap *pmap, smp_invl_cb_t curcpu_cb);

sys/x86/include/x86_smp.h

125

void smp_masked_invltlb(struct pmap *pmap, smp_invl_cb_t curcpu_cb);

sys/x86/include/x86_smp.h

22

struct pmap;

sys/x86/include/x86_smp.h

41

extern struct pmap *smp_tlb_pmap;

sys/x86/include/x86_smp.h

84

typedef void (*smp_invl_cb_t)(struct pmap *, vm_offset_t addr1,

sys/x86/x86/busdma_bounce.c

104

pmap_t pmap, void *buf, bus_size_t buflen, int flags);

sys/x86/x86/busdma_bounce.c

543

_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,

sys/x86/x86/busdma_bounce.c

568

if (pmap == kernel_pmap)

sys/x86/x86/busdma_bounce.c

571

paddr = pmap_extract(pmap, vaddr);

sys/x86/x86/busdma_bounce.c

685

bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,

sys/x86/x86/busdma_bounce.c

700

_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);

sys/x86/x86/busdma_bounce.c

713

if (pmap == kernel_pmap) {

sys/x86/x86/busdma_bounce.c

717

curaddr = pmap_extract(pmap, (vm_offset_t)vaddr);

usr.sbin/rpcbind/pmap_svc.c

164

struct pmap reg;

usr.sbin/rpcbind/pmap_svc.c

258

struct pmap reg;

usr.sbin/rpcbind/rpcb_svc_com.c

1403

struct pmap pmap;

usr.sbin/rpcbind/rpcb_svc_com.c

1409

pmap.pm_prot = IPPROTO_UDP;

usr.sbin/rpcbind/rpcb_svc_com.c

1412

pmap.pm_prot = IPPROTO_TCP;

usr.sbin/rpcbind/rpcb_svc_com.c

1421

pmap.pm_port = ((p1 & 0xff) << 8) + (p2 & 0xff);

usr.sbin/rpcbind/rpcb_svc_com.c

1422

pmap.pm_prog = arg->r_prog;

usr.sbin/rpcbind/rpcb_svc_com.c

1423

pmap.pm_vers = arg->r_vers;

usr.sbin/rpcbind/rpcb_svc_com.c

1432

pml->pml_map = pmap;

usr.sbin/rpcbind/security.c

67

struct pmap *pmap;

usr.sbin/rpcbind/security.c

81

pmap = (struct pmap *)args;

usr.sbin/rpcbind/security.c

82

prog = pmap->pm_prog;

usr.sbin/ypbind/yp_ping.c

112

struct pmap parms;