arch/arm64/include/asm/kvm_host.h

167

struct kvm_pgtable *pgt;

arch/arm64/include/asm/kvm_pgtable.h

476

int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,

arch/arm64/include/asm/kvm_pgtable.h

486

void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt);

arch/arm64/include/asm/kvm_pgtable.h

507

int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,

arch/arm64/include/asm/kvm_pgtable.h

529

u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pgtable.h

566

int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,

arch/arm64/include/asm/kvm_pgtable.h

571

static inline int kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,

arch/arm64/include/asm/kvm_pgtable.h

574

return __kvm_pgtable_stage2_init(pgt, mmu, mm_ops, 0, NULL);

arch/arm64/include/asm/kvm_pgtable.h

584

void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt);

arch/arm64/include/asm/kvm_pgtable.h

595

void kvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,

arch/arm64/include/asm/kvm_pgtable.h

604

void kvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt);

arch/arm64/include/asm/kvm_pgtable.h

638

kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt,

arch/arm64/include/asm/kvm_pgtable.h

672

int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/include/asm/kvm_pgtable.h

697

int kvm_pgtable_stage2_annotate(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/include/asm/kvm_pgtable.h

718

int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pgtable.h

736

int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pgtable.h

749

void kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/include/asm/kvm_pgtable.h

771

bool kvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/include/asm/kvm_pgtable.h

792

int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/include/asm/kvm_pgtable.h

809

int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pgtable.h

827

int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/include/asm/kvm_pgtable.h

851

int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/include/asm/kvm_pgtable.h

871

int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/include/asm/kvm_pkvm.h

202

int pkvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,

arch/arm64/include/asm/kvm_pkvm.h

204

void pkvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,

arch/arm64/include/asm/kvm_pkvm.h

206

void pkvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt);

arch/arm64/include/asm/kvm_pkvm.h

207

int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,

arch/arm64/include/asm/kvm_pkvm.h

210

int pkvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pkvm.h

211

int pkvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pkvm.h

212

int pkvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pkvm.h

213

bool pkvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr, u64 size, bool mkold);

arch/arm64/include/asm/kvm_pkvm.h

214

int pkvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, enum kvm_pgtable_prot prot,

arch/arm64/include/asm/kvm_pkvm.h

216

void pkvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/include/asm/kvm_pkvm.h

218

int pkvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/include/asm/kvm_pkvm.h

221

kvm_pte_t *pkvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, u64 phys, s8 level,

arch/arm64/kvm/hyp/include/nvhe/mem_protect.h

20

struct kvm_pgtable pgt;

arch/arm64/kvm/hyp/include/nvhe/pkvm.h

41

struct kvm_pgtable pgt;

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1017

WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1052

WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1311

ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1350

ret = kvm_pgtable_stage2_annotate(&vm->pgt, ipa, PAGE_SIZE, NULL,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1394

WARN_ON(kvm_pgtable_stage2_unmap(&vm->pgt, ipa, PAGE_SIZE));

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1418

if (vcpu->vcpu.arch.pkvm_memcache.nr_pages < kvm_mmu_cache_min_pages(vm->pgt.mmu))

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1450

WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1521

WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1540

ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1589

ret = kvm_pgtable_stage2_unmap(&vm->pgt, ipa, size);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1640

ret = kvm_pgtable_stage2_relax_perms(&vm->pgt, ipa, prot, 0);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1660

ret = kvm_pgtable_stage2_wrprotect(&vm->pgt, ipa, size);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1680

ret = kvm_pgtable_stage2_test_clear_young(&vm->pgt, ipa, size, mkold);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1696

kvm_pgtable_stage2_mkyoung(&vm->pgt, ipa, 0);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1715

return BIT(selftest_vcpu->vcpu.arch.hw_mmu->pgt->ia_bits - 1);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

172

ret = __kvm_pgtable_stage2_init(&host_mmu.pgt, mmu,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

178

mmu->pgd_phys = __hyp_pa(host_mmu.pgt.pgd);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

179

mmu->pgt = &host_mmu.pgt;

arch/arm64/kvm/hyp/nvhe/mem_protect.c

298

ret = __kvm_pgtable_stage2_init(mmu->pgt, mmu, &vm->mm_ops, 0, NULL);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

303

vm->kvm.arch.mmu.pgd_phys = __hyp_pa(vm->pgt.pgd);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

311

kvm_pgtable_stage2_destroy(&vm->pgt);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

376

struct kvm_pgtable *pgt = &host_mmu.pgt;

arch/arm64/kvm/hyp/nvhe/mem_protect.c

384

ret = kvm_pgtable_stage2_unmap(pgt, addr, reg->base - addr);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

388

return kvm_pgtable_stage2_unmap(pgt, addr, BIT(pgt->ia_bits) - addr);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

489

return kvm_pgtable_stage2_map(&host_mmu.pgt, start, end - start, start,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

528

ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

592

ret = host_stage2_try(kvm_pgtable_stage2_annotate, &host_mmu.pgt,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

795

static int check_page_state_range(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

804

return kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

896

return check_page_state_range(&vm->pgt, addr, size, &d);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

906

ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

955

ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);

arch/arm64/kvm/hyp/nvhe/pkvm.c

445

mmu->pgt = &hyp_vm->pgt;

arch/arm64/kvm/hyp/nvhe/pkvm.c

737

.pgt = &selftest_vm.pgt,

arch/arm64/kvm/hyp/pgtable.c

1002

stage2_pte_cacheable(pgt, new))

arch/arm64/kvm/hyp/pgtable.c

1094

int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/hyp/pgtable.c

1101

.mmu = pgt->mmu,

arch/arm64/kvm/hyp/pgtable.c

1103

.force_pte = pgt->force_pte_cb && pgt->force_pte_cb(addr, addr + size, prot),

arch/arm64/kvm/hyp/pgtable.c

1113

if (WARN_ON((pgt->flags & KVM_PGTABLE_S2_IDMAP) && (addr != phys)))

arch/arm64/kvm/hyp/pgtable.c

1116

ret = stage2_set_prot_attr(pgt, prot, &map_data.attr);

arch/arm64/kvm/hyp/pgtable.c

1120

ret = kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

1125

int kvm_pgtable_stage2_annotate(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/hyp/pgtable.c

1131

.mmu = pgt->mmu,

arch/arm64/kvm/hyp/pgtable.c

1151

ret = kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

1158

struct kvm_pgtable *pgt = ctx->arg;

arch/arm64/kvm/hyp/pgtable.c

1159

struct kvm_s2_mmu *mmu = pgt->mmu;

arch/arm64/kvm/hyp/pgtable.c

1177

} else if (stage2_pte_cacheable(pgt, ctx->old)) {

arch/arm64/kvm/hyp/pgtable.c

1198

int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/hyp/pgtable.c

1203

.arg = pgt,

arch/arm64/kvm/hyp/pgtable.c

1207

ret = kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

1208

if (stage2_unmap_defer_tlb_flush(pgt))

arch/arm64/kvm/hyp/pgtable.c

1210

kvm_tlb_flush_vmid_range(pgt->mmu, addr, size);

arch/arm64/kvm/hyp/pgtable.c

1259

static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/kvm/hyp/pgtable.c

1276

ret = kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

1288

int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/hyp/pgtable.c

1290

return stage2_update_leaf_attrs(pgt, addr, size, 0,

arch/arm64/kvm/hyp/pgtable.c

1296

void kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/kvm/hyp/pgtable.c

1301

ret = stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0,

arch/arm64/kvm/hyp/pgtable.c

1341

bool kvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/kvm/hyp/pgtable.c

1353

WARN_ON(kvm_pgtable_walk(pgt, addr, size, &walker));

arch/arm64/kvm/hyp/pgtable.c

1357

int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/kvm/hyp/pgtable.c

1382

ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags);

arch/arm64/kvm/hyp/pgtable.c

1384

kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr,

arch/arm64/kvm/hyp/pgtable.c

1392

struct kvm_pgtable *pgt = ctx->arg;

arch/arm64/kvm/hyp/pgtable.c

1393

struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;

arch/arm64/kvm/hyp/pgtable.c

1395

if (!stage2_pte_cacheable(pgt, ctx->old))

arch/arm64/kvm/hyp/pgtable.c

1404

int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/hyp/pgtable.c

1409

.arg = pgt,

arch/arm64/kvm/hyp/pgtable.c

1415

return kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

1418

kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt,

arch/arm64/kvm/hyp/pgtable.c

1425

.mmu = pgt->mmu,

arch/arm64/kvm/hyp/pgtable.c

1446

struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;

arch/arm64/kvm/hyp/pgtable.c

1453

ret = stage2_set_prot_attr(pgt, prot, &map_data.attr);

arch/arm64/kvm/hyp/pgtable.c

1540

childp = kvm_pgtable_stage2_create_unlinked(mmu->pgt, phys,

arch/arm64/kvm/hyp/pgtable.c

1560

int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/hyp/pgtable.c

1570

ret = kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

1575

int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,

arch/arm64/kvm/hyp/pgtable.c

1587

pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz);

arch/arm64/kvm/hyp/pgtable.c

1588

if (!pgt->pgd)

arch/arm64/kvm/hyp/pgtable.c

1591

pgt->ia_bits = ia_bits;

arch/arm64/kvm/hyp/pgtable.c

1592

pgt->start_level = start_level;

arch/arm64/kvm/hyp/pgtable.c

1593

pgt->mm_ops = mm_ops;

arch/arm64/kvm/hyp/pgtable.c

1594

pgt->mmu = mmu;

arch/arm64/kvm/hyp/pgtable.c

1595

pgt->flags = flags;

arch/arm64/kvm/hyp/pgtable.c

1596

pgt->force_pte_cb = force_pte_cb;

arch/arm64/kvm/hyp/pgtable.c

1654

void kvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,

arch/arm64/kvm/hyp/pgtable.c

1663

WARN_ON(kvm_pgtable_walk(pgt, addr, size, &walker));

arch/arm64/kvm/hyp/pgtable.c

1666

void kvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt)

arch/arm64/kvm/hyp/pgtable.c

1670

pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;

arch/arm64/kvm/hyp/pgtable.c

1676

pgt->mm_ops->free_pages_exact(kvm_dereference_pteref_raw(pgt->pgd), pgd_sz);

arch/arm64/kvm/hyp/pgtable.c

1677

pgt->pgd = NULL;

arch/arm64/kvm/hyp/pgtable.c

1680

void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt)

arch/arm64/kvm/hyp/pgtable.c

1682

kvm_pgtable_stage2_destroy_range(pgt, 0, BIT(pgt->ia_bits));

arch/arm64/kvm/hyp/pgtable.c

1683

kvm_pgtable_stage2_destroy_pgd(pgt);

arch/arm64/kvm/hyp/pgtable.c

240

static int _kvm_pgtable_walk(struct kvm_pgtable *pgt, struct kvm_pgtable_walk_data *data)

arch/arm64/kvm/hyp/pgtable.c

244

u64 limit = BIT(pgt->ia_bits);

arch/arm64/kvm/hyp/pgtable.c

249

if (!pgt->pgd)

arch/arm64/kvm/hyp/pgtable.c

252

for (idx = kvm_pgd_page_idx(pgt, data->addr); data->addr < data->end; ++idx) {

arch/arm64/kvm/hyp/pgtable.c

253

kvm_pteref_t pteref = &pgt->pgd[idx * PTRS_PER_PTE];

arch/arm64/kvm/hyp/pgtable.c

255

ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level);

arch/arm64/kvm/hyp/pgtable.c

263

int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/hyp/pgtable.c

278

r = _kvm_pgtable_walk(pgt, &walk_data);

arch/arm64/kvm/hyp/pgtable.c

300

int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/kvm/hyp/pgtable.c

311

ret = kvm_pgtable_walk(pgt, ALIGN_DOWN(addr, PAGE_SIZE),

arch/arm64/kvm/hyp/pgtable.c

446

int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,

arch/arm64/kvm/hyp/pgtable.c

463

ret = kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

509

u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/hyp/pgtable.c

518

if (!pgt->mm_ops->page_count)

arch/arm64/kvm/hyp/pgtable.c

521

kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

525

int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,

arch/arm64/kvm/hyp/pgtable.c

535

pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL);

arch/arm64/kvm/hyp/pgtable.c

536

if (!pgt->pgd)

arch/arm64/kvm/hyp/pgtable.c

539

pgt->ia_bits = va_bits;

arch/arm64/kvm/hyp/pgtable.c

540

pgt->start_level = start_level;

arch/arm64/kvm/hyp/pgtable.c

541

pgt->mm_ops = mm_ops;

arch/arm64/kvm/hyp/pgtable.c

542

pgt->mmu = NULL;

arch/arm64/kvm/hyp/pgtable.c

543

pgt->force_pte_cb = NULL;

arch/arm64/kvm/hyp/pgtable.c

56

static u32 kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr)

arch/arm64/kvm/hyp/pgtable.c

564

void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt)

arch/arm64/kvm/hyp/pgtable.c

571

WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));

arch/arm64/kvm/hyp/pgtable.c

572

pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd));

arch/arm64/kvm/hyp/pgtable.c

573

pgt->pgd = NULL;

arch/arm64/kvm/hyp/pgtable.c

58

u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */

arch/arm64/kvm/hyp/pgtable.c

59

u64 mask = BIT(pgt->ia_bits) - 1;

arch/arm64/kvm/hyp/pgtable.c

66

struct kvm_pgtable pgt = {

arch/arm64/kvm/hyp/pgtable.c

665

#define KVM_S2_MEMATTR(pgt, attr) \

arch/arm64/kvm/hyp/pgtable.c

669

if ((pgt)->flags & KVM_PGTABLE_S2_AS_S1) \

arch/arm64/kvm/hyp/pgtable.c

702

static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot prot,

arch/arm64/kvm/hyp/pgtable.c

71

return kvm_pgd_page_idx(&pgt, -1ULL) + 1;

arch/arm64/kvm/hyp/pgtable.c

716

attr = KVM_S2_MEMATTR(pgt, DEVICE_nGnRE);

arch/arm64/kvm/hyp/pgtable.c

721

attr = KVM_S2_MEMATTR(pgt, NORMAL_NC);

arch/arm64/kvm/hyp/pgtable.c

724

attr = KVM_S2_MEMATTR(pgt, NORMAL);

arch/arm64/kvm/hyp/pgtable.c

881

static bool stage2_unmap_defer_tlb_flush(struct kvm_pgtable *pgt)

arch/arm64/kvm/hyp/pgtable.c

898

struct kvm_pgtable *pgt = ctx->arg;

arch/arm64/kvm/hyp/pgtable.c

911

} else if (!stage2_unmap_defer_tlb_flush(pgt)) {

arch/arm64/kvm/hyp/pgtable.c

920

static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte)

arch/arm64/kvm/hyp/pgtable.c

923

return kvm_pte_valid(pte) && memattr == KVM_S2_MEMATTR(pgt, NORMAL);

arch/arm64/kvm/hyp/pgtable.c

962

struct kvm_pgtable *pgt = data->mmu->pgt;

arch/arm64/kvm/mmu.c

1008

pgt = kzalloc_obj(*pgt, GFP_KERNEL_ACCOUNT);

arch/arm64/kvm/mmu.c

1009

if (!pgt)

arch/arm64/kvm/mmu.c

1013

err = KVM_PGT_FN(kvm_pgtable_stage2_init)(pgt, mmu, &kvm_s2_mm_ops);

arch/arm64/kvm/mmu.c

1017

mmu->pgt = pgt;

arch/arm64/kvm/mmu.c

1034

mmu->pgd_phys = __pa(pgt->pgd);

arch/arm64/kvm/mmu.c

1042

kvm_stage2_destroy(pgt);

arch/arm64/kvm/mmu.c

1043

mmu->pgt = NULL;

arch/arm64/kvm/mmu.c

1045

kfree(pgt);

arch/arm64/kvm/mmu.c

1128

struct kvm_pgtable *pgt = NULL;

arch/arm64/kvm/mmu.c

1131

pgt = mmu->pgt;

arch/arm64/kvm/mmu.c

1132

if (pgt) {

arch/arm64/kvm/mmu.c

1134

mmu->pgt = NULL;

arch/arm64/kvm/mmu.c

1143

if (pgt) {

arch/arm64/kvm/mmu.c

1144

kvm_stage2_destroy(pgt);

arch/arm64/kvm/mmu.c

1145

kfree(pgt);

arch/arm64/kvm/mmu.c

1212

struct kvm_pgtable *pgt = mmu->pgt;

arch/arm64/kvm/mmu.c

1230

ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, addr, PAGE_SIZE,

arch/arm64/kvm/mmu.c

124

struct kvm_pgtable *pgt;

arch/arm64/kvm/mmu.c

151

pgt = kvm->arch.mmu.pgt;

arch/arm64/kvm/mmu.c

152

if (!pgt)

arch/arm64/kvm/mmu.c

156

ret = KVM_PGT_FN(kvm_pgtable_stage2_split)(pgt, addr, next - addr, cache);

arch/arm64/kvm/mmu.c

1614

struct kvm_pgtable *pgt = s2fd->vcpu->arch.hw_mmu->pgt;

arch/arm64/kvm/mmu.c

1675

ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, s2fd->fault_ipa,

arch/arm64/kvm/mmu.c

1678

ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, s2fd->fault_ipa, PAGE_SIZE,

arch/arm64/kvm/mmu.c

1712

struct kvm_pgtable *pgt = vcpu->arch.hw_mmu->pgt;

arch/arm64/kvm/mmu.c

1759

ret = pkvm_pgtable_stage2_map(pgt, s2fd->fault_ipa, PAGE_SIZE,

arch/arm64/kvm/mmu.c

2024

struct kvm_pgtable *pgt;

arch/arm64/kvm/mmu.c

2032

pgt = s2fd->vcpu->arch.hw_mmu->pgt;

arch/arm64/kvm/mmu.c

2076

ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, gfn_to_gpa(gfn),

arch/arm64/kvm/mmu.c

2079

ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, gfn_to_gpa(gfn), mapping_size,

arch/arm64/kvm/mmu.c

2153

KVM_PGT_FN(kvm_pgtable_stage2_mkyoung)(mmu->pgt, fault_ipa, flags);

arch/arm64/kvm/mmu.c

2436

if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm))

arch/arm64/kvm/mmu.c

2451

if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm))

arch/arm64/kvm/mmu.c

2454

return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt,

arch/arm64/kvm/mmu.c

2467

if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm))

arch/arm64/kvm/mmu.c

2470

return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt,

arch/arm64/kvm/mmu.c

72

struct kvm_pgtable *pgt = mmu->pgt;

arch/arm64/kvm/mmu.c

73

if (!pgt)

arch/arm64/kvm/mmu.c

77

ret = fn(pgt, addr, next - addr);

arch/arm64/kvm/mmu.c

851

struct kvm_pgtable pgt = {

arch/arm64/kvm/mmu.c

855

ARM64_HW_PGTABLE_LEVELS(pgt.ia_bits) + 1),

arch/arm64/kvm/mmu.c

869

ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level);

arch/arm64/kvm/mmu.c

944

static void stage2_destroy_range(struct kvm_pgtable *pgt, phys_addr_t addr,

arch/arm64/kvm/mmu.c

951

KVM_PGT_FN(kvm_pgtable_stage2_destroy_range)(pgt, addr,

arch/arm64/kvm/mmu.c

958

static void kvm_stage2_destroy(struct kvm_pgtable *pgt)

arch/arm64/kvm/mmu.c

960

unsigned int ia_bits = VTCR_EL2_IPA(pgt->mmu->vtcr);

arch/arm64/kvm/mmu.c

962

stage2_destroy_range(pgt, 0, BIT(ia_bits));

arch/arm64/kvm/mmu.c

963

KVM_PGT_FN(kvm_pgtable_stage2_destroy_pgd)(pgt);

arch/arm64/kvm/mmu.c

984

struct kvm_pgtable *pgt;

arch/arm64/kvm/mmu.c

996

if (mmu->pgt != NULL) {

arch/arm64/kvm/nested.c

106

tmp[i].pgt->mmu = &tmp[i];

arch/arm64/kvm/nested.c

1232

kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));

arch/arm64/kvm/nested.c

1251

kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));

arch/arm64/kvm/nested.c

644

if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL))

arch/arm64/kvm/pkvm.c

328

int pkvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,

arch/arm64/kvm/pkvm.c

331

pgt->pkvm_mappings = RB_ROOT_CACHED;

arch/arm64/kvm/pkvm.c

332

pgt->mmu = mmu;

arch/arm64/kvm/pkvm.c

337

static int __pkvm_pgtable_stage2_reclaim(struct kvm_pgtable *pgt, u64 start, u64 end)

arch/arm64/kvm/pkvm.c

339

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

344

for_each_mapping_in_range_safe(pgt, start, end, mapping) {

arch/arm64/kvm/pkvm.c

356

pkvm_mapping_remove(mapping, &pgt->pkvm_mappings);

arch/arm64/kvm/pkvm.c

363

static int __pkvm_pgtable_stage2_unshare(struct kvm_pgtable *pgt, u64 start, u64 end)

arch/arm64/kvm/pkvm.c

365

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

370

for_each_mapping_in_range_safe(pgt, start, end, mapping) {

arch/arm64/kvm/pkvm.c

375

pkvm_mapping_remove(mapping, &pgt->pkvm_mappings);

arch/arm64/kvm/pkvm.c

382

void pkvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,

arch/arm64/kvm/pkvm.c

385

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

397

__pkvm_pgtable_stage2_reclaim(pgt, addr, addr + size);

arch/arm64/kvm/pkvm.c

399

__pkvm_pgtable_stage2_unshare(pgt, addr, addr + size);

arch/arm64/kvm/pkvm.c

402

void pkvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt)

arch/arm64/kvm/pkvm.c

405

WARN_ON_ONCE(!RB_EMPTY_ROOT(&pgt->pkvm_mappings.rb_root));

arch/arm64/kvm/pkvm.c

408

int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/pkvm.c

412

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

421

mapping = pkvm_mapping_iter_first(&pgt->pkvm_mappings, addr, end - 1);

arch/arm64/kvm/pkvm.c

458

ret = __pkvm_pgtable_stage2_unshare(pgt, addr, end);

arch/arm64/kvm/pkvm.c

476

pkvm_mapping_insert(mapping, &pgt->pkvm_mappings);

arch/arm64/kvm/pkvm.c

481

int pkvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/pkvm.c

483

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

490

return __pkvm_pgtable_stage2_unshare(pgt, addr, addr + size);

arch/arm64/kvm/pkvm.c

493

int pkvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/pkvm.c

495

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

504

for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) {

arch/arm64/kvm/pkvm.c

514

int pkvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/pkvm.c

516

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

520

for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping)

arch/arm64/kvm/pkvm.c

527

bool pkvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr, u64 size, bool mkold)

arch/arm64/kvm/pkvm.c

529

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

538

for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping)

arch/arm64/kvm/pkvm.c

545

int pkvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, enum kvm_pgtable_prot prot,

arch/arm64/kvm/pkvm.c

548

if (WARN_ON(kvm_vm_is_protected(kvm_s2_mmu_to_kvm(pgt->mmu))))

arch/arm64/kvm/pkvm.c

554

void pkvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/kvm/pkvm.c

557

if (WARN_ON(kvm_vm_is_protected(kvm_s2_mmu_to_kvm(pgt->mmu))))

arch/arm64/kvm/pkvm.c

568

kvm_pte_t *pkvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, u64 phys, s8 level,

arch/arm64/kvm/pkvm.c

575

int pkvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/ptdump.c

119

struct kvm_pgtable *pgtable = mmu->pgt;

arch/arm64/kvm/ptdump.c

159

ret = kvm_pgtable_walk(mmu->pgt, 0, BIT(mmu->pgt->ia_bits), &walker);

arch/arm64/kvm/ptdump.c

236

pgtable = mmu->pgt;

arch/powerpc/kvm/book3s_64_mmu_radix.c

1288

pgd_t *pgt;

arch/powerpc/kvm/book3s_64_mmu_radix.c

1326

pgt = NULL;

arch/powerpc/kvm/book3s_64_mmu_radix.c

1330

pgt = NULL;

arch/powerpc/kvm/book3s_64_mmu_radix.c

1340

if (!pgt) {

arch/powerpc/kvm/book3s_64_mmu_radix.c

1342

pgt = kvm->arch.pgtable;

arch/powerpc/kvm/book3s_64_mmu_radix.c

1349

pgt = nested->shadow_pgtable;

arch/powerpc/kvm/book3s_64_mmu_radix.c

1358

"pgdir: %lx\n", (unsigned long)pgt);

arch/powerpc/kvm/book3s_64_mmu_radix.c

1363

pgdp = pgt + pgd_index(gpa);

arch/x86/events/intel/uncore_nhmex.c

879

DEFINE_UNCORE_FORMAT_ATTR(pgt, pgt, "config1:0-31");

arch/x86/kernel/acpi/madt_wakeup.c

68

static void __init free_pgt_page(void *pgt, void *dummy)

arch/x86/kernel/acpi/madt_wakeup.c

70

return memblock_free(pgt, PAGE_SIZE);

drivers/accel/habanalabs/common/habanalabs.h

3944

u64 hl_mmu_hr_pte_phys_to_virt(struct hl_ctx *ctx, struct pgt_info *pgt, u64 phys_pte_addr,

drivers/accel/habanalabs/common/mmu/mmu.c

953

u64 hl_mmu_hr_pte_phys_to_virt(struct hl_ctx *ctx, struct pgt_info *pgt,

drivers/accel/habanalabs/common/mmu/mmu.c

959

return pgt->virt_addr + pte_offset;

drivers/firmware/efi/libstub/x86-5lvl.c

69

u64 *pgt = (void *)la57_toggle + PAGE_SIZE;

drivers/firmware/efi/libstub/x86-5lvl.c

82

new_cr3 = memset(pgt, 0, PAGE_SIZE);

drivers/firmware/efi/libstub/x86-5lvl.c

90

new_cr3 = memcpy(pgt, new_cr3, PAGE_SIZE);

drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c

52

struct nvkm_memory *pgt =

drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c

55

return nvkm_gpuobj_wrap(pgt, pgpuobj);

drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c

56

nvkm_kmap(pgt);

drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c

57

offset = nvkm_ro32(pgt, 8 + (offset >> 10));

drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c

59

nvkm_done(pgt);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

144

struct nvkm_vmm_pt *pgt = it->pt[it->lvl];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

145

struct nvkm_mmu_pt *pt = pgt->pt[type];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

155

pgt->pt[type] = NULL;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

156

if (!pgt->refs[!type]) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

159

if (pgt->sparse) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

192

if (!pgt->refs[!type])

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

193

nvkm_vmm_pt_del(&pgt);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

198

nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

212

pgt->pte[lpti].s.sptes -= pten;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

217

if (!pgt->refs[0])

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

222

if (pgt->pte[pteb].s.sptes) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

224

if (!(pgt->pte[ptei].s.sptes))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

238

if (pgt->pte[ptei].s.sptes)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

243

pgt->pte[pteb].s.spte_valid = false;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

244

if (pgt->pte[pteb].s.sparse) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

246

pair->func->sparse(vmm, pgt->pt[0], pteb, 1);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

247

} else if (!pgt->pte[pteb].s.lpte_valid) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

250

pair->func->invalid(vmm, pgt->pt[0], pteb, 1);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

263

struct nvkm_vmm_pt *pgt = it->pt[0];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

268

dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

273

desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

278

pgt->refs[type] -= ptes;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

281

if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

282

nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

284

if (desc->type == LPT && (pgt->refs[0] || pgt->refs[1])) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

286

pgt->pte[lpti].s.lptes--;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

287

if (pgt->pte[lpti].s.lptes == 0)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

288

pgt->pte[lpti].s.lpte_valid = false;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

294

if (!pgt->refs[type]) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

30

struct nvkm_vmm_pt *pgt = *ppgt;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

306

nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

31

if (pgt) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

32

kvfree(pgt->pde);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

320

pgt->pte[lpti].s.sptes += pten;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

325

if (!pgt->refs[0])

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

33

kfree(pgt);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

330

if (pgt->pte[pteb].s.spte_valid) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

332

if (!pgt->pte[ptei].s.spte_valid)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

344

pgt->pte[ptei].s.spte_valid = true;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

345

pgt->pte[ptei].s.lpte_valid = false;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

347

if (pgt->pte[ptei].s.spte_valid)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

349

pgt->pte[ptei].s.spte_valid = true;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

350

pgt->pte[ptei].s.lpte_valid = false;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

353

if (pgt->pte[pteb].s.sparse) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

360

desc->func->sparse(vmm, pgt->pt[1], spti, sptc);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

363

pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

370

pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

380

struct nvkm_vmm_pt *pgt = it->pt[0];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

383

pgt->refs[type] += ptes;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

387

nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

391

pgt->pte[lpti].s.spte_valid = false;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

392

pgt->pte[lpti].s.lpte_valid = true;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

393

pgt->pte[lpti].s.lptes++;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

403

struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

407

pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

411

memset32(&pgt->pte[ptei].u, sparse.u, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

439

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

44

struct nvkm_vmm_pt *pgt;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

440

const bool zero = !pgt->sparse && !desc->func->invalid;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

450

pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

451

if (!pgt->pt[type]) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

460

pt = pgt->pt[type];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

462

if (desc->type == LPT && pgt->refs[1]) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

470

bool spte = !!pgt->pte[ptei].s.sptes;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

472

bool next = !!pgt->pte[ptei].s.sptes;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

478

if (pgt->sparse)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

482

memset32(&pgt->pte[pteb].u, 0x00, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

486

pgt->pte[pteb++].s.spte_valid = true;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

490

if (pgt->sparse) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

491

nvkm_vmm_sparse_ptes(desc, pgt, 0, pten);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

509

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

511

pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

512

if (!pgt) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

518

pgd->pde[pdei] = pgt;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

554

struct nvkm_vmm_pt *pgt = it.pt[it.lvl];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

56

pgt = kzalloc_flex(*pgt, pte, lpte);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

563

struct nvkm_vmm_pt *pgd = pgt;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

57

if (!pgt)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

570

it.pt[it.lvl - 1] = pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

578

if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

586

struct nvkm_mmu_pt *pt = pgt->pt[type];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

59

pgt->page = page ? page->shift : 0;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

60

pgt->sparse = sparse;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

63

pgt->pde = kvzalloc_objs(*pgt->pde, pten);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

64

if (!pgt->pde) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

65

kfree(pgt);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

70

return pgt;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c

108

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c

113

if ((pt = pgt->pt[0])) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c

127

if ((pt = pgt->pt[1])) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgh100.c

140

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgh100.c

144

if (pgt->pt[0] && !gh100_vmm_pde(pgt->pt[0], &data[0]))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgh100.c

146

if (pgt->pt[1] && !gh100_vmm_pde(pgt->pt[1], &data[1]))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgh100.c

181

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgh100.c

185

if (!gh100_vmm_pde(pgt->pt[0], &data))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c

256

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c

260

if (pgt->pt[0] && !gp100_vmm_pde(pgt->pt[0], &data[0]))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c

262

if (pgt->pt[1] && !gp100_vmm_pde(pgt->pt[1], &data[1]))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c

383

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c

387

if (!gp100_vmm_pde(pgt->pt[0], &data))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c

106

nv50_vmm_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgt, u64 *pdata)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c

110

if (pgt && (pt = pgt->pt[0])) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c

111

switch (pgt->page) {

drivers/gpu/drm/panthor/panthor_mmu.c

848

const struct io_pgtable *pgt = io_pgtable_ops_to_pgtable(vm->pgtbl_ops);

drivers/gpu/drm/panthor/panthor_mmu.c

849

u32 pg_shift = ffs(pgt->cfg.pgsize_bitmap) - 1;

drivers/net/ethernet/broadcom/bnx2x/bnx2x_self_test.c

395

NA, 1, 0, pgt,

drivers/net/ethernet/mellanox/mlxsw/spectrum.h

210

struct mlxsw_sp_pgt *pgt;

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

113

mlxsw_sp_pgt_entry_create(struct mlxsw_sp_pgt *pgt, u16 mid, u16 smpe)

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

123

ret = idr_replace(&pgt->pgt_idr, pgt_entry, mid);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

139

static void mlxsw_sp_pgt_entry_destroy(struct mlxsw_sp_pgt *pgt,

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

144

pgt_entry = idr_replace(&pgt->pgt_idr, NULL, pgt_entry->index);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

152

mlxsw_sp_pgt_entry_get(struct mlxsw_sp_pgt *pgt, u16 mid, u16 smpe)

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

156

pgt_entry = idr_find(&pgt->pgt_idr, mid);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

160

return mlxsw_sp_pgt_entry_create(pgt, mid, smpe);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

163

static void mlxsw_sp_pgt_entry_put(struct mlxsw_sp_pgt *pgt, u16 mid)

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

167

pgt_entry = idr_find(&pgt->pgt_idr, mid);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

172

mlxsw_sp_pgt_entry_destroy(pgt, pgt_entry);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

195

mlxsw_sp->pgt->smpe_index_valid,

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

252

mutex_lock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

254

pgt_entry = mlxsw_sp_pgt_entry_get(mlxsw_sp->pgt, mid, smpe);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

267

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

271

mlxsw_sp_pgt_entry_put(mlxsw_sp->pgt, mid);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

273

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

283

mutex_lock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

285

pgt_entry = idr_find(&mlxsw_sp->pgt->pgt_idr, mid);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

294

mlxsw_sp_pgt_entry_put(mlxsw_sp->pgt, mid);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

297

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

313

struct mlxsw_sp_pgt *pgt;

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

318

pgt = kzalloc_obj(*mlxsw_sp->pgt);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

319

if (!pgt)

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

32

mutex_lock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

322

idr_init(&pgt->pgt_idr);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

323

pgt->end_index = MLXSW_CORE_RES_GET(mlxsw_sp->core, PGT_SIZE);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

324

mutex_init(&pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

325

pgt->smpe_index_valid = mlxsw_sp->pgt_smpe_index_valid;

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

326

mlxsw_sp->pgt = pgt;

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

33

index = idr_alloc(&mlxsw_sp->pgt->pgt_idr, NULL, 0,

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

332

mutex_destroy(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

333

WARN_ON(!idr_is_empty(&mlxsw_sp->pgt->pgt_idr));

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

334

idr_destroy(&mlxsw_sp->pgt->pgt_idr);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

335

kfree(mlxsw_sp->pgt);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

34

mlxsw_sp->pgt->end_index, GFP_KERNEL);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

42

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

46

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

52

mutex_lock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

53

WARN_ON(idr_remove(&mlxsw_sp->pgt->pgt_idr, mid_base));

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

54

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

63

mutex_lock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

65

mid_base = idr_get_cursor(&mlxsw_sp->pgt->pgt_idr);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

67

err = idr_alloc_cyclic(&mlxsw_sp->pgt->pgt_idr, NULL,

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

73

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

79

idr_remove(&mlxsw_sp->pgt->pgt_idr, mid_base + i);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

80

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

87

struct idr *pgt_idr = &mlxsw_sp->pgt->pgt_idr;

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

90

mutex_lock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

95

mutex_unlock(&mlxsw_sp->pgt->lock);