arch/arm64/include/asm/kvm_host.h

167

struct kvm_pgtable *pgt;

arch/arm64/include/asm/kvm_pgtable.h

460

int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,

arch/arm64/include/asm/kvm_pgtable.h

470

void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt);

arch/arm64/include/asm/kvm_pgtable.h

491

int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,

arch/arm64/include/asm/kvm_pgtable.h

513

u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pgtable.h

550

int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,

arch/arm64/include/asm/kvm_pgtable.h

555

static inline int kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,

arch/arm64/include/asm/kvm_pgtable.h

558

return __kvm_pgtable_stage2_init(pgt, mmu, mm_ops, 0, NULL);

arch/arm64/include/asm/kvm_pgtable.h

568

void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt);

arch/arm64/include/asm/kvm_pgtable.h

579

void kvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,

arch/arm64/include/asm/kvm_pgtable.h

588

void kvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt);

arch/arm64/include/asm/kvm_pgtable.h

622

kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt,

arch/arm64/include/asm/kvm_pgtable.h

656

int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/include/asm/kvm_pgtable.h

677

int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/include/asm/kvm_pgtable.h

697

int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pgtable.h

715

int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pgtable.h

728

void kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/include/asm/kvm_pgtable.h

750

bool kvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/include/asm/kvm_pgtable.h

771

int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/include/asm/kvm_pgtable.h

788

int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pgtable.h

806

int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/include/asm/kvm_pgtable.h

830

int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/include/asm/kvm_pgtable.h

850

int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/include/asm/kvm_pkvm.h

205

int pkvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,

arch/arm64/include/asm/kvm_pkvm.h

207

void pkvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,

arch/arm64/include/asm/kvm_pkvm.h

209

void pkvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt);

arch/arm64/include/asm/kvm_pkvm.h

210

int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,

arch/arm64/include/asm/kvm_pkvm.h

213

int pkvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pkvm.h

214

int pkvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pkvm.h

215

int pkvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size);

arch/arm64/include/asm/kvm_pkvm.h

216

bool pkvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr, u64 size, bool mkold);

arch/arm64/include/asm/kvm_pkvm.h

217

int pkvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, enum kvm_pgtable_prot prot,

arch/arm64/include/asm/kvm_pkvm.h

219

void pkvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/include/asm/kvm_pkvm.h

221

int pkvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/include/asm/kvm_pkvm.h

224

kvm_pte_t *pkvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, u64 phys, s8 level,

arch/arm64/kvm/hyp/include/nvhe/mem_protect.h

20

struct kvm_pgtable pgt;

arch/arm64/kvm/hyp/include/nvhe/pkvm.h

41

struct kvm_pgtable pgt;

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1019

WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1038

ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1087

ret = kvm_pgtable_stage2_unmap(&vm->pgt, ipa, size);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1138

ret = kvm_pgtable_stage2_relax_perms(&vm->pgt, ipa, prot, 0);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1158

ret = kvm_pgtable_stage2_wrprotect(&vm->pgt, ipa, size);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1178

ret = kvm_pgtable_stage2_test_clear_young(&vm->pgt, ipa, size, mkold);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1194

kvm_pgtable_stage2_mkyoung(&vm->pgt, ipa, 0);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1215

.pgt = &selftest_vm.pgt,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

1248

return BIT(selftest_vm.pgt.ia_bits - 1);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

156

ret = __kvm_pgtable_stage2_init(&host_mmu.pgt, mmu,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

162

mmu->pgd_phys = __hyp_pa(host_mmu.pgt.pgd);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

163

mmu->pgt = &host_mmu.pgt;

arch/arm64/kvm/hyp/nvhe/mem_protect.c

283

ret = __kvm_pgtable_stage2_init(mmu->pgt, mmu, &vm->mm_ops, 0, NULL);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

288

vm->kvm.arch.mmu.pgd_phys = __hyp_pa(vm->pgt.pgd);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

300

kvm_pgtable_stage2_destroy(&vm->pgt);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

357

struct kvm_pgtable *pgt = &host_mmu.pgt;

arch/arm64/kvm/hyp/nvhe/mem_protect.c

365

ret = kvm_pgtable_stage2_unmap(pgt, addr, reg->base - addr);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

369

return kvm_pgtable_stage2_unmap(pgt, addr, BIT(pgt->ia_bits) - addr);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

464

return kvm_pgtable_stage2_map(&host_mmu.pgt, start, end - start, start,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

502

ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

551

ret = host_stage2_try(kvm_pgtable_stage2_set_owner, &host_mmu.pgt,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

648

static int check_page_state_range(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/hyp/nvhe/mem_protect.c

657

return kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/nvhe/mem_protect.c

727

return check_page_state_range(&vm->pgt, addr, size, &d);

arch/arm64/kvm/hyp/nvhe/pkvm.c

436

mmu->pgt = &hyp_vm->pgt;

arch/arm64/kvm/hyp/pgtable.c

1090

int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/hyp/pgtable.c

1097

.mmu = pgt->mmu,

arch/arm64/kvm/hyp/pgtable.c

1099

.force_pte = pgt->force_pte_cb && pgt->force_pte_cb(addr, addr + size, prot),

arch/arm64/kvm/hyp/pgtable.c

1109

if (WARN_ON((pgt->flags & KVM_PGTABLE_S2_IDMAP) && (addr != phys)))

arch/arm64/kvm/hyp/pgtable.c

1112

ret = stage2_set_prot_attr(pgt, prot, &map_data.attr);

arch/arm64/kvm/hyp/pgtable.c

1116

ret = kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

1121

int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/hyp/pgtable.c

1126

.mmu = pgt->mmu,

arch/arm64/kvm/hyp/pgtable.c

1142

ret = kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

1149

struct kvm_pgtable *pgt = ctx->arg;

arch/arm64/kvm/hyp/pgtable.c

1150

struct kvm_s2_mmu *mmu = pgt->mmu;

arch/arm64/kvm/hyp/pgtable.c

1168

} else if (stage2_pte_cacheable(pgt, ctx->old)) {

arch/arm64/kvm/hyp/pgtable.c

1189

int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/hyp/pgtable.c

1194

.arg = pgt,

arch/arm64/kvm/hyp/pgtable.c

1198

ret = kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

1199

if (stage2_unmap_defer_tlb_flush(pgt))

arch/arm64/kvm/hyp/pgtable.c

1201

kvm_tlb_flush_vmid_range(pgt->mmu, addr, size);

arch/arm64/kvm/hyp/pgtable.c

1250

static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/kvm/hyp/pgtable.c

1267

ret = kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

1279

int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/hyp/pgtable.c

1281

return stage2_update_leaf_attrs(pgt, addr, size, 0,

arch/arm64/kvm/hyp/pgtable.c

1287

void kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/kvm/hyp/pgtable.c

1292

ret = stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0,

arch/arm64/kvm/hyp/pgtable.c

1332

bool kvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/kvm/hyp/pgtable.c

1344

WARN_ON(kvm_pgtable_walk(pgt, addr, size, &walker));

arch/arm64/kvm/hyp/pgtable.c

1348

int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/kvm/hyp/pgtable.c

1371

ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags);

arch/arm64/kvm/hyp/pgtable.c

1373

kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, level);

arch/arm64/kvm/hyp/pgtable.c

1380

struct kvm_pgtable *pgt = ctx->arg;

arch/arm64/kvm/hyp/pgtable.c

1381

struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;

arch/arm64/kvm/hyp/pgtable.c

1383

if (!stage2_pte_cacheable(pgt, ctx->old))

arch/arm64/kvm/hyp/pgtable.c

1392

int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/hyp/pgtable.c

1397

.arg = pgt,

arch/arm64/kvm/hyp/pgtable.c

1403

return kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

1406

kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt,

arch/arm64/kvm/hyp/pgtable.c

1413

.mmu = pgt->mmu,

arch/arm64/kvm/hyp/pgtable.c

1434

struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;

arch/arm64/kvm/hyp/pgtable.c

1441

ret = stage2_set_prot_attr(pgt, prot, &map_data.attr);

arch/arm64/kvm/hyp/pgtable.c

1528

childp = kvm_pgtable_stage2_create_unlinked(mmu->pgt, phys,

arch/arm64/kvm/hyp/pgtable.c

1548

int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/hyp/pgtable.c

1558

ret = kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

1563

int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,

arch/arm64/kvm/hyp/pgtable.c

1575

pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz);

arch/arm64/kvm/hyp/pgtable.c

1576

if (!pgt->pgd)

arch/arm64/kvm/hyp/pgtable.c

1579

pgt->ia_bits = ia_bits;

arch/arm64/kvm/hyp/pgtable.c

1580

pgt->start_level = start_level;

arch/arm64/kvm/hyp/pgtable.c

1581

pgt->mm_ops = mm_ops;

arch/arm64/kvm/hyp/pgtable.c

1582

pgt->mmu = mmu;

arch/arm64/kvm/hyp/pgtable.c

1583

pgt->flags = flags;

arch/arm64/kvm/hyp/pgtable.c

1584

pgt->force_pte_cb = force_pte_cb;

arch/arm64/kvm/hyp/pgtable.c

1642

void kvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,

arch/arm64/kvm/hyp/pgtable.c

1651

WARN_ON(kvm_pgtable_walk(pgt, addr, size, &walker));

arch/arm64/kvm/hyp/pgtable.c

1654

void kvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt)

arch/arm64/kvm/hyp/pgtable.c

1658

pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;

arch/arm64/kvm/hyp/pgtable.c

1664

pgt->mm_ops->free_pages_exact(kvm_dereference_pteref_raw(pgt->pgd), pgd_sz);

arch/arm64/kvm/hyp/pgtable.c

1665

pgt->pgd = NULL;

arch/arm64/kvm/hyp/pgtable.c

1668

void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt)

arch/arm64/kvm/hyp/pgtable.c

1670

kvm_pgtable_stage2_destroy_range(pgt, 0, BIT(pgt->ia_bits));

arch/arm64/kvm/hyp/pgtable.c

1671

kvm_pgtable_stage2_destroy_pgd(pgt);

arch/arm64/kvm/hyp/pgtable.c

245

static int _kvm_pgtable_walk(struct kvm_pgtable *pgt, struct kvm_pgtable_walk_data *data)

arch/arm64/kvm/hyp/pgtable.c

249

u64 limit = BIT(pgt->ia_bits);

arch/arm64/kvm/hyp/pgtable.c

254

if (!pgt->pgd)

arch/arm64/kvm/hyp/pgtable.c

257

for (idx = kvm_pgd_page_idx(pgt, data->addr); data->addr < data->end; ++idx) {

arch/arm64/kvm/hyp/pgtable.c

258

kvm_pteref_t pteref = &pgt->pgd[idx * PTRS_PER_PTE];

arch/arm64/kvm/hyp/pgtable.c

260

ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level);

arch/arm64/kvm/hyp/pgtable.c

268

int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/hyp/pgtable.c

283

r = _kvm_pgtable_walk(pgt, &walk_data);

arch/arm64/kvm/hyp/pgtable.c

305

int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/kvm/hyp/pgtable.c

316

ret = kvm_pgtable_walk(pgt, ALIGN_DOWN(addr, PAGE_SIZE),

arch/arm64/kvm/hyp/pgtable.c

451

int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,

arch/arm64/kvm/hyp/pgtable.c

468

ret = kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

514

u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/hyp/pgtable.c

523

if (!pgt->mm_ops->page_count)

arch/arm64/kvm/hyp/pgtable.c

526

kvm_pgtable_walk(pgt, addr, size, &walker);

arch/arm64/kvm/hyp/pgtable.c

530

int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,

arch/arm64/kvm/hyp/pgtable.c

540

pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL);

arch/arm64/kvm/hyp/pgtable.c

541

if (!pgt->pgd)

arch/arm64/kvm/hyp/pgtable.c

544

pgt->ia_bits = va_bits;

arch/arm64/kvm/hyp/pgtable.c

545

pgt->start_level = start_level;

arch/arm64/kvm/hyp/pgtable.c

546

pgt->mm_ops = mm_ops;

arch/arm64/kvm/hyp/pgtable.c

547

pgt->mmu = NULL;

arch/arm64/kvm/hyp/pgtable.c

548

pgt->force_pte_cb = NULL;

arch/arm64/kvm/hyp/pgtable.c

56

static u32 kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr)

arch/arm64/kvm/hyp/pgtable.c

569

void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt)

arch/arm64/kvm/hyp/pgtable.c

576

WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));

arch/arm64/kvm/hyp/pgtable.c

577

pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd));

arch/arm64/kvm/hyp/pgtable.c

578

pgt->pgd = NULL;

arch/arm64/kvm/hyp/pgtable.c

58

u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */

arch/arm64/kvm/hyp/pgtable.c

59

u64 mask = BIT(pgt->ia_bits) - 1;

arch/arm64/kvm/hyp/pgtable.c

66

struct kvm_pgtable pgt = {

arch/arm64/kvm/hyp/pgtable.c

670

#define KVM_S2_MEMATTR(pgt, attr) \

arch/arm64/kvm/hyp/pgtable.c

674

if ((pgt)->flags & KVM_PGTABLE_S2_AS_S1) \

arch/arm64/kvm/hyp/pgtable.c

707

static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot prot,

arch/arm64/kvm/hyp/pgtable.c

71

return kvm_pgd_page_idx(&pgt, -1ULL) + 1;

arch/arm64/kvm/hyp/pgtable.c

721

attr = KVM_S2_MEMATTR(pgt, DEVICE_nGnRE);

arch/arm64/kvm/hyp/pgtable.c

726

attr = KVM_S2_MEMATTR(pgt, NORMAL_NC);

arch/arm64/kvm/hyp/pgtable.c

729

attr = KVM_S2_MEMATTR(pgt, NORMAL);

arch/arm64/kvm/hyp/pgtable.c

879

static bool stage2_unmap_defer_tlb_flush(struct kvm_pgtable *pgt)

arch/arm64/kvm/hyp/pgtable.c

896

struct kvm_pgtable *pgt = ctx->arg;

arch/arm64/kvm/hyp/pgtable.c

909

} else if (!stage2_unmap_defer_tlb_flush(pgt)) {

arch/arm64/kvm/hyp/pgtable.c

918

static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte)

arch/arm64/kvm/hyp/pgtable.c

921

return kvm_pte_valid(pte) && memattr == KVM_S2_MEMATTR(pgt, NORMAL);

arch/arm64/kvm/hyp/pgtable.c

958

struct kvm_pgtable *pgt = data->mmu->pgt;

arch/arm64/kvm/hyp/pgtable.c

998

stage2_pte_cacheable(pgt, new))

arch/arm64/kvm/mmu.c

1007

mmu->pgd_phys = __pa(pgt->pgd);

arch/arm64/kvm/mmu.c

1015

kvm_stage2_destroy(pgt);

arch/arm64/kvm/mmu.c

1017

kfree(pgt);

arch/arm64/kvm/mmu.c

1100

struct kvm_pgtable *pgt = NULL;

arch/arm64/kvm/mmu.c

1103

pgt = mmu->pgt;

arch/arm64/kvm/mmu.c

1104

if (pgt) {

arch/arm64/kvm/mmu.c

1106

mmu->pgt = NULL;

arch/arm64/kvm/mmu.c

1115

if (pgt) {

arch/arm64/kvm/mmu.c

1116

kvm_stage2_destroy(pgt);

arch/arm64/kvm/mmu.c

1117

kfree(pgt);

arch/arm64/kvm/mmu.c

1184

struct kvm_pgtable *pgt = mmu->pgt;

arch/arm64/kvm/mmu.c

1202

ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, addr, PAGE_SIZE,

arch/arm64/kvm/mmu.c

124

struct kvm_pgtable *pgt;

arch/arm64/kvm/mmu.c

151

pgt = kvm->arch.mmu.pgt;

arch/arm64/kvm/mmu.c

152

if (!pgt)

arch/arm64/kvm/mmu.c

156

ret = KVM_PGT_FN(kvm_pgtable_stage2_split)(pgt, addr, next - addr, cache);

arch/arm64/kvm/mmu.c

1574

struct kvm_pgtable *pgt = vcpu->arch.hw_mmu->pgt;

arch/arm64/kvm/mmu.c

1628

ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, fault_ipa, PAGE_SIZE,

arch/arm64/kvm/mmu.c

1664

struct kvm_pgtable *pgt;

arch/arm64/kvm/mmu.c

1860

pgt = vcpu->arch.hw_mmu->pgt;

arch/arm64/kvm/mmu.c

1923

ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, fault_ipa, prot, flags);

arch/arm64/kvm/mmu.c

1925

ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, fault_ipa, vma_pagesize,

arch/arm64/kvm/mmu.c

1955

KVM_PGT_FN(kvm_pgtable_stage2_mkyoung)(mmu->pgt, fault_ipa, flags);

arch/arm64/kvm/mmu.c

2226

if (!kvm->arch.mmu.pgt)

arch/arm64/kvm/mmu.c

2241

if (!kvm->arch.mmu.pgt)

arch/arm64/kvm/mmu.c

2244

return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt,

arch/arm64/kvm/mmu.c

2257

if (!kvm->arch.mmu.pgt)

arch/arm64/kvm/mmu.c

2260

return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt,

arch/arm64/kvm/mmu.c

72

struct kvm_pgtable *pgt = mmu->pgt;

arch/arm64/kvm/mmu.c

73

if (!pgt)

arch/arm64/kvm/mmu.c

77

ret = fn(pgt, addr, next - addr);

arch/arm64/kvm/mmu.c

821

struct kvm_pgtable pgt = {

arch/arm64/kvm/mmu.c

825

ARM64_HW_PGTABLE_LEVELS(pgt.ia_bits) + 1),

arch/arm64/kvm/mmu.c

839

ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level);

arch/arm64/kvm/mmu.c

917

static void stage2_destroy_range(struct kvm_pgtable *pgt, phys_addr_t addr,

arch/arm64/kvm/mmu.c

924

KVM_PGT_FN(kvm_pgtable_stage2_destroy_range)(pgt, addr,

arch/arm64/kvm/mmu.c

931

static void kvm_stage2_destroy(struct kvm_pgtable *pgt)

arch/arm64/kvm/mmu.c

933

unsigned int ia_bits = VTCR_EL2_IPA(pgt->mmu->vtcr);

arch/arm64/kvm/mmu.c

935

stage2_destroy_range(pgt, 0, BIT(ia_bits));

arch/arm64/kvm/mmu.c

936

KVM_PGT_FN(kvm_pgtable_stage2_destroy_pgd)(pgt);

arch/arm64/kvm/mmu.c

957

struct kvm_pgtable *pgt;

arch/arm64/kvm/mmu.c

969

if (mmu->pgt != NULL) {

arch/arm64/kvm/mmu.c

981

pgt = kzalloc_obj(*pgt, GFP_KERNEL_ACCOUNT);

arch/arm64/kvm/mmu.c

982

if (!pgt)

arch/arm64/kvm/mmu.c

986

err = KVM_PGT_FN(kvm_pgtable_stage2_init)(pgt, mmu, &kvm_s2_mm_ops);

arch/arm64/kvm/mmu.c

990

mmu->pgt = pgt;

arch/arm64/kvm/nested.c

106

kvm->arch.nested_mmus[i].pgt->mmu = &kvm->arch.nested_mmus[i];

arch/arm64/kvm/nested.c

1146

kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));

arch/arm64/kvm/nested.c

1165

kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));

arch/arm64/kvm/nested.c

562

if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL))

arch/arm64/kvm/pkvm.c

316

int pkvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,

arch/arm64/kvm/pkvm.c

319

pgt->pkvm_mappings = RB_ROOT_CACHED;

arch/arm64/kvm/pkvm.c

320

pgt->mmu = mmu;

arch/arm64/kvm/pkvm.c

325

static int __pkvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 start, u64 end)

arch/arm64/kvm/pkvm.c

327

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

335

for_each_mapping_in_range_safe(pgt, start, end, mapping) {

arch/arm64/kvm/pkvm.c

340

pkvm_mapping_remove(mapping, &pgt->pkvm_mappings);

arch/arm64/kvm/pkvm.c

347

void pkvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,

arch/arm64/kvm/pkvm.c

350

__pkvm_pgtable_stage2_unmap(pgt, addr, addr + size);

arch/arm64/kvm/pkvm.c

353

void pkvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt)

arch/arm64/kvm/pkvm.c

356

WARN_ON_ONCE(!RB_EMPTY_ROOT(&pgt->pkvm_mappings.rb_root));

arch/arm64/kvm/pkvm.c

359

int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/pkvm.c

363

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

380

mapping = pkvm_mapping_iter_first(&pgt->pkvm_mappings, addr, addr + size - 1);

arch/arm64/kvm/pkvm.c

386

ret = __pkvm_pgtable_stage2_unmap(pgt, addr, addr + size);

arch/arm64/kvm/pkvm.c

400

pkvm_mapping_insert(mapping, &pgt->pkvm_mappings);

arch/arm64/kvm/pkvm.c

405

int pkvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/pkvm.c

407

lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(pgt->mmu)->mmu_lock);

arch/arm64/kvm/pkvm.c

409

return __pkvm_pgtable_stage2_unmap(pgt, addr, addr + size);

arch/arm64/kvm/pkvm.c

412

int pkvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/pkvm.c

414

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

420

for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) {

arch/arm64/kvm/pkvm.c

430

int pkvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size)

arch/arm64/kvm/pkvm.c

432

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

436

for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping)

arch/arm64/kvm/pkvm.c

443

bool pkvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr, u64 size, bool mkold)

arch/arm64/kvm/pkvm.c

445

struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);

arch/arm64/kvm/pkvm.c

451

for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping)

arch/arm64/kvm/pkvm.c

458

int pkvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, enum kvm_pgtable_prot prot,

arch/arm64/kvm/pkvm.c

464

void pkvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,

arch/arm64/kvm/pkvm.c

475

kvm_pte_t *pkvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, u64 phys, s8 level,

arch/arm64/kvm/pkvm.c

482

int pkvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/ptdump.c

119

struct kvm_pgtable *pgtable = mmu->pgt;

arch/arm64/kvm/ptdump.c

164

ret = kvm_pgtable_walk(mmu->pgt, 0, BIT(mmu->pgt->ia_bits), &walker);

arch/arm64/kvm/ptdump.c

239

pgtable = kvm->arch.mmu.pgt;

arch/powerpc/kvm/book3s_64_mmu_radix.c

1288

pgd_t *pgt;

arch/powerpc/kvm/book3s_64_mmu_radix.c

1326

pgt = NULL;

arch/powerpc/kvm/book3s_64_mmu_radix.c

1330

pgt = NULL;

arch/powerpc/kvm/book3s_64_mmu_radix.c

1340

if (!pgt) {

arch/powerpc/kvm/book3s_64_mmu_radix.c

1342

pgt = kvm->arch.pgtable;

arch/powerpc/kvm/book3s_64_mmu_radix.c

1349

pgt = nested->shadow_pgtable;

arch/powerpc/kvm/book3s_64_mmu_radix.c

1358

"pgdir: %lx\n", (unsigned long)pgt);

arch/powerpc/kvm/book3s_64_mmu_radix.c

1363

pgdp = pgt + pgd_index(gpa);

arch/x86/events/intel/uncore_nhmex.c

879

DEFINE_UNCORE_FORMAT_ATTR(pgt, pgt, "config1:0-31");

arch/x86/kernel/acpi/madt_wakeup.c

68

static void __init free_pgt_page(void *pgt, void *dummy)

arch/x86/kernel/acpi/madt_wakeup.c

70

return memblock_free(pgt, PAGE_SIZE);

drivers/accel/habanalabs/common/habanalabs.h

3944

u64 hl_mmu_hr_pte_phys_to_virt(struct hl_ctx *ctx, struct pgt_info *pgt, u64 phys_pte_addr,

drivers/accel/habanalabs/common/mmu/mmu.c

953

u64 hl_mmu_hr_pte_phys_to_virt(struct hl_ctx *ctx, struct pgt_info *pgt,

drivers/accel/habanalabs/common/mmu/mmu.c

959

return pgt->virt_addr + pte_offset;

drivers/firmware/efi/libstub/x86-5lvl.c

68

u64 *pgt = (void *)la57_toggle + PAGE_SIZE;

drivers/firmware/efi/libstub/x86-5lvl.c

81

new_cr3 = memset(pgt, 0, PAGE_SIZE);

drivers/firmware/efi/libstub/x86-5lvl.c

89

new_cr3 = memcpy(pgt, new_cr3, PAGE_SIZE);

drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c

52

struct nvkm_memory *pgt =

drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c

55

return nvkm_gpuobj_wrap(pgt, pgpuobj);

drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c

56

nvkm_kmap(pgt);

drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c

57

offset = nvkm_ro32(pgt, 8 + (offset >> 10));

drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c

59

nvkm_done(pgt);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

143

struct nvkm_vmm_pt *pgt = it->pt[it->lvl];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

144

struct nvkm_mmu_pt *pt = pgt->pt[type];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

154

pgt->pt[type] = NULL;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

155

if (!pgt->refs[!type]) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

158

if (pgt->sparse) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

191

if (!pgt->refs[!type])

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

192

nvkm_vmm_pt_del(&pgt);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

197

nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

211

pgt->pte[lpti].s.sptes -= pten;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

216

if (!pgt->refs[0])

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

221

if (pgt->pte[pteb].s.sptes) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

223

if (!(pgt->pte[ptei].s.sptes))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

235

pgt->pte[ptei].s.spte_valid = false;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

237

if (pgt->pte[ptei].s.sptes)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

239

pgt->pte[ptei].s.spte_valid = false;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

242

if (pgt->pte[pteb].s.sparse) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

244

pair->func->sparse(vmm, pgt->pt[0], pteb, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

245

} else if (!pgt->pte[pteb].s.lpte_valid) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

252

pair->func->invalid(vmm, pgt->pt[0], pteb, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

265

struct nvkm_vmm_pt *pgt = it->pt[0];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

270

dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

275

desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

280

pgt->refs[type] -= ptes;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

283

if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

284

nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

286

if (desc->type == LPT && (pgt->refs[0] || pgt->refs[1])) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

288

pgt->pte[lpti].s.lptes--;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

289

if (pgt->pte[lpti].s.lptes == 0)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

290

pgt->pte[lpti].s.lpte_valid = false;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

296

if (!pgt->refs[type]) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

30

struct nvkm_vmm_pt *pgt = *ppgt;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

308

nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

31

if (pgt) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

32

kvfree(pgt->pde);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

322

pgt->pte[lpti].s.sptes += pten;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

327

if (!pgt->refs[0])

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

33

kfree(pgt);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

332

if (pgt->pte[pteb].s.spte_valid) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

334

if (!pgt->pte[ptei].s.spte_valid)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

346

pgt->pte[ptei].s.spte_valid = true;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

347

pgt->pte[ptei].s.lpte_valid = false;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

349

if (pgt->pte[ptei].s.spte_valid)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

351

pgt->pte[ptei].s.spte_valid = true;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

352

pgt->pte[ptei].s.lpte_valid = false;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

355

if (pgt->pte[pteb].s.sparse) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

362

desc->func->sparse(vmm, pgt->pt[1], spti, sptc);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

365

pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

372

pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

382

struct nvkm_vmm_pt *pgt = it->pt[0];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

385

pgt->refs[type] += ptes;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

389

nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

393

pgt->pte[lpti].s.spte_valid = false;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

394

pgt->pte[lpti].s.lpte_valid = true;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

395

pgt->pte[lpti].s.lptes++;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

405

struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

409

pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

413

memset32(&pgt->pte[ptei].u, sparse.u, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

44

struct nvkm_vmm_pt *pgt;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

441

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

442

const bool zero = !pgt->sparse && !desc->func->invalid;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

452

pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

453

if (!pgt->pt[type]) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

462

pt = pgt->pt[type];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

464

if (desc->type == LPT && pgt->refs[1]) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

472

bool spte = !!pgt->pte[ptei].s.sptes;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

474

bool next = !!pgt->pte[ptei].s.sptes;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

480

if (pgt->sparse)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

484

memset32(&pgt->pte[pteb].u, 0x00, ptes);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

488

pgt->pte[pteb++].s.spte_valid = true;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

492

if (pgt->sparse) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

493

nvkm_vmm_sparse_ptes(desc, pgt, 0, pten);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

511

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

513

pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

514

if (!pgt) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

520

pgd->pde[pdei] = pgt;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

556

struct nvkm_vmm_pt *pgt = it.pt[it.lvl];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

56

if (!(pgt = kzalloc(sizeof(*pgt) + (sizeof(pgt->pte[0]) * lpte), GFP_KERNEL)))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

565

struct nvkm_vmm_pt *pgd = pgt;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

572

it.pt[it.lvl - 1] = pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

58

pgt->page = page ? page->shift : 0;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

580

if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

588

struct nvkm_mmu_pt *pt = pgt->pt[type];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

59

pgt->sparse = sparse;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

62

pgt->pde = kvzalloc_objs(*pgt->pde, pten);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

63

if (!pgt->pde) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

64

kfree(pgt);

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

69

return pgt;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c

108

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c

113

if ((pt = pgt->pt[0])) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c

127

if ((pt = pgt->pt[1])) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgh100.c

140

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgh100.c

144

if (pgt->pt[0] && !gh100_vmm_pde(pgt->pt[0], &data[0]))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgh100.c

146

if (pgt->pt[1] && !gh100_vmm_pde(pgt->pt[1], &data[1]))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgh100.c

181

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgh100.c

185

if (!gh100_vmm_pde(pgt->pt[0], &data))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c

256

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c

260

if (pgt->pt[0] && !gp100_vmm_pde(pgt->pt[0], &data[0]))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c

262

if (pgt->pt[1] && !gp100_vmm_pde(pgt->pt[1], &data[1]))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c

383

struct nvkm_vmm_pt *pgt = pgd->pde[pdei];

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c

387

if (!gp100_vmm_pde(pgt->pt[0], &data))

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c

106

nv50_vmm_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgt, u64 *pdata)

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c

110

if (pgt && (pt = pgt->pt[0])) {

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c

111

switch (pgt->page) {

drivers/gpu/drm/panthor/panthor_mmu.c

806

const struct io_pgtable *pgt = io_pgtable_ops_to_pgtable(vm->pgtbl_ops);

drivers/gpu/drm/panthor/panthor_mmu.c

807

u32 pg_shift = ffs(pgt->cfg.pgsize_bitmap) - 1;

drivers/net/ethernet/broadcom/bnx2x/bnx2x_self_test.c

395

NA, 1, 0, pgt,

drivers/net/ethernet/mellanox/mlxsw/spectrum.h

210

struct mlxsw_sp_pgt *pgt;

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

113

mlxsw_sp_pgt_entry_create(struct mlxsw_sp_pgt *pgt, u16 mid, u16 smpe)

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

123

ret = idr_replace(&pgt->pgt_idr, pgt_entry, mid);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

139

static void mlxsw_sp_pgt_entry_destroy(struct mlxsw_sp_pgt *pgt,

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

144

pgt_entry = idr_replace(&pgt->pgt_idr, NULL, pgt_entry->index);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

152

mlxsw_sp_pgt_entry_get(struct mlxsw_sp_pgt *pgt, u16 mid, u16 smpe)

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

156

pgt_entry = idr_find(&pgt->pgt_idr, mid);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

160

return mlxsw_sp_pgt_entry_create(pgt, mid, smpe);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

163

static void mlxsw_sp_pgt_entry_put(struct mlxsw_sp_pgt *pgt, u16 mid)

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

167

pgt_entry = idr_find(&pgt->pgt_idr, mid);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

172

mlxsw_sp_pgt_entry_destroy(pgt, pgt_entry);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

195

mlxsw_sp->pgt->smpe_index_valid,

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

252

mutex_lock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

254

pgt_entry = mlxsw_sp_pgt_entry_get(mlxsw_sp->pgt, mid, smpe);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

267

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

271

mlxsw_sp_pgt_entry_put(mlxsw_sp->pgt, mid);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

273

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

283

mutex_lock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

285

pgt_entry = idr_find(&mlxsw_sp->pgt->pgt_idr, mid);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

294

mlxsw_sp_pgt_entry_put(mlxsw_sp->pgt, mid);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

297

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

313

struct mlxsw_sp_pgt *pgt;

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

318

pgt = kzalloc_obj(*mlxsw_sp->pgt);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

319

if (!pgt)

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

32

mutex_lock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

322

idr_init(&pgt->pgt_idr);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

323

pgt->end_index = MLXSW_CORE_RES_GET(mlxsw_sp->core, PGT_SIZE);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

324

mutex_init(&pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

325

pgt->smpe_index_valid = mlxsw_sp->pgt_smpe_index_valid;

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

326

mlxsw_sp->pgt = pgt;

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

33

index = idr_alloc(&mlxsw_sp->pgt->pgt_idr, NULL, 0,

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

332

mutex_destroy(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

333

WARN_ON(!idr_is_empty(&mlxsw_sp->pgt->pgt_idr));

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

334

idr_destroy(&mlxsw_sp->pgt->pgt_idr);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

335

kfree(mlxsw_sp->pgt);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

34

mlxsw_sp->pgt->end_index, GFP_KERNEL);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

42

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

46

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

52

mutex_lock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

53

WARN_ON(idr_remove(&mlxsw_sp->pgt->pgt_idr, mid_base));

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

54

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

63

mutex_lock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

65

mid_base = idr_get_cursor(&mlxsw_sp->pgt->pgt_idr);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

67

err = idr_alloc_cyclic(&mlxsw_sp->pgt->pgt_idr, NULL,

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

73

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

79

idr_remove(&mlxsw_sp->pgt->pgt_idr, mid_base + i);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

80

mutex_unlock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

87

struct idr *pgt_idr = &mlxsw_sp->pgt->pgt_idr;

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

90

mutex_lock(&mlxsw_sp->pgt->lock);

drivers/net/ethernet/mellanox/mlxsw/spectrum_pgt.c

95

mutex_unlock(&mlxsw_sp->pgt->lock);