Symbol: kvm
arch/arm64/include/asm/kvm_asm.h
246
struct kvm;
arch/arm64/include/asm/kvm_emulate.h
661
struct kvm *kvm = vcpu->kvm;
arch/arm64/include/asm/kvm_emulate.h
672
if (kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP))
arch/arm64/include/asm/kvm_emulate.h
675
if (kvm_has_tcr2(kvm))
arch/arm64/include/asm/kvm_emulate.h
678
if (kvm_has_fpmr(kvm))
arch/arm64/include/asm/kvm_emulate.h
681
if (kvm_has_sctlr2(kvm))
arch/arm64/include/asm/kvm_emulate.h
684
if (kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64))
arch/arm64/include/asm/kvm_emulate.h
687
if (kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V))
arch/arm64/include/asm/kvm_host.h
1127
#define kvm_has_sve(kvm) (system_supports_sve() && \
arch/arm64/include/asm/kvm_host.h
1128
test_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &(kvm)->arch.flags))
arch/arm64/include/asm/kvm_host.h
1131
#define vcpu_has_sve(vcpu) kvm_has_sve(kern_hyp_va((vcpu)->kvm))
arch/arm64/include/asm/kvm_host.h
1133
#define vcpu_has_sve(vcpu) kvm_has_sve((vcpu)->kvm)
arch/arm64/include/asm/kvm_host.h
1250
void kvm_arm_halt_guest(struct kvm *kvm);
arch/arm64/include/asm/kvm_host.h
1251
void kvm_arm_resume_guest(struct kvm *kvm);
arch/arm64/include/asm/kvm_host.h
1311
void kvm_sys_regs_create_debugfs(struct kvm *kvm);
arch/arm64/include/asm/kvm_host.h
1367
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
arch/arm64/include/asm/kvm_host.h
1451
int kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm,
arch/arm64/include/asm/kvm_host.h
1453
int kvm_vm_ioctl_set_counter_offset(struct kvm *kvm,
arch/arm64/include/asm/kvm_host.h
1455
int kvm_vm_ioctl_get_reg_writable_masks(struct kvm *kvm,
arch/arm64/include/asm/kvm_host.h
1492
u32 kvm_get_pa_bits(struct kvm *kvm);
arch/arm64/include/asm/kvm_host.h
1495
struct kvm *kvm_arch_alloc_vm(void);
arch/arm64/include/asm/kvm_host.h
1501
#define kvm_vm_is_protected(kvm) (is_protected_kvm_enabled() && (kvm)->arch.pkvm.is_protected)
arch/arm64/include/asm/kvm_host.h
1503
#define vcpu_is_protected(vcpu) kvm_vm_is_protected((vcpu)->kvm)
arch/arm64/include/asm/kvm_host.h
1510
#define kvm_has_mte(kvm) \
arch/arm64/include/asm/kvm_host.h
1512
test_bit(KVM_ARCH_FLAG_MTE_ENABLED, &(kvm)->arch.flags))
arch/arm64/include/asm/kvm_host.h
1518
#define kvm_vm_has_ran_once(kvm) \
arch/arm64/include/asm/kvm_host.h
1519
(test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &(kvm)->arch.flags))
arch/arm64/include/asm/kvm_host.h
1527
#define vcpu_has_feature(v, f) __vcpu_has_feature(&(v)->kvm->arch, (f))
arch/arm64/include/asm/kvm_host.h
1562
#define kvm_read_vm_id_reg(kvm, reg) \
arch/arm64/include/asm/kvm_host.h
1563
({ u64 __val = *__vm_id_reg(&(kvm)->arch, reg); __val; })
arch/arm64/include/asm/kvm_host.h
1565
void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val);
arch/arm64/include/asm/kvm_host.h
1576
#define get_idreg_field_unsigned(kvm, id, fld) \
arch/arm64/include/asm/kvm_host.h
1578
u64 __val = kvm_read_vm_id_reg((kvm), SYS_##id); \
arch/arm64/include/asm/kvm_host.h
1582
#define get_idreg_field_signed(kvm, id, fld) \
arch/arm64/include/asm/kvm_host.h
1584
u64 __val = get_idreg_field_unsigned(kvm, id, fld); \
arch/arm64/include/asm/kvm_host.h
1588
#define get_idreg_field_enum(kvm, id, fld) \
arch/arm64/include/asm/kvm_host.h
1589
get_idreg_field_unsigned(kvm, id, fld)
arch/arm64/include/asm/kvm_host.h
1591
#define kvm_cmp_feat_signed(kvm, id, fld, op, limit) \
arch/arm64/include/asm/kvm_host.h
1592
(get_idreg_field_signed((kvm), id, fld) op __expand_field_sign_signed(id, fld, limit))
arch/arm64/include/asm/kvm_host.h
1594
#define kvm_cmp_feat_unsigned(kvm, id, fld, op, limit) \
arch/arm64/include/asm/kvm_host.h
1595
(get_idreg_field_unsigned((kvm), id, fld) op __expand_field_sign_unsigned(id, fld, limit))
arch/arm64/include/asm/kvm_host.h
1597
#define kvm_cmp_feat(kvm, id, fld, op, limit) \
arch/arm64/include/asm/kvm_host.h
1599
kvm_cmp_feat_signed(kvm, id, fld, op, limit) : \
arch/arm64/include/asm/kvm_host.h
1600
kvm_cmp_feat_unsigned(kvm, id, fld, op, limit))
arch/arm64/include/asm/kvm_host.h
1602
#define __kvm_has_feat(kvm, id, fld, limit) \
arch/arm64/include/asm/kvm_host.h
1603
kvm_cmp_feat(kvm, id, fld, >=, limit)
arch/arm64/include/asm/kvm_host.h
1605
#define kvm_has_feat(kvm, ...) __kvm_has_feat(kvm, __VA_ARGS__)
arch/arm64/include/asm/kvm_host.h
1607
#define __kvm_has_feat_enum(kvm, id, fld, val) \
arch/arm64/include/asm/kvm_host.h
1608
kvm_cmp_feat_unsigned(kvm, id, fld, ==, val)
arch/arm64/include/asm/kvm_host.h
1610
#define kvm_has_feat_enum(kvm, ...) __kvm_has_feat_enum(kvm, __VA_ARGS__)
arch/arm64/include/asm/kvm_host.h
1612
#define kvm_has_feat_range(kvm, id, fld, min, max) \
arch/arm64/include/asm/kvm_host.h
1613
(kvm_cmp_feat(kvm, id, fld, >=, min) && \
arch/arm64/include/asm/kvm_host.h
1614
kvm_cmp_feat(kvm, id, fld, <=, max))
arch/arm64/include/asm/kvm_host.h
1656
void compute_fgu(struct kvm *kvm, enum fgt_group_id fgt);
arch/arm64/include/asm/kvm_host.h
1657
struct resx get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg);
arch/arm64/include/asm/kvm_mmu.h
177
void stage2_unmap_vm(struct kvm *kvm);
arch/arm64/include/asm/kvm_mmu.h
178
int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long type);
arch/arm64/include/asm/kvm_mmu.h
179
void kvm_uninit_stage2_mmu(struct kvm *kvm);
arch/arm64/include/asm/kvm_mmu.h
181
int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
arch/arm64/include/asm/kvm_mmu.h
199
struct kvm;
arch/arm64/include/asm/kvm_mmu.h
276
static inline int kvm_read_guest_lock(struct kvm *kvm,
arch/arm64/include/asm/kvm_mmu.h
279
int srcu_idx = srcu_read_lock(&kvm->srcu);
arch/arm64/include/asm/kvm_mmu.h
280
int ret = kvm_read_guest(kvm, gpa, data, len);
arch/arm64/include/asm/kvm_mmu.h
282
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/arm64/include/asm/kvm_mmu.h
287
static inline int kvm_write_guest_lock(struct kvm *kvm, gpa_t gpa,
arch/arm64/include/asm/kvm_mmu.h
290
int srcu_idx = srcu_read_lock(&kvm->srcu);
arch/arm64/include/asm/kvm_mmu.h
291
int ret = kvm_write_guest(kvm, gpa, data, len);
arch/arm64/include/asm/kvm_mmu.h
293
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/arm64/include/asm/kvm_mmu.h
334
static inline struct kvm *kvm_s2_mmu_to_kvm(struct kvm_s2_mmu *mmu)
arch/arm64/include/asm/kvm_mmu.h
336
return container_of(mmu->arch, struct kvm, arch);
arch/arm64/include/asm/kvm_mmu.h
350
static inline bool kvm_is_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
arch/arm64/include/asm/kvm_mmu.h
356
return &kvm->arch.mmu != mmu;
arch/arm64/include/asm/kvm_mmu.h
359
static inline void kvm_fault_lock(struct kvm *kvm)
arch/arm64/include/asm/kvm_mmu.h
362
write_lock(&kvm->mmu_lock);
arch/arm64/include/asm/kvm_mmu.h
364
read_lock(&kvm->mmu_lock);
arch/arm64/include/asm/kvm_mmu.h
367
static inline void kvm_fault_unlock(struct kvm *kvm)
arch/arm64/include/asm/kvm_mmu.h
370
write_unlock(&kvm->mmu_lock);
arch/arm64/include/asm/kvm_mmu.h
372
read_unlock(&kvm->mmu_lock);
arch/arm64/include/asm/kvm_mmu.h
394
void kvm_s2_ptdump_create_debugfs(struct kvm *kvm);
arch/arm64/include/asm/kvm_mmu.h
398
static inline void kvm_s2_ptdump_create_debugfs(struct kvm *kvm) {}
arch/arm64/include/asm/kvm_nested.h
124
static inline bool kvm_has_xnx(struct kvm *kvm)
arch/arm64/include/asm/kvm_nested.h
127
kvm_has_feat(kvm, ID_AA64MMFR1_EL1, XNX, IMP);
arch/arm64/include/asm/kvm_nested.h
130
static inline bool kvm_s2_trans_exec_el0(struct kvm *kvm, struct kvm_s2_trans *trans)
arch/arm64/include/asm/kvm_nested.h
134
if (!kvm_has_xnx(kvm))
arch/arm64/include/asm/kvm_nested.h
146
static inline bool kvm_s2_trans_exec_el1(struct kvm *kvm, struct kvm_s2_trans *trans)
arch/arm64/include/asm/kvm_nested.h
150
if (!kvm_has_xnx(kvm))
arch/arm64/include/asm/kvm_nested.h
167
extern void kvm_nested_s2_wp(struct kvm *kvm);
arch/arm64/include/asm/kvm_nested.h
168
extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block);
arch/arm64/include/asm/kvm_nested.h
169
extern void kvm_nested_s2_flush(struct kvm *kvm);
arch/arm64/include/asm/kvm_nested.h
175
struct kvm *kvm = vpcu->kvm;
arch/arm64/include/asm/kvm_nested.h
184
kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))
arch/arm64/include/asm/kvm_nested.h
188
!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
arch/arm64/include/asm/kvm_nested.h
193
!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
arch/arm64/include/asm/kvm_nested.h
201
struct kvm *kvm = vpcu->kvm;
arch/arm64/include/asm/kvm_nested.h
210
kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))
arch/arm64/include/asm/kvm_nested.h
217
!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
arch/arm64/include/asm/kvm_nested.h
222
!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
arch/arm64/include/asm/kvm_nested.h
229
u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val);
arch/arm64/include/asm/kvm_nested.h
418
int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new);
arch/arm64/include/asm/kvm_nested.h
69
extern void kvm_init_nested(struct kvm *kvm);
arch/arm64/include/asm/kvm_nested.h
76
extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
arch/arm64/include/asm/kvm_pkvm.h
20
int pkvm_init_host_vm(struct kvm *kvm, unsigned long type);
arch/arm64/include/asm/kvm_pkvm.h
21
int pkvm_create_hyp_vm(struct kvm *kvm);
arch/arm64/include/asm/kvm_pkvm.h
22
bool pkvm_hyp_vm_is_created(struct kvm *kvm);
arch/arm64/include/asm/kvm_pkvm.h
23
void pkvm_destroy_hyp_vm(struct kvm *kvm);
arch/arm64/include/asm/kvm_pkvm.h
32
static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext)
arch/arm64/include/asm/kvm_pkvm.h
49
return !kvm || !kvm_vm_is_protected(kvm);
arch/arm64/include/asm/kvm_pkvm.h
59
static inline bool kvm_pkvm_ioctl_allowed(struct kvm *kvm, unsigned int ioctl)
arch/arm64/include/asm/kvm_pkvm.h
69
return kvm_pkvm_ext_allowed(kvm, ext);
arch/arm64/kvm/arch_timer.c
1019
if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
arch/arm64/kvm/arch_timer.c
1048
offs->vm_offset = &vcpu->kvm->arch.timer_data.poffset;
arch/arm64/kvm/arch_timer.c
1056
if (irqchip_in_kernel(vcpu->kvm)) {
arch/arm64/kvm/arch_timer.c
1072
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/arch_timer.c
1076
if (!kvm_vm_is_protected(vcpu->kvm)) {
arch/arm64/kvm/arch_timer.c
1078
ctxt->offset.vm_offset = &kvm->arch.timer_data.voffset;
arch/arm64/kvm/arch_timer.c
1080
ctxt->offset.vm_offset = &kvm->arch.timer_data.poffset;
arch/arm64/kvm/arch_timer.c
1108
!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) {
arch/arm64/kvm/arch_timer.c
1121
void kvm_timer_init_vm(struct kvm *kvm)
arch/arm64/kvm/arch_timer.c
1129
kvm->arch.timer_data.ppi[i] = get_vgic_ppi(kvm, default_ppi[i]);
arch/arm64/kvm/arch_timer.c
1288
vgic_is_v3(kvm_get_running_vcpu()->kvm));
arch/arm64/kvm/arch_timer.c
1524
mutex_lock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/arch_timer.c
1536
if (vgic_is_v5(vcpu->kvm) && irq != get_vgic_ppi(vcpu->kvm, default_ppi[i]))
arch/arm64/kvm/arch_timer.c
1552
set_bit(KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE, &vcpu->kvm->arch.flags);
arch/arm64/kvm/arch_timer.c
1554
mutex_unlock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/arch_timer.c
1591
if (!irqchip_in_kernel(vcpu->kvm))
arch/arm64/kvm/arch_timer.c
1605
ops = vgic_is_v5(vcpu->kvm) ? &arch_timer_irq_ops_vgic_v5 :
arch/arm64/kvm/arch_timer.c
1641
if (!irqchip_in_kernel(vcpu->kvm))
arch/arm64/kvm/arch_timer.c
1647
if (!(irq_is_ppi(vcpu->kvm, irq)))
arch/arm64/kvm/arch_timer.c
1650
guard(mutex)(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/arch_timer.c
1653
&vcpu->kvm->arch.flags)) {
arch/arm64/kvm/arch_timer.c
1679
vcpu->kvm->arch.timer_data.ppi[idx] = irq;
arch/arm64/kvm/arch_timer.c
1724
int kvm_vm_ioctl_set_counter_offset(struct kvm *kvm,
arch/arm64/kvm/arch_timer.c
1732
if (kvm_vm_is_protected(kvm))
arch/arm64/kvm/arch_timer.c
1735
mutex_lock(&kvm->lock);
arch/arm64/kvm/arch_timer.c
1737
if (!kvm_trylock_all_vcpus(kvm)) {
arch/arm64/kvm/arch_timer.c
1738
set_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &kvm->arch.flags);
arch/arm64/kvm/arch_timer.c
1746
kvm->arch.timer_data.voffset = offset->counter_offset;
arch/arm64/kvm/arch_timer.c
1747
kvm->arch.timer_data.poffset = offset->counter_offset;
arch/arm64/kvm/arch_timer.c
1749
kvm_unlock_all_vcpus(kvm);
arch/arm64/kvm/arch_timer.c
1754
mutex_unlock(&kvm->lock);
arch/arm64/kvm/arch_timer.c
194
static inline bool userspace_irqchip(struct kvm *kvm)
arch/arm64/kvm/arch_timer.c
196
return unlikely(!irqchip_in_kernel(kvm));
arch/arm64/kvm/arch_timer.c
235
if (userspace_irqchip(vcpu->kvm) &&
arch/arm64/kvm/arch_timer.c
465
if (userspace_irqchip(vcpu->kvm))
arch/arm64/kvm/arch_timer.c
469
if (vgic_is_v5(vcpu->kvm)) {
arch/arm64/kvm/arch_timer.c
479
kvm_vgic_inject_irq(vcpu->kvm, vcpu,
arch/arm64/kvm/arch_timer.c
700
if (irqchip_in_kernel(vcpu->kvm))
arch/arm64/kvm/arch_timer.c
704
phys_active |= vgic_is_v5(vcpu->kvm);
arch/arm64/kvm/arch_timer.c
752
if (!irqchip_in_kernel(vcpu->kvm))
arch/arm64/kvm/arch_timer.c
895
if (vcpu_has_nv(vcpu) && !vgic_is_v5(vcpu->kvm))
arch/arm64/kvm/arch_timer.c
949
if (vgic_is_v5(vcpu->kvm)) {
arch/arm64/kvm/arm.c
1000
return vgic_initialized(kvm);
arch/arm64/kvm/arm.c
1003
void kvm_arm_halt_guest(struct kvm *kvm)
arch/arm64/kvm/arm.c
1008
kvm_for_each_vcpu(i, vcpu, kvm)
arch/arm64/kvm/arm.c
1010
kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
arch/arm64/kvm/arm.c
1013
void kvm_arm_resume_guest(struct kvm *kvm)
arch/arm64/kvm/arm.c
1018
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/arm64/kvm/arm.c
1216
if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
arch/arm64/kvm/arm.c
1329
if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
arch/arm64/kvm/arm.c
135
int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
arch/arm64/kvm/arm.c
1373
if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
arch/arm64/kvm/arm.c
143
if (is_protected_kvm_enabled() && !kvm_pkvm_ext_allowed(kvm, cap->cap))
arch/arm64/kvm/arm.c
1432
if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
arch/arm64/kvm/arm.c
1487
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
arch/arm64/kvm/arm.c
150
&kvm->arch.flags);
arch/arm64/kvm/arm.c
1505
if (irqchip_in_kernel(kvm))
arch/arm64/kvm/arm.c
1508
vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
arch/arm64/kvm/arm.c
1517
if (irqchip_in_kernel(kvm)) {
arch/arm64/kvm/arm.c
1518
int ret = vgic_lazy_init(kvm);
arch/arm64/kvm/arm.c
1525
vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
arch/arm64/kvm/arm.c
1529
if (vgic_is_v5(kvm)) {
arch/arm64/kvm/arm.c
1537
mask = kvm->arch.vgic.gicv5_vm.userspace_ppis;
arch/arm64/kvm/arm.c
1548
return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL);
arch/arm64/kvm/arm.c
1550
if (irqchip_in_kernel(kvm)) {
arch/arm64/kvm/arm.c
1551
int ret = vgic_lazy_init(kvm);
arch/arm64/kvm/arm.c
1558
if (vgic_is_v5(kvm)) {
arch/arm64/kvm/arm.c
156
guard(mutex)(&kvm->lock);
arch/arm64/kvm/arm.c
1566
return kvm_vgic_inject_irq(kvm, NULL, irq_num, level, NULL);
arch/arm64/kvm/arm.c
157
if (!system_supports_mte() || kvm->created_vcpus)
arch/arm64/kvm/arm.c
161
guard(mutex)(&kvm->slots_lock);
arch/arm64/kvm/arm.c
162
kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm)) {
arch/arm64/kvm/arm.c
1625
if (kvm_has_mte(vcpu->kvm))
arch/arm64/kvm/arm.c
1640
return !bitmap_equal(vcpu->kvm->arch.vcpu_features, &features,
arch/arm64/kvm/arm.c
1646
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/arm.c
1653
if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu)
arch/arm64/kvm/arm.c
1654
ret = kvm_arm_set_default_pmu(kvm);
arch/arm64/kvm/arm.c
1667
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/arm.c
1670
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/arm.c
1672
if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) &&
arch/arm64/kvm/arm.c
1676
bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES);
arch/arm64/kvm/arm.c
1685
set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags);
arch/arm64/kvm/arm.c
1689
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/arm.c
169
set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags);
arch/arm64/kvm/arm.c
1747
stage2_unmap_vm(vcpu->kvm);
arch/arm64/kvm/arm.c
175
set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags);
arch/arm64/kvm/arm.c
178
mutex_lock(&kvm->slots_lock);
arch/arm64/kvm/arm.c
183
if (kvm_are_all_memslots_empty(kvm)) {
arch/arm64/kvm/arm.c
188
kvm->arch.mmu.split_page_chunk_size = new_cap;
arch/arm64/kvm/arm.c
191
mutex_unlock(&kvm->slots_lock);
arch/arm64/kvm/arm.c
194
mutex_lock(&kvm->lock);
arch/arm64/kvm/arm.c
195
if (!kvm->created_vcpus) {
arch/arm64/kvm/arm.c
197
set_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags);
arch/arm64/kvm/arm.c
1978
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
arch/arm64/kvm/arm.c
1983
static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
arch/arm64/kvm/arm.c
199
mutex_unlock(&kvm->lock);
arch/arm64/kvm/arm.c
1990
return kvm_set_legacy_vgic_v2_addr(kvm, dev_addr);
arch/arm64/kvm/arm.c
1996
static int kvm_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
arch/arm64/kvm/arm.c
2000
return kvm_vm_smccc_has_attr(kvm, attr);
arch/arm64/kvm/arm.c
2006
static int kvm_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
arch/arm64/kvm/arm.c
2010
return kvm_vm_smccc_set_attr(kvm, attr);
arch/arm64/kvm/arm.c
2018
struct kvm *kvm = filp->private_data;
arch/arm64/kvm/arm.c
2022
if (is_protected_kvm_enabled() && !kvm_pkvm_ioctl_allowed(kvm, ioctl))
arch/arm64/kvm/arm.c
203
set_bit(KVM_ARCH_FLAG_EXIT_SEA, &kvm->arch.flags);
arch/arm64/kvm/arm.c
2030
mutex_lock(&kvm->lock);
arch/arm64/kvm/arm.c
2031
ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
arch/arm64/kvm/arm.c
2032
mutex_unlock(&kvm->lock);
arch/arm64/kvm/arm.c
2040
return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
arch/arm64/kvm/arm.c
2057
return kvm_vm_ioctl_mte_copy_tags(kvm, &copy_tags);
arch/arm64/kvm/arm.c
2064
return kvm_vm_ioctl_set_counter_offset(kvm, &offset);
arch/arm64/kvm/arm.c
2070
return kvm_vm_has_attr(kvm, &attr);
arch/arm64/kvm/arm.c
2076
return kvm_vm_set_attr(kvm, &attr);
arch/arm64/kvm/arm.c
2083
return kvm_vm_ioctl_get_reg_writable_masks(kvm, &range);
arch/arm64/kvm/arm.c
222
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
arch/arm64/kvm/arm.c
229
mutex_init(&kvm->arch.config_lock);
arch/arm64/kvm/arm.c
233
mutex_lock(&kvm->lock);
arch/arm64/kvm/arm.c
234
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/arm.c
235
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/arm.c
236
mutex_unlock(&kvm->lock);
arch/arm64/kvm/arm.c
239
kvm_init_nested(kvm);
arch/arm64/kvm/arm.c
241
ret = kvm_share_hyp(kvm, kvm + 1);
arch/arm64/kvm/arm.c
245
if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL_ACCOUNT)) {
arch/arm64/kvm/arm.c
249
cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask);
arch/arm64/kvm/arm.c
251
ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type);
arch/arm64/kvm/arm.c
260
ret = pkvm_init_host_vm(kvm, type);
arch/arm64/kvm/arm.c
268
kvm_vgic_early_init(kvm);
arch/arm64/kvm/arm.c
270
kvm_timer_init_vm(kvm);
arch/arm64/kvm/arm.c
273
kvm->max_vcpus = kvm_arm_default_max_vcpus();
arch/arm64/kvm/arm.c
275
kvm_arm_init_hypercalls(kvm);
arch/arm64/kvm/arm.c
277
bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES);
arch/arm64/kvm/arm.c
282
kvm_uninit_stage2_mmu(kvm);
arch/arm64/kvm/arm.c
284
free_cpumask_var(kvm->arch.supported_cpus);
arch/arm64/kvm/arm.c
286
kvm_unshare_hyp(kvm, kvm + 1);
arch/arm64/kvm/arm.c
2909
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
arch/arm64/kvm/arm.c
2918
data = rcu_dereference(kvm->arch.mpidr_data);
arch/arm64/kvm/arm.c
2923
vcpu = kvm_get_vcpu(kvm, data->cmpidr_to_idx[idx]);
arch/arm64/kvm/arm.c
2933
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/arm64/kvm/arm.c
2940
bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
arch/arm64/kvm/arm.c
2942
return irqchip_in_kernel(kvm);
arch/arm64/kvm/arm.c
295
void kvm_arch_create_vm_debugfs(struct kvm *kvm)
arch/arm64/kvm/arm.c
2959
return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
arch/arm64/kvm/arm.c
297
kvm_sys_regs_create_debugfs(kvm);
arch/arm64/kvm/arm.c
2973
kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq);
arch/arm64/kvm/arm.c
298
kvm_s2_ptdump_create_debugfs(kvm);
arch/arm64/kvm/arm.c
2992
return kvm_vgic_v4_unset_forwarding(irqfd->kvm, irqfd->producer->irq);
arch/arm64/kvm/arm.c
3000
kvm_arm_halt_guest(irqfd->kvm);
arch/arm64/kvm/arm.c
3008
kvm_arm_resume_guest(irqfd->kvm);
arch/arm64/kvm/arm.c
301
static void kvm_destroy_mpidr_data(struct kvm *kvm)
arch/arm64/kvm/arm.c
305
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/arm.c
307
data = rcu_dereference_protected(kvm->arch.mpidr_data,
arch/arm64/kvm/arm.c
308
lockdep_is_held(&kvm->arch.config_lock));
arch/arm64/kvm/arm.c
310
rcu_assign_pointer(kvm->arch.mpidr_data, NULL);
arch/arm64/kvm/arm.c
315
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/arm.c
322
void kvm_arch_destroy_vm(struct kvm *kvm)
arch/arm64/kvm/arm.c
324
bitmap_free(kvm->arch.pmu_filter);
arch/arm64/kvm/arm.c
325
free_cpumask_var(kvm->arch.supported_cpus);
arch/arm64/kvm/arm.c
327
kvm_vgic_destroy(kvm);
arch/arm64/kvm/arm.c
330
pkvm_destroy_hyp_vm(kvm);
arch/arm64/kvm/arm.c
332
kvm_uninit_stage2_mmu(kvm);
arch/arm64/kvm/arm.c
333
kvm_destroy_mpidr_data(kvm);
arch/arm64/kvm/arm.c
335
kfree(kvm->arch.sysreg_masks);
arch/arm64/kvm/arm.c
336
kvm_destroy_vcpus(kvm);
arch/arm64/kvm/arm.c
338
kvm_unshare_hyp(kvm, kvm + 1);
arch/arm64/kvm/arm.c
340
kvm_arm_teardown_hypercalls(kvm);
arch/arm64/kvm/arm.c
377
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
arch/arm64/kvm/arm.c
381
if (is_protected_kvm_enabled() && !kvm_pkvm_ext_allowed(kvm, ext))
arch/arm64/kvm/arm.c
428
if (kvm)
arch/arm64/kvm/arm.c
429
r = kvm->max_vcpus;
arch/arm64/kvm/arm.c
434
if (!kvm)
arch/arm64/kvm/arm.c
437
r = kvm->arch.vgic.msis_require_devid;
arch/arm64/kvm/arm.c
484
if (kvm)
arch/arm64/kvm/arm.c
485
r = kvm->arch.mmu.split_page_chunk_size;
arch/arm64/kvm/arm.c
496
if (!kvm)
arch/arm64/kvm/arm.c
515
struct kvm *kvm_arch_alloc_vm(void)
arch/arm64/kvm/arm.c
517
size_t sz = sizeof(struct kvm);
arch/arm64/kvm/arm.c
525
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
arch/arm64/kvm/arm.c
527
if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
arch/arm64/kvm/arm.c
530
if (id >= kvm->max_vcpus)
arch/arm64/kvm/arm.c
545
mutex_lock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/arm.c
546
mutex_unlock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/arm.c
562
vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
arch/arm64/kvm/arm.c
569
kvm_destroy_mpidr_data(vcpu->kvm);
arch/arm64/kvm/arm.c
647
if (vgic_is_v5(vcpu->kvm))
arch/arm64/kvm/arm.c
651
vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 &&
arch/arm64/kvm/arm.c
653
vcpu->kvm->arch.vgic.nassgireq);
arch/arm64/kvm/arm.c
734
vcpu->kvm->arch.pkvm.handle,
arch/arm64/kvm/arm.c
740
if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus))
arch/arm64/kvm/arm.c
844
irq_lines |= (!irqchip_in_kernel(v->kvm) &&
arch/arm64/kvm/arm.c
864
static void kvm_init_mpidr_data(struct kvm *kvm)
arch/arm64/kvm/arm.c
871
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/arm.c
873
if (rcu_access_pointer(kvm->arch.mpidr_data) ||
arch/arm64/kvm/arm.c
874
atomic_read(&kvm->online_vcpus) == 1)
arch/arm64/kvm/arm.c
877
kvm_for_each_vcpu(c, vcpu, kvm) {
arch/arm64/kvm/arm.c
904
kvm_for_each_vcpu(c, vcpu, kvm) {
arch/arm64/kvm/arm.c
911
rcu_assign_pointer(kvm->arch.mpidr_data, data);
arch/arm64/kvm/arm.c
913
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/arm.c
923
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/arm.c
935
kvm_init_mpidr_data(kvm);
arch/arm64/kvm/arm.c
937
if (likely(irqchip_in_kernel(kvm))) {
arch/arm64/kvm/arm.c
942
ret = kvm_vgic_map_resources(kvm);
arch/arm64/kvm/arm.c
977
ret = vgic_v5_finalize_ppi_state(kvm);
arch/arm64/kvm/arm.c
982
ret = pkvm_create_hyp_vm(kvm);
arch/arm64/kvm/arm.c
991
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/arm.c
992
set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags);
arch/arm64/kvm/arm.c
993
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/arm.c
998
bool kvm_arch_intc_initialized(struct kvm *kvm)
arch/arm64/kvm/at.c
1004
if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, PAN, PAN3))
arch/arm64/kvm/at.c
108
if (!kvm_has_s1pie(vcpu->kvm))
arch/arm64/kvm/at.c
119
if (!kvm_has_s1poe(vcpu->kvm)) {
arch/arm64/kvm/at.c
1341
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/arm64/kvm/at.c
1345
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/arm64/kvm/at.c
1412
guard(write_lock_irqsave)(&vcpu->kvm->mmu_lock);
arch/arm64/kvm/at.c
1442
if (kvm_has_tcr2(vcpu->kvm)) {
arch/arm64/kvm/at.c
1444
if (kvm_has_s1pie(vcpu->kvm)) {
arch/arm64/kvm/at.c
1448
if (kvm_has_s1poe(vcpu->kvm)) {
arch/arm64/kvm/at.c
1548
scoped_guard(write_lock_irqsave, &vcpu->kvm->mmu_lock) {
arch/arm64/kvm/at.c
1646
scoped_guard(srcu, &vcpu->kvm->srcu)
arch/arm64/kvm/at.c
1743
scoped_guard(srcu, &vcpu->kvm->srcu)
arch/arm64/kvm/at.c
1841
int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new)
arch/arm64/kvm/at.c
1851
lockdep_assert(srcu_read_lock_held(&kvm->srcu));
arch/arm64/kvm/at.c
1855
slot = gfn_to_memslot(kvm, gfn);
arch/arm64/kvm/at.c
1873
mark_page_dirty_in_slot(kvm, slot, gfn);
arch/arm64/kvm/at.c
199
static unsigned int tcr_tg_pgshift(struct kvm *kvm, u64 tcr, bool upper_range)
arch/arm64/kvm/at.c
201
u64 mmfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1);
arch/arm64/kvm/at.c
266
wi->pgshift = tcr_tg_pgshift(vcpu->kvm, tcr, upper_range);
arch/arm64/kvm/at.c
323
if (va >= BIT(kvm_get_pa_bits(vcpu->kvm)))
arch/arm64/kvm/at.c
332
wi->hpd = kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, HPDS, IMP);
arch/arm64/kvm/at.c
34
if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, PARANGE, 52))
arch/arm64/kvm/at.c
348
if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR2_EL1, ST, 48_47)) {
arch/arm64/kvm/at.c
363
lva = kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, VARange, 52);
arch/arm64/kvm/at.c
381
if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, E0PD, IMP) &&
arch/arm64/kvm/at.c
40
if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, TGRAN16, 52_BIT))
arch/arm64/kvm/at.c
411
wi->ha = kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, HAFDBS, AF);
arch/arm64/kvm/at.c
438
r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val));
arch/arm64/kvm/at.c
44
if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, TGRAN4, 52_BIT))
arch/arm64/kvm/at.c
461
return __kvm_at_swap_desc(vcpu->kvm, pa, old, new);
arch/arm64/kvm/at.c
559
bool lpa = kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, PARANGE, 52);
arch/arm64/kvm/at.c
876
if (!kvm_has_feat(vcpu->kvm, ID_AA64PFR2_EL1, MTEPERM, IMP))
arch/arm64/kvm/config.c
1398
static bool idreg_feat_match(struct kvm *kvm, const struct reg_bits_to_feat_map *map)
arch/arm64/kvm/config.c
1400
u64 regval = kvm->arch.id_regs[map->regidx];
arch/arm64/kvm/config.c
1412
static struct resx compute_resx_bits(struct kvm *kvm,
arch/arm64/kvm/config.c
1418
bool e2h0 = kvm_has_feat(kvm, FEAT_E2H0);
arch/arm64/kvm/config.c
1433
match = map[i].match(kvm);
arch/arm64/kvm/config.c
1435
match = idreg_feat_match(kvm, &map[i]);
arch/arm64/kvm/config.c
1455
static struct resx compute_reg_resx_bits(struct kvm *kvm,
arch/arm64/kvm/config.c
1462
resx = compute_resx_bits(kvm, r->bit_feat_map, r->bit_feat_map_sz,
arch/arm64/kvm/config.c
1476
resx.res0 |= compute_resx_bits(kvm, &r->feat_map, 1, require, exclude).res0;
arch/arm64/kvm/config.c
1482
static u64 compute_fgu_bits(struct kvm *kvm, const struct reg_feat_map_desc *r)
arch/arm64/kvm/config.c
1492
resx = compute_resx_bits(kvm, r->bit_feat_map, r->bit_feat_map_sz,
arch/arm64/kvm/config.c
1498
void compute_fgu(struct kvm *kvm, enum fgt_group_id fgt)
arch/arm64/kvm/config.c
1504
val |= compute_fgu_bits(kvm, &hfgrtr_desc);
arch/arm64/kvm/config.c
1505
val |= compute_fgu_bits(kvm, &hfgwtr_desc);
arch/arm64/kvm/config.c
1508
val |= compute_fgu_bits(kvm, &hfgitr_desc);
arch/arm64/kvm/config.c
1511
val |= compute_fgu_bits(kvm, &hdfgrtr_desc);
arch/arm64/kvm/config.c
1512
val |= compute_fgu_bits(kvm, &hdfgwtr_desc);
arch/arm64/kvm/config.c
1515
val |= compute_fgu_bits(kvm, &hafgrtr_desc);
arch/arm64/kvm/config.c
1518
val |= compute_fgu_bits(kvm, &hfgrtr2_desc);
arch/arm64/kvm/config.c
1519
val |= compute_fgu_bits(kvm, &hfgwtr2_desc);
arch/arm64/kvm/config.c
1522
val |= compute_fgu_bits(kvm, &hfgitr2_desc);
arch/arm64/kvm/config.c
1525
val |= compute_fgu_bits(kvm, &hdfgrtr2_desc);
arch/arm64/kvm/config.c
1526
val |= compute_fgu_bits(kvm, &hdfgwtr2_desc);
arch/arm64/kvm/config.c
1529
val |= compute_fgu_bits(kvm, &ich_hfgrtr_desc);
arch/arm64/kvm/config.c
1530
val |= compute_fgu_bits(kvm, &ich_hfgwtr_desc);
arch/arm64/kvm/config.c
1533
val |= compute_fgu_bits(kvm, &ich_hfgitr_desc);
arch/arm64/kvm/config.c
1539
kvm->arch.fgu[fgt] = val;
arch/arm64/kvm/config.c
1542
struct resx get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg)
arch/arm64/kvm/config.c
1548
resx = compute_reg_resx_bits(kvm, &hfgrtr_desc, 0, 0);
arch/arm64/kvm/config.c
1551
resx = compute_reg_resx_bits(kvm, &hfgwtr_desc, 0, 0);
arch/arm64/kvm/config.c
1554
resx = compute_reg_resx_bits(kvm, &hfgitr_desc, 0, 0);
arch/arm64/kvm/config.c
1557
resx = compute_reg_resx_bits(kvm, &hdfgrtr_desc, 0, 0);
arch/arm64/kvm/config.c
1560
resx = compute_reg_resx_bits(kvm, &hdfgwtr_desc, 0, 0);
arch/arm64/kvm/config.c
1563
resx = compute_reg_resx_bits(kvm, &hafgrtr_desc, 0, 0);
arch/arm64/kvm/config.c
1566
resx = compute_reg_resx_bits(kvm, &hfgrtr2_desc, 0, 0);
arch/arm64/kvm/config.c
1569
resx = compute_reg_resx_bits(kvm, &hfgwtr2_desc, 0, 0);
arch/arm64/kvm/config.c
1572
resx = compute_reg_resx_bits(kvm, &hfgitr2_desc, 0, 0);
arch/arm64/kvm/config.c
1575
resx = compute_reg_resx_bits(kvm, &hdfgrtr2_desc, 0, 0);
arch/arm64/kvm/config.c
1578
resx = compute_reg_resx_bits(kvm, &hdfgwtr2_desc, 0, 0);
arch/arm64/kvm/config.c
1581
resx = compute_reg_resx_bits(kvm, &hcrx_desc, 0, 0);
arch/arm64/kvm/config.c
1585
resx = compute_reg_resx_bits(kvm, &hcr_desc, 0, 0);
arch/arm64/kvm/config.c
1589
resx = compute_reg_resx_bits(kvm, &sctlr2_desc, 0, 0);
arch/arm64/kvm/config.c
1592
resx = compute_reg_resx_bits(kvm, &tcr2_el2_desc, 0, 0);
arch/arm64/kvm/config.c
1595
resx = compute_reg_resx_bits(kvm, &sctlr_el1_desc, 0, 0);
arch/arm64/kvm/config.c
1598
resx = compute_reg_resx_bits(kvm, &sctlr_el2_desc, 0, 0);
arch/arm64/kvm/config.c
1601
resx = compute_reg_resx_bits(kvm, &mdcr_el2_desc, 0, 0);
arch/arm64/kvm/config.c
1604
resx = compute_reg_resx_bits(kvm, &vtcr_el2_desc, 0, 0);
arch/arm64/kvm/config.c
1607
resx = compute_reg_resx_bits(kvm, &ich_hfgrtr_desc, 0, 0);
arch/arm64/kvm/config.c
1610
resx = compute_reg_resx_bits(kvm, &ich_hfgwtr_desc, 0, 0);
arch/arm64/kvm/config.c
1613
resx = compute_reg_resx_bits(kvm, &ich_hfgitr_desc, 0, 0);
arch/arm64/kvm/config.c
1662
u64 fgu = vcpu->kvm->arch.fgu[__fgt_reg_to_group_id(reg)];
arch/arm64/kvm/config.c
229
static bool not_feat_aa64el3(struct kvm *kvm)
arch/arm64/kvm/config.c
231
return !kvm_has_feat(kvm, FEAT_AA64EL3);
arch/arm64/kvm/config.c
234
static bool feat_nv2(struct kvm *kvm)
arch/arm64/kvm/config.c
236
return ((kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY) &&
arch/arm64/kvm/config.c
237
kvm_has_feat_enum(kvm, ID_AA64MMFR2_EL1, NV, NI)) ||
arch/arm64/kvm/config.c
238
kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, NV2));
arch/arm64/kvm/config.c
241
static bool feat_nv2_e2h0_ni(struct kvm *kvm)
arch/arm64/kvm/config.c
243
return feat_nv2(kvm) && !kvm_has_feat(kvm, FEAT_E2H0);
arch/arm64/kvm/config.c
246
static bool feat_rasv1p1(struct kvm *kvm)
arch/arm64/kvm/config.c
248
return (kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, V1P1) ||
arch/arm64/kvm/config.c
249
(kvm_has_feat_enum(kvm, ID_AA64PFR0_EL1, RAS, IMP) &&
arch/arm64/kvm/config.c
250
kvm_has_feat(kvm, ID_AA64PFR1_EL1, RAS_frac, RASv1p1)));
arch/arm64/kvm/config.c
253
static bool feat_csv2_2_csv2_1p2(struct kvm *kvm)
arch/arm64/kvm/config.c
255
return (kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) ||
arch/arm64/kvm/config.c
256
(kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2) &&
arch/arm64/kvm/config.c
257
kvm_has_feat_enum(kvm, ID_AA64PFR0_EL1, CSV2, IMP)));
arch/arm64/kvm/config.c
260
static bool feat_pauth(struct kvm *kvm)
arch/arm64/kvm/config.c
262
return kvm_has_pauth(kvm, PAuth);
arch/arm64/kvm/config.c
265
static bool feat_pauth_lr(struct kvm *kvm)
arch/arm64/kvm/config.c
267
return kvm_has_pauth(kvm, PAuth_LR);
arch/arm64/kvm/config.c
270
static bool feat_aderr(struct kvm *kvm)
arch/arm64/kvm/config.c
272
return (kvm_has_feat(kvm, ID_AA64MMFR3_EL1, ADERR, FEAT_ADERR) &&
arch/arm64/kvm/config.c
273
kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SDERR, FEAT_ADERR));
arch/arm64/kvm/config.c
276
static bool feat_anerr(struct kvm *kvm)
arch/arm64/kvm/config.c
278
return (kvm_has_feat(kvm, ID_AA64MMFR3_EL1, ANERR, FEAT_ANERR) &&
arch/arm64/kvm/config.c
279
kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SNERR, FEAT_ANERR));
arch/arm64/kvm/config.c
282
static bool feat_sme_smps(struct kvm *kvm)
arch/arm64/kvm/config.c
290
return (kvm_has_feat(kvm, FEAT_SME) &&
arch/arm64/kvm/config.c
294
static bool feat_spe_fds(struct kvm *kvm)
arch/arm64/kvm/config.c
300
return (kvm_has_feat(kvm, FEAT_SPEv1p4) &&
arch/arm64/kvm/config.c
304
static bool feat_spe_fne(struct kvm *kvm)
arch/arm64/kvm/config.c
310
return (kvm_has_feat(kvm, FEAT_SPEv1p2) &&
arch/arm64/kvm/config.c
314
static bool feat_trbe_mpam(struct kvm *kvm)
arch/arm64/kvm/config.c
320
return (kvm_has_feat(kvm, FEAT_TRBE) &&
arch/arm64/kvm/config.c
321
kvm_has_feat(kvm, FEAT_MPAM) &&
arch/arm64/kvm/config.c
325
static bool feat_ebep_pmuv3_ss(struct kvm *kvm)
arch/arm64/kvm/config.c
327
return kvm_has_feat(kvm, FEAT_EBEP) || kvm_has_feat(kvm, FEAT_PMUv3_SS);
arch/arm64/kvm/config.c
330
static bool feat_mixedendel0(struct kvm *kvm)
arch/arm64/kvm/config.c
332
return kvm_has_feat(kvm, FEAT_MixedEnd) || kvm_has_feat(kvm, FEAT_MixedEndEL0);
arch/arm64/kvm/config.c
335
static bool feat_mte_async(struct kvm *kvm)
arch/arm64/kvm/config.c
337
return kvm_has_feat(kvm, FEAT_MTE2) && kvm_has_feat_enum(kvm, FEAT_MTE_ASYNC);
arch/arm64/kvm/config.c
346
static bool feat_pmuv3p1(struct kvm *kvm)
arch/arm64/kvm/config.c
348
return check_pmu_revision(kvm, V3P1);
arch/arm64/kvm/config.c
351
static bool feat_pmuv3p5(struct kvm *kvm)
arch/arm64/kvm/config.c
353
return check_pmu_revision(kvm, V3P5);
arch/arm64/kvm/config.c
356
static bool feat_pmuv3p7(struct kvm *kvm)
arch/arm64/kvm/config.c
358
return check_pmu_revision(kvm, V3P7);
arch/arm64/kvm/config.c
361
static bool feat_pmuv3p9(struct kvm *kvm)
arch/arm64/kvm/config.c
363
return check_pmu_revision(kvm, V3P9);
arch/arm64/kvm/config.c
367
((kvm_has_feat_enum(kvm, ID_AA64MMFR0_EL1, TGRAN##s##_2, TGRAN##s) && \
arch/arm64/kvm/config.c
368
kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN##s, IMP)) || \
arch/arm64/kvm/config.c
369
kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN##s##_2, IMP))
arch/arm64/kvm/config.c
371
static bool feat_lpa2(struct kvm *kvm)
arch/arm64/kvm/config.c
373
return ((kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN4, 52_BIT) ||
arch/arm64/kvm/config.c
374
!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN4, IMP)) &&
arch/arm64/kvm/config.c
375
(kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN16, 52_BIT) ||
arch/arm64/kvm/config.c
376
!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN16, IMP)) &&
arch/arm64/kvm/config.c
377
(kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN4_2, 52_BIT) ||
arch/arm64/kvm/config.c
378
!has_feat_s2tgran(kvm, 4)) &&
arch/arm64/kvm/config.c
379
(kvm_has_feat(kvm, ID_AA64MMFR0_EL1, TGRAN16_2, 52_BIT) ||
arch/arm64/kvm/config.c
380
!has_feat_s2tgran(kvm, 16)));
arch/arm64/kvm/config.c
383
static bool feat_vmid16(struct kvm *kvm)
arch/arm64/kvm/config.c
385
return kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16);
arch/arm64/kvm/config.c
42
bool (*match)(struct kvm *);
arch/arm64/kvm/debug.c
157
KVM_BUG_ON(vcpu_get_flag(vcpu, SYSREGS_ON_CPU), vcpu->kvm);
arch/arm64/kvm/emulate-nested.c
2505
static u64 kvm_get_sysreg_res0(struct kvm *kvm, enum vcpu_sysreg sr)
arch/arm64/kvm/emulate-nested.c
2507
return kvm_get_sysreg_resx(kvm, sr).res0;
arch/arm64/kvm/emulate-nested.c
2513
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/emulate-nested.c
2539
return !(kvm_get_sysreg_res0(kvm, sr) & BIT(tc.bit));
arch/arm64/kvm/emulate-nested.c
2571
(vcpu->kvm->arch.fgu[tc.fgt] & BIT(tc.bit))) {
arch/arm64/kvm/emulate-nested.c
2672
if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, IDS, IMP))
arch/arm64/kvm/emulate-nested.c
2761
if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, PAN, IMP))
arch/arm64/kvm/emulate-nested.c
2763
if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, NMI, IMP))
arch/arm64/kvm/emulate-nested.c
2766
if (kvm_has_feat(vcpu->kvm, ID_AA64DFR1_EL1, EBEP, IMP))
arch/arm64/kvm/emulate-nested.c
2772
if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, GCS, IMP))
arch/arm64/kvm/emulate-nested.c
2802
if (kvm_has_pauth(vcpu->kvm, FPACCOMBINE) && !(spsr & PSR_IL_BIT)) {
arch/arm64/kvm/guest.c
1003
mutex_lock(&kvm->slots_lock);
arch/arm64/kvm/guest.c
1005
if (write && atomic_read(&kvm->nr_memslots_dirty_logging)) {
arch/arm64/kvm/guest.c
1011
struct page *page = __gfn_to_page(kvm, gfn, write);
arch/arm64/kvm/guest.c
1076
mutex_unlock(&kvm->slots_lock);
arch/arm64/kvm/guest.c
911
mutex_lock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/guest.c
913
mutex_unlock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/guest.c
975
int kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm,
arch/arm64/kvm/guest.c
985
if (!kvm_has_mte(kvm))
arch/arm64/kvm/handle_exit.c
310
if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, GCS, IMP))
arch/arm64/kvm/handle_exit.c
323
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/handle_exit.c
341
allowed = kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V);
arch/arm64/kvm/handle_exit.c
345
allowed = kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA);
arch/arm64/kvm/handle_exit.c
349
allowed = kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64);
arch/arm64/kvm/handle_exit.c
353
allowed = kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, TRBE_V1P1);
arch/arm64/kvm/handle_exit.c
357
allowed = kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P5);
arch/arm64/kvm/hyp/exception.c
128
if (kvm_has_mte(kern_hyp_va(vcpu->kvm)))
arch/arm64/kvm/hyp/include/hyp/switch.h
144
if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR3_EL1, S1POE, IMP))
arch/arm64/kvm/hyp/include/hyp/switch.h
411
else if (kvm_has_ras(kern_hyp_va(vcpu->kvm)))
arch/arm64/kvm/hyp/include/hyp/switch.h
578
if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm)))
arch/arm64/kvm/hyp/include/hyp/switch.h
635
if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm)))
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
109
return kvm_has_s1poe(kern_hyp_va(vcpu->kvm));
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
120
return kvm_has_ras(kern_hyp_va(vcpu->kvm));
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
131
return kvm_has_sctlr2(kern_hyp_va(vcpu->kvm));
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
48
struct kvm *kvm = kern_hyp_va(ctxt_to_vcpu(ctxt)->kvm);
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
51
test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags)))
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
54
return kvm_read_vm_id_reg(kvm, SYS_MIDR_EL1);
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
76
return kvm_has_mte(kern_hyp_va(vcpu->kvm));
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
87
return kvm_has_s1pie(kern_hyp_va(vcpu->kvm));
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
98
return kvm_has_tcr2(kern_hyp_va(vcpu->kvm));
arch/arm64/kvm/hyp/include/nvhe/pkvm.h
35
struct kvm kvm;
arch/arm64/kvm/hyp/include/nvhe/pkvm.h
38
struct kvm *host_kvm;
arch/arm64/kvm/hyp/include/nvhe/pkvm.h
55
return container_of(hyp_vcpu->vcpu.kvm, struct pkvm_hyp_vm, kvm);
arch/arm64/kvm/hyp/include/nvhe/pkvm.h
65
return kvm_vm_is_protected(&hyp_vm->kvm);
arch/arm64/kvm/hyp/include/nvhe/pkvm.h
72
int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
arch/arm64/kvm/hyp/nvhe/hyp-main.c
450
__kvm_tlb_flush_vmid(&hyp_vm->kvm.arch.mmu);
arch/arm64/kvm/hyp/nvhe/hyp-main.c
579
DECLARE_REG(struct kvm *, host_kvm, host_ctxt, 1);
arch/arm64/kvm/hyp/nvhe/hyp-main.c
90
has_fpmr = kvm_has_fpmr(kern_hyp_va(vcpu->kvm));
arch/arm64/kvm/hyp/nvhe/mem_protect.c
274
struct kvm_s2_mmu *mmu = &vm->kvm.arch.mmu;
arch/arm64/kvm/hyp/nvhe/mem_protect.c
303
vm->kvm.arch.mmu.pgd_phys = __hyp_pa(vm->pgt.pgd);
arch/arm64/kvm/hyp/nvhe/mem_protect.c
312
vm->kvm.arch.mmu.pgd_phys = 0ULL;
arch/arm64/kvm/hyp/nvhe/mem_protect.c
634
pkvm_handle_t handle = vm->kvm.arch.pkvm.handle;
arch/arm64/kvm/hyp/nvhe/pkvm.c
100
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/hyp/nvhe/pkvm.c
1005
if (!hyp_vm || !hyp_vm->kvm.arch.pkvm.is_dying) {
arch/arm64/kvm/hyp/nvhe/pkvm.c
1013
__kvm_tlb_flush_vmid(&hyp_vm->kvm.arch.mmu);
arch/arm64/kvm/hyp/nvhe/pkvm.c
1021
unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->kvm.created_vcpus);
arch/arm64/kvm/hyp/nvhe/pkvm.c
1024
for (idx = 0; idx < hyp_vm->kvm.created_vcpus; ++idx) {
arch/arm64/kvm/hyp/nvhe/pkvm.c
103
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) {
arch/arm64/kvm/hyp/nvhe/pkvm.c
1043
vm_size = pkvm_get_hyp_vm_size(hyp_vm->kvm.created_vcpus);
arch/arm64/kvm/hyp/nvhe/pkvm.c
108
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, IMP))
arch/arm64/kvm/hyp/nvhe/pkvm.c
111
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DoubleLock, IMP))
arch/arm64/kvm/hyp/nvhe/pkvm.c
114
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP)) {
arch/arm64/kvm/hyp/nvhe/pkvm.c
119
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP))
arch/arm64/kvm/hyp/nvhe/pkvm.c
122
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP))
arch/arm64/kvm/hyp/nvhe/pkvm.c
126
if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, FGT, IMP))
arch/arm64/kvm/hyp/nvhe/pkvm.c
138
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/hyp/nvhe/pkvm.c
141
if (kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL0, AARCH32) ||
arch/arm64/kvm/hyp/nvhe/pkvm.c
142
kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL1, AARCH32))
arch/arm64/kvm/hyp/nvhe/pkvm.c
149
if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, FP, IMP) ||
arch/arm64/kvm/hyp/nvhe/pkvm.c
150
!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AdvSIMD, IMP))
arch/arm64/kvm/hyp/nvhe/pkvm.c
154
if (kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP))
arch/arm64/kvm/hyp/nvhe/pkvm.c
263
if (!hyp_vm || hyp_vm->kvm.arch.pkvm.is_dying)
arch/arm64/kvm/hyp/nvhe/pkvm.c
266
if (hyp_vm->kvm.created_vcpus <= vcpu_idx)
arch/arm64/kvm/hyp/nvhe/pkvm.c
339
static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struct kvm *host_kvm)
arch/arm64/kvm/hyp/nvhe/pkvm.c
341
struct kvm *kvm = &hyp_vm->kvm;
arch/arm64/kvm/hyp/nvhe/pkvm.c
346
hyp_vm->kvm.arch.ctr_el0 = host_kvm->arch.ctr_el0;
arch/arm64/kvm/hyp/nvhe/pkvm.c
349
hyp_vm->kvm.arch.vgic.vgic_model = host_kvm->arch.vgic.vgic_model;
arch/arm64/kvm/hyp/nvhe/pkvm.c
352
if (!kvm_vm_is_protected(kvm)) {
arch/arm64/kvm/hyp/nvhe/pkvm.c
353
hyp_vm->kvm.arch.flags = host_arch_flags;
arch/arm64/kvm/hyp/nvhe/pkvm.c
354
hyp_vm->kvm.arch.flags &= ~BIT_ULL(KVM_ARCH_FLAG_ID_REGS_INITIALIZED);
arch/arm64/kvm/hyp/nvhe/pkvm.c
356
bitmap_copy(kvm->arch.vcpu_features,
arch/arm64/kvm/hyp/nvhe/pkvm.c
361
hyp_vm->kvm.arch.midr_el1 = host_kvm->arch.midr_el1;
arch/arm64/kvm/hyp/nvhe/pkvm.c
366
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_MTE))
arch/arm64/kvm/hyp/nvhe/pkvm.c
367
kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_MTE_ENABLED);
arch/arm64/kvm/hyp/nvhe/pkvm.c
373
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PMU_V3))
arch/arm64/kvm/hyp/nvhe/pkvm.c
376
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_ADDRESS))
arch/arm64/kvm/hyp/nvhe/pkvm.c
379
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC))
arch/arm64/kvm/hyp/nvhe/pkvm.c
382
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) {
arch/arm64/kvm/hyp/nvhe/pkvm.c
384
kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_GUEST_HAS_SVE);
arch/arm64/kvm/hyp/nvhe/pkvm.c
387
bitmap_and(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features,
arch/arm64/kvm/hyp/nvhe/pkvm.c
425
static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm,
arch/arm64/kvm/hyp/nvhe/pkvm.c
428
struct kvm_s2_mmu *mmu = &hyp_vm->kvm.arch.mmu;
arch/arm64/kvm/hyp/nvhe/pkvm.c
431
hyp_vm->kvm.arch.pkvm.handle = handle;
arch/arm64/kvm/hyp/nvhe/pkvm.c
434
hyp_vm->kvm.created_vcpus = nr_vcpus;
arch/arm64/kvm/hyp/nvhe/pkvm.c
435
hyp_vm->kvm.arch.pkvm.is_protected = READ_ONCE(host_kvm->arch.pkvm.is_protected);
arch/arm64/kvm/hyp/nvhe/pkvm.c
436
hyp_vm->kvm.arch.pkvm.is_created = true;
arch/arm64/kvm/hyp/nvhe/pkvm.c
437
hyp_vm->kvm.arch.flags = 0;
arch/arm64/kvm/hyp/nvhe/pkvm.c
444
mmu->arch = &hyp_vm->kvm.arch;
arch/arm64/kvm/hyp/nvhe/pkvm.c
480
clear_bit(KVM_ARM_VCPU_SVE, vcpu->kvm->arch.vcpu_features);
arch/arm64/kvm/hyp/nvhe/pkvm.c
487
const struct kvm *host_kvm = hyp_vm->host_kvm;
arch/arm64/kvm/hyp/nvhe/pkvm.c
488
struct kvm *kvm = &hyp_vm->kvm;
arch/arm64/kvm/hyp/nvhe/pkvm.c
493
if (test_and_set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags))
arch/arm64/kvm/hyp/nvhe/pkvm.c
496
memcpy(kvm->arch.id_regs, host_kvm->arch.id_regs, sizeof(kvm->arch.id_regs));
arch/arm64/kvm/hyp/nvhe/pkvm.c
52
kvm_read_vm_id_reg(vcpu->kvm, SYS_CTR_EL0) == read_cpuid(CTR_EL0))
arch/arm64/kvm/hyp/nvhe/pkvm.c
524
hyp_vcpu->vcpu.kvm = &hyp_vm->kvm;
arch/arm64/kvm/hyp/nvhe/pkvm.c
528
hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu;
arch/arm64/kvm/hyp/nvhe/pkvm.c
60
if (kvm_has_mte(vcpu->kvm))
arch/arm64/kvm/hyp/nvhe/pkvm.c
66
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/hyp/nvhe/pkvm.c
733
.kvm = {
arch/arm64/kvm/hyp/nvhe/pkvm.c
736
.arch = &selftest_vm.kvm.arch,
arch/arm64/kvm/hyp/nvhe/pkvm.c
746
.hw_mmu = &selftest_vm.kvm.arch.mmu,
arch/arm64/kvm/hyp/nvhe/pkvm.c
748
.kvm = &selftest_vm.kvm,
arch/arm64/kvm/hyp/nvhe/pkvm.c
758
selftest_vm.kvm.arch.mmu.vtcr = host_mmu.arch.mmu.vtcr;
arch/arm64/kvm/hyp/nvhe/pkvm.c
766
min_pages = kvm_mmu_cache_min_pages(&selftest_vm.kvm.arch.mmu);
arch/arm64/kvm/hyp/nvhe/pkvm.c
781
selftest_vm.kvm.arch.pkvm.handle = __pkvm_reserve_vm();
arch/arm64/kvm/hyp/nvhe/pkvm.c
782
insert_vm_table_entry(selftest_vm.kvm.arch.pkvm.handle, &selftest_vm);
arch/arm64/kvm/hyp/nvhe/pkvm.c
789
remove_vm_table_entry(selftest_vm.kvm.arch.pkvm.handle);
arch/arm64/kvm/hyp/nvhe/pkvm.c
79
if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) {
arch/arm64/kvm/hyp/nvhe/pkvm.c
808
int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
arch/arm64/kvm/hyp/nvhe/pkvm.c
84
if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, IMP))
arch/arm64/kvm/hyp/nvhe/pkvm.c
87
if (!kvm_has_mte(kvm)) {
arch/arm64/kvm/hyp/nvhe/pkvm.c
885
if (idx >= hyp_vm->kvm.created_vcpus)
arch/arm64/kvm/hyp/nvhe/pkvm.c
92
if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP))
arch/arm64/kvm/hyp/nvhe/pkvm.c
955
if (hyp_vm->kvm.arch.pkvm.is_dying)
arch/arm64/kvm/hyp/nvhe/pkvm.c
982
if (!hyp_vm || hyp_vm->kvm.arch.pkvm.is_dying) {
arch/arm64/kvm/hyp/nvhe/pkvm.c
987
hyp_vm->kvm.arch.pkvm.is_dying = true;
arch/arm64/kvm/hyp/nvhe/pkvm.c
998
struct kvm *host_kvm;
arch/arm64/kvm/hyp/nvhe/switch.c
116
if (vgic_is_v5(kern_hyp_va(vcpu->kvm))) {
arch/arm64/kvm/hyp/nvhe/switch.c
131
if (vgic_is_v5(kern_hyp_va(vcpu->kvm))) {
arch/arm64/kvm/hyp/nvhe/sys_regs.c
202
bool (*vm_supported)(const struct kvm *) = restrictions[i].vm_supported;
arch/arm64/kvm/hyp/nvhe/sys_regs.c
212
if (vm_supported && !vm_supported(vcpu->kvm))
arch/arm64/kvm/hyp/nvhe/sys_regs.c
288
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/hyp/nvhe/sys_regs.c
291
if (WARN_ON_ONCE(!test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags)))
arch/arm64/kvm/hyp/nvhe/sys_regs.c
295
return kvm->arch.id_regs[IDREG_IDX(reg)];
arch/arm64/kvm/hyp/nvhe/sys_regs.c
363
if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, IDS, IMP))
arch/arm64/kvm/hyp/nvhe/sys_regs.c
37
bool (*vm_supported)(const struct kvm *kvm);
arch/arm64/kvm/hyp/nvhe/sys_regs.c
532
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/hyp/nvhe/sys_regs.c
533
struct kvm_arch *ka = &kvm->arch;
arch/arm64/kvm/hyp/nvhe/sys_regs.c
538
if (test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags))
arch/arm64/kvm/hyp/nvhe/sys_regs.c
548
set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags);
arch/arm64/kvm/hyp/nvhe/sys_regs.c
60
static bool vm_has_ptrauth(const struct kvm *kvm)
arch/arm64/kvm/hyp/nvhe/sys_regs.c
67
kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_GENERIC);
arch/arm64/kvm/hyp/nvhe/sys_regs.c
70
static bool vm_has_sve(const struct kvm *kvm)
arch/arm64/kvm/hyp/nvhe/sys_regs.c
72
return system_supports_sve() && kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_SVE);
arch/arm64/kvm/hyp/nvhe/timer-sr.c
52
!kern_hyp_va(vcpu->kvm)->arch.timer_data.poffset)
arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c
39
struct kvm *kvm = kern_hyp_va(vcpu->kvm);
arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c
40
struct vgic_dist *vgic = &kvm->arch.vgic;
arch/arm64/kvm/hyp/vgic-v3-sr.c
1150
if (kern_hyp_va(vcpu->kvm)->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
arch/arm64/kvm/hyp/vhe/switch.c
431
atomic_read(&vcpu->kvm->arch.vncr_map_count))
arch/arm64/kvm/hypercalls.c
100
struct kvm_smccc_features *smccc_feat = &vcpu->kvm->arch.smccc_feat;
arch/arm64/kvm/hypercalls.c
138
static int kvm_smccc_filter_insert_reserved(struct kvm *kvm)
arch/arm64/kvm/hypercalls.c
147
r = mtree_insert_range(&kvm->arch.smccc_filter,
arch/arm64/kvm/hypercalls.c
154
r = mtree_insert_range(&kvm->arch.smccc_filter,
arch/arm64/kvm/hypercalls.c
163
mtree_destroy(&kvm->arch.smccc_filter);
arch/arm64/kvm/hypercalls.c
167
static bool kvm_smccc_filter_configured(struct kvm *kvm)
arch/arm64/kvm/hypercalls.c
169
return !mtree_empty(&kvm->arch.smccc_filter);
arch/arm64/kvm/hypercalls.c
172
static int kvm_smccc_set_filter(struct kvm *kvm, struct kvm_smccc_filter __user *uaddr)
arch/arm64/kvm/hypercalls.c
191
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/hypercalls.c
193
if (kvm_vm_has_ran_once(kvm)) {
arch/arm64/kvm/hypercalls.c
198
if (!kvm_smccc_filter_configured(kvm)) {
arch/arm64/kvm/hypercalls.c
199
r = kvm_smccc_filter_insert_reserved(kvm);
arch/arm64/kvm/hypercalls.c
204
r = mtree_insert_range(&kvm->arch.smccc_filter, start, end,
arch/arm64/kvm/hypercalls.c
207
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/hypercalls.c
211
static u8 kvm_smccc_filter_get_action(struct kvm *kvm, u32 func_id)
arch/arm64/kvm/hypercalls.c
216
if (!kvm_smccc_filter_configured(kvm))
arch/arm64/kvm/hypercalls.c
225
val = mt_find(&kvm->arch.smccc_filter, &idx, idx);
arch/arm64/kvm/hypercalls.c
235
u8 action = kvm_smccc_filter_get_action(vcpu->kvm, func_id);
arch/arm64/kvm/hypercalls.c
267
struct kvm_smccc_features *smccc_feat = &vcpu->kvm->arch.smccc_feat;
arch/arm64/kvm/hypercalls.c
323
if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SSBS, IMP))
arch/arm64/kvm/hypercalls.c
399
void kvm_arm_init_hypercalls(struct kvm *kvm)
arch/arm64/kvm/hypercalls.c
401
struct kvm_smccc_features *smccc_feat = &kvm->arch.smccc_feat;
arch/arm64/kvm/hypercalls.c
407
mt_init(&kvm->arch.smccc_filter);
arch/arm64/kvm/hypercalls.c
410
void kvm_arm_teardown_hypercalls(struct kvm *kvm)
arch/arm64/kvm/hypercalls.c
412
mtree_destroy(&kvm->arch.smccc_filter);
arch/arm64/kvm/hypercalls.c
459
if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SSBS, IMP))
arch/arm64/kvm/hypercalls.c
485
struct kvm_smccc_features *smccc_feat = &vcpu->kvm->arch.smccc_feat;
arch/arm64/kvm/hypercalls.c
49
cycles = systime_snapshot.cycles - vcpu->kvm->arch.timer_data.voffset;
arch/arm64/kvm/hypercalls.c
52
cycles = systime_snapshot.cycles - vcpu->kvm->arch.timer_data.poffset;
arch/arm64/kvm/hypercalls.c
523
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/hypercalls.c
524
struct kvm_smccc_features *smccc_feat = &kvm->arch.smccc_feat;
arch/arm64/kvm/hypercalls.c
552
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/hypercalls.c
554
if (kvm_vm_has_ran_once(kvm) && val != *fw_reg_bmap) {
arch/arm64/kvm/hypercalls.c
561
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/hypercalls.c
587
vcpu->kvm->arch.psci_version = val;
arch/arm64/kvm/hypercalls.c
596
vcpu->kvm->arch.psci_version = val;
arch/arm64/kvm/hypercalls.c
659
int kvm_vm_smccc_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
arch/arm64/kvm/hypercalls.c
669
int kvm_vm_smccc_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
arch/arm64/kvm/hypercalls.c
675
return kvm_smccc_set_filter(kvm, uaddr);
arch/arm64/kvm/inject_fault.c
80
if (!kvm_has_sctlr2(vcpu->kvm))
arch/arm64/kvm/mmio.c
182
&vcpu->kvm->arch.flags)) {
arch/arm64/kvm/mmu.c
1012
mmu->arch = &kvm->arch;
arch/arm64/kvm/mmu.c
1036
if (kvm_is_nested_s2_mmu(kvm, mmu))
arch/arm64/kvm/mmu.c
1049
void kvm_uninit_stage2_mmu(struct kvm *kvm)
arch/arm64/kvm/mmu.c
1051
kvm_free_stage2_pgd(&kvm->arch.mmu);
arch/arm64/kvm/mmu.c
1052
kvm_mmu_free_memory_cache(&kvm->arch.mmu.split_page_cache);
arch/arm64/kvm/mmu.c
1055
static void stage2_unmap_memslot(struct kvm *kvm,
arch/arm64/kvm/mmu.c
106
static bool need_split_memcache_topup_or_resched(struct kvm *kvm)
arch/arm64/kvm/mmu.c
1091
kvm_stage2_unmap_range(&kvm->arch.mmu, gpa, vm_end - vm_start, true);
arch/arm64/kvm/mmu.c
1104
void stage2_unmap_vm(struct kvm *kvm)
arch/arm64/kvm/mmu.c
111
if (need_resched() || rwlock_needbreak(&kvm->mmu_lock))
arch/arm64/kvm/mmu.c
1110
idx = srcu_read_lock(&kvm->srcu);
arch/arm64/kvm/mmu.c
1112
write_lock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1114
slots = kvm_memslots(kvm);
arch/arm64/kvm/mmu.c
1116
stage2_unmap_memslot(kvm, memslot);
arch/arm64/kvm/mmu.c
1118
kvm_nested_s2_unmap(kvm, true);
arch/arm64/kvm/mmu.c
1120
write_unlock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1122
srcu_read_unlock(&kvm->srcu, idx);
arch/arm64/kvm/mmu.c
1127
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
arch/arm64/kvm/mmu.c
1130
write_lock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1138
if (kvm_is_nested_s2_mmu(kvm, mmu))
arch/arm64/kvm/mmu.c
114
chunk_size = kvm->arch.mmu.split_page_chunk_size;
arch/arm64/kvm/mmu.c
1141
write_unlock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
116
cache = &kvm->arch.mmu.split_page_cache;
arch/arm64/kvm/mmu.c
120
static int kvm_mmu_split_huge_pages(struct kvm *kvm, phys_addr_t addr,
arch/arm64/kvm/mmu.c
1205
int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
arch/arm64/kvm/mmu.c
1211
struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
arch/arm64/kvm/mmu.c
1229
write_lock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1232
write_unlock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1267
static void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)
arch/arm64/kvm/mmu.c
1269
struct kvm_memslots *slots = kvm_memslots(kvm);
arch/arm64/kvm/mmu.c
1279
write_lock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
128
lockdep_assert_held_write(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1280
kvm_stage2_wp_range(&kvm->arch.mmu, start, end);
arch/arm64/kvm/mmu.c
1281
kvm_nested_s2_wp(kvm);
arch/arm64/kvm/mmu.c
1282
write_unlock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1283
kvm_flush_remote_tlbs_memslot(kvm, memslot);
arch/arm64/kvm/mmu.c
1295
static void kvm_mmu_split_memory_region(struct kvm *kvm, int slot)
arch/arm64/kvm/mmu.c
130
chunk_size = kvm->arch.mmu.split_page_chunk_size;
arch/arm64/kvm/mmu.c
1301
lockdep_assert_held(&kvm->slots_lock);
arch/arm64/kvm/mmu.c
1303
slots = kvm_memslots(kvm);
arch/arm64/kvm/mmu.c
1309
write_lock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1310
kvm_mmu_split_huge_pages(kvm, start, end);
arch/arm64/kvm/mmu.c
1311
write_unlock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1325
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
arch/arm64/kvm/mmu.c
1333
lockdep_assert_held_write(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1335
kvm_stage2_wp_range(&kvm->arch.mmu, start, end);
arch/arm64/kvm/mmu.c
1345
if (kvm_dirty_log_manual_protect_and_init_set(kvm))
arch/arm64/kvm/mmu.c
1346
kvm_mmu_split_huge_pages(kvm, start, end);
arch/arm64/kvm/mmu.c
1348
kvm_nested_s2_wp(kvm);
arch/arm64/kvm/mmu.c
136
cache = &kvm->arch.mmu.split_page_cache;
arch/arm64/kvm/mmu.c
139
if (need_split_memcache_topup_or_resched(kvm)) {
arch/arm64/kvm/mmu.c
140
write_unlock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1430
transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot,
arch/arm64/kvm/mmu.c
1442
int sz = get_user_mapping_size(kvm, hva);
arch/arm64/kvm/mmu.c
146
write_lock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1499
static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn,
arch/arm64/kvm/mmu.c
1506
if (!kvm_has_mte(kvm))
arch/arm64/kvm/mmu.c
151
pgt = kvm->arch.mmu.pgt;
arch/arm64/kvm/mmu.c
1588
static enum kvm_pgtable_prot adjust_nested_exec_perms(struct kvm *kvm,
arch/arm64/kvm/mmu.c
1592
if (!kvm_s2_trans_exec_el0(kvm, nested))
arch/arm64/kvm/mmu.c
1594
if (!kvm_s2_trans_exec_el1(kvm, nested))
arch/arm64/kvm/mmu.c
1617
struct kvm *kvm = s2fd->vcpu->kvm;
arch/arm64/kvm/mmu.c
1640
mmu_seq = kvm->mmu_invalidate_seq;
arch/arm64/kvm/mmu.c
1644
ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, gfn, &pfn, &page, NULL);
arch/arm64/kvm/mmu.c
1661
prot = adjust_nested_exec_perms(kvm, s2fd->nested, prot);
arch/arm64/kvm/mmu.c
1663
kvm_fault_lock(kvm);
arch/arm64/kvm/mmu.c
1664
if (mmu_invalidate_retry(kvm, mmu_seq)) {
arch/arm64/kvm/mmu.c
1684
kvm_release_faultin_page(kvm, page, !!ret, prot & KVM_PGTABLE_PROT_W);
arch/arm64/kvm/mmu.c
1685
kvm_fault_unlock(kvm);
arch/arm64/kvm/mmu.c
1688
mark_page_dirty_in_slot(kvm, s2fd->memslot, gfn);
arch/arm64/kvm/mmu.c
1714
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/mmu.c
175
int kvm_arch_flush_remote_tlbs(struct kvm *kvm)
arch/arm64/kvm/mmu.c
1758
write_lock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
1762
write_unlock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
178
kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle);
arch/arm64/kvm/mmu.c
180
kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu);
arch/arm64/kvm/mmu.c
184
int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm,
arch/arm64/kvm/mmu.c
1852
struct kvm *kvm = s2fd->vcpu->kvm;
arch/arm64/kvm/mmu.c
1885
s2vi->mmu_seq = kvm->mmu_invalidate_seq;
arch/arm64/kvm/mmu.c
191
kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle);
arch/arm64/kvm/mmu.c
193
kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size);
arch/arm64/kvm/mmu.c
1942
kvm_release_faultin_page(s2fd->vcpu->kvm, s2vi->page, true, false);
arch/arm64/kvm/mmu.c
1969
struct kvm *kvm = s2fd->vcpu->kvm;
arch/arm64/kvm/mmu.c
2005
*prot = adjust_nested_exec_perms(kvm, s2fd->nested, *prot);
arch/arm64/kvm/mmu.c
2007
if (!kvm_s2_fault_is_perm(s2fd) && !s2vi->map_non_cacheable && kvm_has_mte(kvm)) {
arch/arm64/kvm/mmu.c
2023
struct kvm *kvm = s2fd->vcpu->kvm;
arch/arm64/kvm/mmu.c
2031
kvm_fault_lock(kvm);
arch/arm64/kvm/mmu.c
2034
if (mmu_invalidate_retry(kvm, s2vi->mmu_seq))
arch/arm64/kvm/mmu.c
2052
mapping_size = transparent_hugepage_adjust(kvm, s2fd->memslot,
arch/arm64/kvm/mmu.c
2062
if (!perm_fault_granule && !s2vi->map_non_cacheable && kvm_has_mte(kvm))
arch/arm64/kvm/mmu.c
2063
sanitise_mte_tags(kvm, pfn, mapping_size);
arch/arm64/kvm/mmu.c
2085
kvm_release_faultin_page(kvm, s2vi->page, !!ret, writable);
arch/arm64/kvm/mmu.c
2086
kvm_fault_unlock(kvm);
arch/arm64/kvm/mmu.c
2096
mark_page_dirty_in_slot(kvm, s2fd->memslot, gpa_to_gfn(ipa));
arch/arm64/kvm/mmu.c
2151
read_lock(&vcpu->kvm->mmu_lock);
arch/arm64/kvm/mmu.c
2154
read_unlock(&vcpu->kvm->mmu_lock);
arch/arm64/kvm/mmu.c
2187
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/mmu.c
2208
!test_bit(KVM_ARCH_FLAG_EXIT_SEA, &vcpu->kvm->arch.flags))
arch/arm64/kvm/mmu.c
2212
if (kvm_has_ras(kvm))
arch/arm64/kvm/mmu.c
2269
if (KVM_BUG_ON(ipa == INVALID_GPA, vcpu->kvm))
arch/arm64/kvm/mmu.c
2304
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/arm64/kvm/mmu.c
2319
if (kvm_is_nested_s2_mmu(vcpu->kvm,vcpu->arch.hw_mmu) &&
arch/arm64/kvm/mmu.c
2347
memslot = gfn_to_memslot(vcpu->kvm, gfn);
arch/arm64/kvm/mmu.c
2411
if (kvm_vm_is_protected(vcpu->kvm)) {
arch/arm64/kvm/mmu.c
2430
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/arm64/kvm/mmu.c
2434
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
arch/arm64/kvm/mmu.c
2436
if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm))
arch/arm64/kvm/mmu.c
2439
__unmap_stage2_range(&kvm->arch.mmu, range->start << PAGE_SHIFT,
arch/arm64/kvm/mmu.c
2443
kvm_nested_s2_unmap(kvm, range->may_block);
arch/arm64/kvm/mmu.c
2447
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/arm64/kvm/mmu.c
2451
if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm))
arch/arm64/kvm/mmu.c
2454
return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt,
arch/arm64/kvm/mmu.c
2463
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/arm64/kvm/mmu.c
2467
if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm))
arch/arm64/kvm/mmu.c
2470
return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt,
arch/arm64/kvm/mmu.c
2572
void kvm_arch_commit_memory_region(struct kvm *kvm,
arch/arm64/kvm/mmu.c
2595
if (kvm_dirty_log_manual_protect_and_init_set(kvm))
arch/arm64/kvm/mmu.c
2602
kvm_mmu_wp_memory_region(kvm, new->id);
arch/arm64/kvm/mmu.c
2603
kvm_mmu_split_memory_region(kvm, new->id);
arch/arm64/kvm/mmu.c
2612
kvm_mmu_free_memory_cache(&kvm->arch.mmu.split_page_cache);
arch/arm64/kvm/mmu.c
2616
int kvm_arch_prepare_memory_region(struct kvm *kvm,
arch/arm64/kvm/mmu.c
2624
if (kvm_vm_is_protected(kvm)) {
arch/arm64/kvm/mmu.c
2626
if (pkvm_hyp_vm_is_created(kvm) &&
arch/arm64/kvm/mmu.c
2645
if ((new->base_gfn + new->npages) > (kvm_phys_size(&kvm->arch.mmu) >> PAGE_SHIFT))
arch/arm64/kvm/mmu.c
2656
if (kvm_slot_has_gmem(new) && kvm_has_mte(kvm))
arch/arm64/kvm/mmu.c
2681
if (kvm_has_mte(kvm) && !kvm_vma_mte_allowed(vma)) {
arch/arm64/kvm/mmu.c
2709
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
arch/arm64/kvm/mmu.c
2713
void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
arch/arm64/kvm/mmu.c
2717
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
arch/arm64/kvm/mmu.c
2723
write_lock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
2724
kvm_stage2_unmap_range(&kvm->arch.mmu, gpa, size, true);
arch/arm64/kvm/mmu.c
2725
kvm_nested_s2_unmap(kvm, true);
arch/arm64/kvm/mmu.c
2726
write_unlock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
2773
stage2_flush_vm(vcpu->kvm);
arch/arm64/kvm/mmu.c
2788
stage2_flush_vm(vcpu->kvm);
arch/arm64/kvm/mmu.c
331
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
arch/arm64/kvm/mmu.c
334
lockdep_assert_held_write(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
354
static void stage2_flush_memslot(struct kvm *kvm,
arch/arm64/kvm/mmu.c
360
kvm_stage2_flush_range(&kvm->arch.mmu, addr, end);
arch/arm64/kvm/mmu.c
370
static void stage2_flush_vm(struct kvm *kvm)
arch/arm64/kvm/mmu.c
376
idx = srcu_read_lock(&kvm->srcu);
arch/arm64/kvm/mmu.c
377
write_lock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
379
slots = kvm_memslots(kvm);
arch/arm64/kvm/mmu.c
381
stage2_flush_memslot(kvm, memslot);
arch/arm64/kvm/mmu.c
383
kvm_nested_s2_flush(kvm);
arch/arm64/kvm/mmu.c
385
write_unlock(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
386
srcu_read_unlock(&kvm->srcu, idx);
arch/arm64/kvm/mmu.c
67
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
arch/arm64/kvm/mmu.c
82
cond_resched_rwlock_write(&kvm->mmu_lock);
arch/arm64/kvm/mmu.c
849
static int get_user_mapping_size(struct kvm *kvm, u64 addr)
arch/arm64/kvm/mmu.c
852
.pgd = (kvm_pteref_t)kvm->mm->pgd,
arch/arm64/kvm/mmu.c
981
int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long type)
arch/arm64/kvm/mmu.c
997
if (kvm_is_nested_s2_mmu(kvm, mmu))
arch/arm64/kvm/nested.c
1005
lockdep_assert_held_write(&kvm->mmu_lock);
arch/arm64/kvm/nested.c
1007
if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
arch/arm64/kvm/nested.c
101
if (kvm->arch.nested_mmus_size) {
arch/arm64/kvm/nested.c
1010
kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) {
arch/arm64/kvm/nested.c
102
memcpy(tmp, kvm->arch.nested_mmus,
arch/arm64/kvm/nested.c
103
size_mul(sizeof(*tmp), kvm->arch.nested_mmus_size));
arch/arm64/kvm/nested.c
1038
static void invalidate_vncr_va(struct kvm *kvm,
arch/arm64/kvm/nested.c
1045
lockdep_assert_held_write(&kvm->mmu_lock);
arch/arm64/kvm/nested.c
1047
kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) {
arch/arm64/kvm/nested.c
105
for (int i = 0; i < kvm->arch.nested_mmus_size; i++)
arch/arm64/kvm/nested.c
109
swap(kvm->arch.nested_mmus, tmp);
arch/arm64/kvm/nested.c
111
write_unlock(&kvm->mmu_lock);
arch/arm64/kvm/nested.c
116
for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++)
arch/arm64/kvm/nested.c
117
ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]);
arch/arm64/kvm/nested.c
120
for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++)
arch/arm64/kvm/nested.c
121
kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]);
arch/arm64/kvm/nested.c
1212
guard(write_lock)(&vcpu->kvm->mmu_lock);
arch/arm64/kvm/nested.c
1213
invalidate_vncr_va(vcpu->kvm, &scope);
arch/arm64/kvm/nested.c
1216
void kvm_nested_s2_wp(struct kvm *kvm)
arch/arm64/kvm/nested.c
1220
lockdep_assert_held_write(&kvm->mmu_lock);
arch/arm64/kvm/nested.c
1222
if (!kvm->arch.nested_mmus_size)
arch/arm64/kvm/nested.c
1225
for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
arch/arm64/kvm/nested.c
1226
struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
arch/arm64/kvm/nested.c
1232
kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
arch/arm64/kvm/nested.c
1235
void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block)
arch/arm64/kvm/nested.c
1239
lockdep_assert_held_write(&kvm->mmu_lock);
arch/arm64/kvm/nested.c
1241
if (!kvm->arch.nested_mmus_size)
arch/arm64/kvm/nested.c
1244
for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
arch/arm64/kvm/nested.c
1245
struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
arch/arm64/kvm/nested.c
1251
kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
arch/arm64/kvm/nested.c
1254
void kvm_nested_s2_flush(struct kvm *kvm)
arch/arm64/kvm/nested.c
1258
lockdep_assert_held_write(&kvm->mmu_lock);
arch/arm64/kvm/nested.c
1260
if (!kvm->arch.nested_mmus_size)
arch/arm64/kvm/nested.c
1263
for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
arch/arm64/kvm/nested.c
1264
struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
arch/arm64/kvm/nested.c
1271
void kvm_arch_flush_shadow_all(struct kvm *kvm)
arch/arm64/kvm/nested.c
1275
for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
arch/arm64/kvm/nested.c
1276
struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
arch/arm64/kvm/nested.c
1281
kvfree(kvm->arch.nested_mmus);
arch/arm64/kvm/nested.c
1282
kvm->arch.nested_mmus = NULL;
arch/arm64/kvm/nested.c
1283
kvm->arch.nested_mmus_size = 0;
arch/arm64/kvm/nested.c
1284
kvm_uninit_stage2_mmu(kvm);
arch/arm64/kvm/nested.c
129
kvm->arch.nested_mmus_size = num_mmus;
arch/arm64/kvm/nested.c
1330
if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
arch/arm64/kvm/nested.c
1343
scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
arch/arm64/kvm/nested.c
1378
scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
arch/arm64/kvm/nested.c
1390
guard(srcu)(&vcpu->kvm->srcu);
arch/arm64/kvm/nested.c
1400
mmu_seq = vcpu->kvm->mmu_invalidate_seq;
arch/arm64/kvm/nested.c
1404
memslot = gfn_to_memslot(vcpu->kvm, gfn);
arch/arm64/kvm/nested.c
1419
ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL);
arch/arm64/kvm/nested.c
1435
kvm_release_faultin_page(vcpu->kvm, page, true, false);
arch/arm64/kvm/nested.c
1440
scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
arch/arm64/kvm/nested.c
1441
if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) {
arch/arm64/kvm/nested.c
1442
kvm_release_faultin_page(vcpu->kvm, page, true, false);
arch/arm64/kvm/nested.c
1453
kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable);
arch/arm64/kvm/nested.c
1457
mark_page_dirty(vcpu->kvm, gfn);
arch/arm64/kvm/nested.c
1501
KVM_BUG(1, vcpu->kvm, "Unhandled VNCR abort, ESR=%llx\n", esr);
arch/arm64/kvm/nested.c
1553
guard(read_lock)(&vcpu->kvm->mmu_lock);
arch/arm64/kvm/nested.c
1595
atomic_inc(&vcpu->kvm->arch.vncr_map_count);
arch/arm64/kvm/nested.c
1606
u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val)
arch/arm64/kvm/nested.c
168
pa_max = kvm_get_pa_bits(vcpu->kvm);
arch/arm64/kvm/nested.c
1708
if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features))
arch/arm64/kvm/nested.c
1740
if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) {
arch/arm64/kvm/nested.c
1777
resx = kvm_get_sysreg_resx(vcpu->kvm, sr);
arch/arm64/kvm/nested.c
1784
static __always_inline void set_sysreg_masks(struct kvm *kvm, int sr, struct resx resx)
arch/arm64/kvm/nested.c
1790
kvm_set_sysreg_resx(kvm, sr, resx);
arch/arm64/kvm/nested.c
1795
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/nested.c
1798
lockdep_assert_held(&kvm->arch.config_lock);
arch/arm64/kvm/nested.c
1800
if (kvm->arch.sysreg_masks)
arch/arm64/kvm/nested.c
1803
kvm->arch.sysreg_masks = kzalloc_obj(*(kvm->arch.sysreg_masks),
arch/arm64/kvm/nested.c
1805
if (!kvm->arch.sysreg_masks)
arch/arm64/kvm/nested.c
1810
if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16))
arch/arm64/kvm/nested.c
1812
if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP))
arch/arm64/kvm/nested.c
1814
set_sysreg_masks(kvm, VTTBR_EL2, resx);
arch/arm64/kvm/nested.c
1817
resx = get_reg_fixed_bits(kvm, VTCR_EL2);
arch/arm64/kvm/nested.c
1818
set_sysreg_masks(kvm, VTCR_EL2, resx);
arch/arm64/kvm/nested.c
1823
set_sysreg_masks(kvm, VMPIDR_EL2, resx);
arch/arm64/kvm/nested.c
1826
resx = get_reg_fixed_bits(kvm, HCR_EL2);
arch/arm64/kvm/nested.c
1827
set_sysreg_masks(kvm, HCR_EL2, resx);
arch/arm64/kvm/nested.c
1830
resx = get_reg_fixed_bits(kvm, HCRX_EL2);
arch/arm64/kvm/nested.c
1831
set_sysreg_masks(kvm, HCRX_EL2, resx);
arch/arm64/kvm/nested.c
1834
resx = get_reg_fixed_bits(kvm, HFGRTR_EL2);
arch/arm64/kvm/nested.c
1835
set_sysreg_masks(kvm, HFGRTR_EL2, resx);
arch/arm64/kvm/nested.c
1836
resx = get_reg_fixed_bits(kvm, HFGWTR_EL2);
arch/arm64/kvm/nested.c
1837
set_sysreg_masks(kvm, HFGWTR_EL2, resx);
arch/arm64/kvm/nested.c
1840
resx = get_reg_fixed_bits(kvm, HDFGRTR_EL2);
arch/arm64/kvm/nested.c
1841
set_sysreg_masks(kvm, HDFGRTR_EL2, resx);
arch/arm64/kvm/nested.c
1842
resx = get_reg_fixed_bits(kvm, HDFGWTR_EL2);
arch/arm64/kvm/nested.c
1843
set_sysreg_masks(kvm, HDFGWTR_EL2, resx);
arch/arm64/kvm/nested.c
1846
resx = get_reg_fixed_bits(kvm, HFGITR_EL2);
arch/arm64/kvm/nested.c
1847
set_sysreg_masks(kvm, HFGITR_EL2, resx);
arch/arm64/kvm/nested.c
1850
resx = get_reg_fixed_bits(kvm, HAFGRTR_EL2);
arch/arm64/kvm/nested.c
1851
set_sysreg_masks(kvm, HAFGRTR_EL2, resx);
arch/arm64/kvm/nested.c
1854
resx = get_reg_fixed_bits(kvm, HFGRTR2_EL2);
arch/arm64/kvm/nested.c
1855
set_sysreg_masks(kvm, HFGRTR2_EL2, resx);
arch/arm64/kvm/nested.c
1856
resx = get_reg_fixed_bits(kvm, HFGWTR2_EL2);
arch/arm64/kvm/nested.c
1857
set_sysreg_masks(kvm, HFGWTR2_EL2, resx);
arch/arm64/kvm/nested.c
1860
resx = get_reg_fixed_bits(kvm, HDFGRTR2_EL2);
arch/arm64/kvm/nested.c
1861
set_sysreg_masks(kvm, HDFGRTR2_EL2, resx);
arch/arm64/kvm/nested.c
1862
resx = get_reg_fixed_bits(kvm, HDFGWTR2_EL2);
arch/arm64/kvm/nested.c
1863
set_sysreg_masks(kvm, HDFGWTR2_EL2, resx);
arch/arm64/kvm/nested.c
1866
resx = get_reg_fixed_bits(kvm, HFGITR2_EL2);
arch/arm64/kvm/nested.c
1867
set_sysreg_masks(kvm, HFGITR2_EL2, resx);
arch/arm64/kvm/nested.c
1870
resx = get_reg_fixed_bits(kvm, TCR2_EL2);
arch/arm64/kvm/nested.c
1871
set_sysreg_masks(kvm, TCR2_EL2, resx);
arch/arm64/kvm/nested.c
1874
resx = get_reg_fixed_bits(kvm, SCTLR_EL1);
arch/arm64/kvm/nested.c
1875
set_sysreg_masks(kvm, SCTLR_EL1, resx);
arch/arm64/kvm/nested.c
1878
resx = get_reg_fixed_bits(kvm, SCTLR_EL2);
arch/arm64/kvm/nested.c
1879
set_sysreg_masks(kvm, SCTLR_EL2, resx);
arch/arm64/kvm/nested.c
1882
resx = get_reg_fixed_bits(kvm, SCTLR2_EL1);
arch/arm64/kvm/nested.c
1883
set_sysreg_masks(kvm, SCTLR2_EL1, resx);
arch/arm64/kvm/nested.c
1884
resx = get_reg_fixed_bits(kvm, SCTLR2_EL2);
arch/arm64/kvm/nested.c
1885
set_sysreg_masks(kvm, SCTLR2_EL2, resx);
arch/arm64/kvm/nested.c
1888
resx = get_reg_fixed_bits(kvm, MDCR_EL2);
arch/arm64/kvm/nested.c
1889
set_sysreg_masks(kvm, MDCR_EL2, resx);
arch/arm64/kvm/nested.c
1894
if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RME, IMP))
arch/arm64/kvm/nested.c
1896
if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, CNTPOFF)) {
arch/arm64/kvm/nested.c
1898
if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, IMP))
arch/arm64/kvm/nested.c
1902
if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP))
arch/arm64/kvm/nested.c
1904
set_sysreg_masks(kvm, CNTHCTL_EL2, resx);
arch/arm64/kvm/nested.c
1913
set_sysreg_masks(kvm, ICH_HCR_EL2, resx);
arch/arm64/kvm/nested.c
1918
set_sysreg_masks(kvm, VNCR_EL2, resx);
arch/arm64/kvm/nested.c
1923
set_sysreg_masks(kvm, ZCR_EL2, resx);
arch/arm64/kvm/nested.c
1937
write_lock(&vcpu->kvm->mmu_lock);
arch/arm64/kvm/nested.c
1942
write_unlock(&vcpu->kvm->mmu_lock);
arch/arm64/kvm/nested.c
215
r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val));
arch/arm64/kvm/nested.c
242
return __kvm_at_swap_desc(vcpu->kvm, pa, old, new);
arch/arm64/kvm/nested.c
438
static unsigned int vtcr_to_tg0_pgshift(struct kvm *kvm, u64 vtcr)
arch/arm64/kvm/nested.c
441
u64 mmfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1);
arch/arm64/kvm/nested.c
469
static size_t vtcr_to_tg0_pgsize(struct kvm *kvm, u64 vtcr)
arch/arm64/kvm/nested.c
47
void kvm_init_nested(struct kvm *kvm)
arch/arm64/kvm/nested.c
471
return BIT(vtcr_to_tg0_pgshift(kvm, vtcr));
arch/arm64/kvm/nested.c
480
wi->pgshift = vtcr_to_tg0_pgshift(vcpu->kvm, vtcr);
arch/arm64/kvm/nested.c
49
kvm->arch.nested_mmus = NULL;
arch/arm64/kvm/nested.c
50
kvm->arch.nested_mmus_size = 0;
arch/arm64/kvm/nested.c
51
atomic_set(&kvm->arch.vncr_map_count, 0);
arch/arm64/kvm/nested.c
54
static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
arch/arm64/kvm/nested.c
66
return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm));
arch/arm64/kvm/nested.c
667
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
arch/arm64/kvm/nested.c
673
if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) {
arch/arm64/kvm/nested.c
683
switch (vtcr_to_tg0_pgsize(kvm, mmu->tlb_vtcr)) {
arch/arm64/kvm/nested.c
71
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/nested.c
720
void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
arch/arm64/kvm/nested.c
725
write_lock(&kvm->mmu_lock);
arch/arm64/kvm/nested.c
727
for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
arch/arm64/kvm/nested.c
728
struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
arch/arm64/kvm/nested.c
737
write_unlock(&kvm->mmu_lock);
arch/arm64/kvm/nested.c
742
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/nested.c
746
lockdep_assert_held_write(&kvm->mmu_lock);
arch/arm64/kvm/nested.c
75
if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) &&
arch/arm64/kvm/nested.c
769
for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
arch/arm64/kvm/nested.c
770
struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
arch/arm64/kvm/nested.c
791
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/nested.c
795
lockdep_assert_held_write(&vcpu->kvm->mmu_lock);
arch/arm64/kvm/nested.c
806
for (i = kvm->arch.nested_mmus_next;
arch/arm64/kvm/nested.c
807
i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next);
arch/arm64/kvm/nested.c
809
s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size];
arch/arm64/kvm/nested.c
817
kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size;
arch/arm64/kvm/nested.c
870
vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
arch/arm64/kvm/nested.c
873
scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
arch/arm64/kvm/nested.c
893
atomic_dec(&vcpu->kvm->arch.vncr_map_count);
arch/arm64/kvm/nested.c
909
if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu))
arch/arm64/kvm/nested.c
92
num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU;
arch/arm64/kvm/nested.c
930
forward_fault = !kvm_s2_trans_exec_el1(vcpu->kvm, trans);
arch/arm64/kvm/nested.c
932
forward_fault = !kvm_s2_trans_exec_el0(vcpu->kvm, trans);
arch/arm64/kvm/nested.c
94
if (num_mmus > kvm->arch.nested_mmus_size) {
arch/arm64/kvm/nested.c
974
if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) ||
arch/arm64/kvm/nested.c
99
write_lock(&kvm->mmu_lock);
arch/arm64/kvm/nested.c
994
#define kvm_for_each_vncr_tlb(idx, vcpup, tlbp, kvm) \
arch/arm64/kvm/nested.c
995
kvm_for_each_vcpu(idx, vcpup, kvm) \
arch/arm64/kvm/nested.c
999
static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end)
arch/arm64/kvm/pauth.c
199
if (!kvm_has_pauth(vcpu->kvm, PAuth2))
arch/arm64/kvm/pkvm.c
101
kvm->arch.pkvm.handle = 0;
arch/arm64/kvm/pkvm.c
102
kvm->arch.pkvm.is_created = false;
arch/arm64/kvm/pkvm.c
103
free_hyp_memcache(&kvm->arch.pkvm.teardown_mc);
arch/arm64/kvm/pkvm.c
104
free_hyp_memcache(&kvm->arch.pkvm.stage2_teardown_mc);
arch/arm64/kvm/pkvm.c
110
pkvm_handle_t handle = vcpu->kvm->arch.pkvm.handle;
arch/arm64/kvm/pkvm.c
139
static int __pkvm_create_hyp_vm(struct kvm *kvm)
arch/arm64/kvm/pkvm.c
145
if (kvm->created_vcpus < 1)
arch/arm64/kvm/pkvm.c
148
pgd_sz = kvm_pgtable_stage2_pgd_size(kvm->arch.mmu.vtcr);
arch/arm64/kvm/pkvm.c
162
kvm->created_vcpus)));
arch/arm64/kvm/pkvm.c
170
ret = kvm_call_hyp_nvhe(__pkvm_init_vm, kvm, hyp_vm, pgd);
arch/arm64/kvm/pkvm.c
174
kvm->arch.pkvm.is_created = true;
arch/arm64/kvm/pkvm.c
175
kvm->arch.pkvm.stage2_teardown_mc.flags |= HYP_MEMCACHE_ACCOUNT_STAGE2;
arch/arm64/kvm/pkvm.c
186
bool pkvm_hyp_vm_is_created(struct kvm *kvm)
arch/arm64/kvm/pkvm.c
188
return READ_ONCE(kvm->arch.pkvm.is_created);
arch/arm64/kvm/pkvm.c
191
int pkvm_create_hyp_vm(struct kvm *kvm)
arch/arm64/kvm/pkvm.c
199
mutex_lock(&kvm->slots_lock);
arch/arm64/kvm/pkvm.c
200
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/pkvm.c
201
if (!pkvm_hyp_vm_is_created(kvm))
arch/arm64/kvm/pkvm.c
202
ret = __pkvm_create_hyp_vm(kvm);
arch/arm64/kvm/pkvm.c
203
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/pkvm.c
204
mutex_unlock(&kvm->slots_lock);
arch/arm64/kvm/pkvm.c
213
mutex_lock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/pkvm.c
216
mutex_unlock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/pkvm.c
221
void pkvm_destroy_hyp_vm(struct kvm *kvm)
arch/arm64/kvm/pkvm.c
223
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/pkvm.c
224
__pkvm_destroy_hyp_vm(kvm);
arch/arm64/kvm/pkvm.c
225
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/pkvm.c
228
int pkvm_init_host_vm(struct kvm *kvm, unsigned long type)
arch/arm64/kvm/pkvm.c
233
if (pkvm_hyp_vm_is_created(kvm))
arch/arm64/kvm/pkvm.c
237
if (kvm->arch.pkvm.handle)
arch/arm64/kvm/pkvm.c
245
kvm->arch.pkvm.handle = ret;
arch/arm64/kvm/pkvm.c
246
kvm->arch.pkvm.is_protected = protected;
arch/arm64/kvm/pkvm.c
339
struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);
arch/arm64/kvm/pkvm.c
340
pkvm_handle_t handle = kvm->arch.pkvm.handle;
arch/arm64/kvm/pkvm.c
355
account_locked_vm(kvm->mm, 1, false);
arch/arm64/kvm/pkvm.c
365
struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);
arch/arm64/kvm/pkvm.c
366
pkvm_handle_t handle = kvm->arch.pkvm.handle;
arch/arm64/kvm/pkvm.c
385
struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);
arch/arm64/kvm/pkvm.c
386
pkvm_handle_t handle = kvm->arch.pkvm.handle;
arch/arm64/kvm/pkvm.c
391
if (pkvm_hyp_vm_is_created(kvm) && !kvm->arch.pkvm.is_dying) {
arch/arm64/kvm/pkvm.c
393
kvm->arch.pkvm.is_dying = true;
arch/arm64/kvm/pkvm.c
396
if (kvm_vm_is_protected(kvm))
arch/arm64/kvm/pkvm.c
412
struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);
arch/arm64/kvm/pkvm.c
420
lockdep_assert_held_write(&kvm->mmu_lock);
arch/arm64/kvm/pkvm.c
423
if (kvm_vm_is_protected(kvm)) {
arch/arm64/kvm/pkvm.c
483
struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);
arch/arm64/kvm/pkvm.c
485
if (WARN_ON(kvm_vm_is_protected(kvm)))
arch/arm64/kvm/pkvm.c
488
lockdep_assert_held_write(&kvm->mmu_lock);
arch/arm64/kvm/pkvm.c
495
struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);
arch/arm64/kvm/pkvm.c
496
pkvm_handle_t handle = kvm->arch.pkvm.handle;
arch/arm64/kvm/pkvm.c
500
if (WARN_ON(kvm_vm_is_protected(kvm)))
arch/arm64/kvm/pkvm.c
503
lockdep_assert_held(&kvm->mmu_lock);
arch/arm64/kvm/pkvm.c
516
struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);
arch/arm64/kvm/pkvm.c
519
lockdep_assert_held(&kvm->mmu_lock);
arch/arm64/kvm/pkvm.c
529
struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);
arch/arm64/kvm/pkvm.c
530
pkvm_handle_t handle = kvm->arch.pkvm.handle;
arch/arm64/kvm/pkvm.c
534
if (WARN_ON(kvm_vm_is_protected(kvm)))
arch/arm64/kvm/pkvm.c
537
lockdep_assert_held(&kvm->mmu_lock);
arch/arm64/kvm/pkvm.c
88
static void __pkvm_destroy_hyp_vm(struct kvm *kvm)
arch/arm64/kvm/pkvm.c
90
if (pkvm_hyp_vm_is_created(kvm)) {
arch/arm64/kvm/pkvm.c
92
kvm->arch.pkvm.handle));
arch/arm64/kvm/pkvm.c
93
} else if (kvm->arch.pkvm.handle) {
arch/arm64/kvm/pkvm.c
98
kvm_call_hyp_nvhe(__pkvm_unreserve_vm, kvm->arch.pkvm.handle);
arch/arm64/kvm/pmu-emul.c
1007
u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm)
arch/arm64/kvm/pmu-emul.c
1009
struct arm_pmu *arm_pmu = kvm->arch.arm_pmu;
arch/arm64/kvm/pmu-emul.c
1025
static void kvm_arm_set_nr_counters(struct kvm *kvm, unsigned int nr)
arch/arm64/kvm/pmu-emul.c
1027
kvm->arch.nr_pmu_counters = nr;
arch/arm64/kvm/pmu-emul.c
1030
if (test_bit(KVM_ARM_VCPU_HAS_EL2, kvm->arch.vcpu_features)) {
arch/arm64/kvm/pmu-emul.c
1034
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/arm64/kvm/pmu-emul.c
1037
val |= FIELD_PREP(MDCR_EL2_HPMN, kvm->arch.nr_pmu_counters);
arch/arm64/kvm/pmu-emul.c
1043
static void kvm_arm_set_pmu(struct kvm *kvm, struct arm_pmu *arm_pmu)
arch/arm64/kvm/pmu-emul.c
1045
lockdep_assert_held(&kvm->arch.config_lock);
arch/arm64/kvm/pmu-emul.c
1047
kvm->arch.arm_pmu = arm_pmu;
arch/arm64/kvm/pmu-emul.c
1048
kvm_arm_set_nr_counters(kvm, kvm_arm_pmu_get_max_counters(kvm));
arch/arm64/kvm/pmu-emul.c
1063
int kvm_arm_set_default_pmu(struct kvm *kvm)
arch/arm64/kvm/pmu-emul.c
1070
kvm_arm_set_pmu(kvm, arm_pmu);
arch/arm64/kvm/pmu-emul.c
1076
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/pmu-emul.c
1081
lockdep_assert_held(&kvm->arch.config_lock);
arch/arm64/kvm/pmu-emul.c
1087
if (kvm_vm_has_ran_once(kvm) ||
arch/arm64/kvm/pmu-emul.c
1088
(kvm->arch.pmu_filter && kvm->arch.arm_pmu != arm_pmu)) {
arch/arm64/kvm/pmu-emul.c
1093
kvm_arm_set_pmu(kvm, arm_pmu);
arch/arm64/kvm/pmu-emul.c
1094
cpumask_copy(kvm->arch.supported_cpus, &arm_pmu->supported_cpus);
arch/arm64/kvm/pmu-emul.c
1106
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/pmu-emul.c
1108
if (!kvm->arch.arm_pmu)
arch/arm64/kvm/pmu-emul.c
1111
if (n > kvm_arm_pmu_get_max_counters(kvm))
arch/arm64/kvm/pmu-emul.c
1114
kvm_arm_set_nr_counters(kvm, n);
arch/arm64/kvm/pmu-emul.c
1120
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/pmu-emul.c
1122
lockdep_assert_held(&kvm->arch.config_lock);
arch/arm64/kvm/pmu-emul.c
1135
if (!irqchip_in_kernel(kvm))
arch/arm64/kvm/pmu-emul.c
1142
if (!(irq_is_ppi(vcpu->kvm, irq) || irq_is_spi(vcpu->kvm, irq)))
arch/arm64/kvm/pmu-emul.c
1145
if (!pmu_irq_is_valid(kvm, irq))
arch/arm64/kvm/pmu-emul.c
1178
if (kvm_vm_has_ran_once(kvm))
arch/arm64/kvm/pmu-emul.c
1181
if (!kvm->arch.pmu_filter) {
arch/arm64/kvm/pmu-emul.c
1182
kvm->arch.pmu_filter = bitmap_alloc(nr_events, GFP_KERNEL_ACCOUNT);
arch/arm64/kvm/pmu-emul.c
1183
if (!kvm->arch.pmu_filter)
arch/arm64/kvm/pmu-emul.c
1193
bitmap_zero(kvm->arch.pmu_filter, nr_events);
arch/arm64/kvm/pmu-emul.c
1195
bitmap_fill(kvm->arch.pmu_filter, nr_events);
arch/arm64/kvm/pmu-emul.c
1199
bitmap_set(kvm->arch.pmu_filter, filter.base_event, filter.nevents);
arch/arm64/kvm/pmu-emul.c
1201
bitmap_clear(kvm->arch.pmu_filter, filter.base_event, filter.nevents);
arch/arm64/kvm/pmu-emul.c
1237
if (!irqchip_in_kernel(vcpu->kvm))
arch/arm64/kvm/pmu-emul.c
1300
u64 n = vcpu->kvm->arch.nr_pmu_counters;
arch/arm64/kvm/pmu-emul.c
283
n = vcpu->kvm->arch.nr_pmu_counters;
arch/arm64/kvm/pmu-emul.c
400
if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
arch/arm64/kvm/pmu-emul.c
403
WARN_ON(kvm_vgic_inject_irq(vcpu->kvm, vcpu, pmu->irq_num,
arch/arm64/kvm/pmu-emul.c
485
type &= kvm_pmu_event_mask(vcpu->kvm);
arch/arm64/kvm/pmu-emul.c
584
if (!kvm_has_feat(vcpu->kvm, ID_AA64DFR0_EL1, PMUVer, V3P5))
arch/arm64/kvm/pmu-emul.c
59
static u32 kvm_pmu_event_mask(struct kvm *kvm)
arch/arm64/kvm/pmu-emul.c
61
u64 dfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1);
arch/arm64/kvm/pmu-emul.c
652
static int kvm_map_pmu_event(struct kvm *kvm, unsigned int eventsel)
arch/arm64/kvm/pmu-emul.c
654
struct arm_pmu *pmu = kvm->arch.arm_pmu;
arch/arm64/kvm/pmu-emul.c
67
u64 kvm_pmu_evtyper_mask(struct kvm *kvm)
arch/arm64/kvm/pmu-emul.c
673
struct arm_pmu *arm_pmu = vcpu->kvm->arch.arm_pmu;
arch/arm64/kvm/pmu-emul.c
685
eventsel = evtreg & kvm_pmu_event_mask(vcpu->kvm);
arch/arm64/kvm/pmu-emul.c
699
if (vcpu->kvm->arch.pmu_filter &&
arch/arm64/kvm/pmu-emul.c
70
kvm_pmu_event_mask(kvm);
arch/arm64/kvm/pmu-emul.c
700
!test_bit(eventsel, vcpu->kvm->arch.pmu_filter))
arch/arm64/kvm/pmu-emul.c
707
eventsel = kvm_map_pmu_event(vcpu->kvm, eventsel);
arch/arm64/kvm/pmu-emul.c
72
if (kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL2, IMP))
arch/arm64/kvm/pmu-emul.c
75
if (kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL3, IMP))
arch/arm64/kvm/pmu-emul.c
769
__vcpu_assign_sys_reg(vcpu, reg, (data & kvm_pmu_evtyper_mask(vcpu->kvm)));
arch/arm64/kvm/pmu-emul.c
868
struct arm_pmu *cpu_pmu = vcpu->kvm->arch.arm_pmu;
arch/arm64/kvm/pmu-emul.c
869
unsigned long *bmap = vcpu->kvm->arch.pmu_filter;
arch/arm64/kvm/pmu-emul.c
884
nr_events = kvm_pmu_event_mask(vcpu->kvm) + 1;
arch/arm64/kvm/pmu-emul.c
92
kvm_has_feat(vcpu->kvm, ID_AA64DFR0_EL1, PMUVer, V3P5));
arch/arm64/kvm/pmu-emul.c
921
if (irqchip_in_kernel(vcpu->kvm)) {
arch/arm64/kvm/pmu-emul.c
929
if (!irq_is_ppi(vcpu->kvm, irq) &&
arch/arm64/kvm/pmu-emul.c
930
!vgic_valid_spi(vcpu->kvm, irq))
arch/arm64/kvm/pmu-emul.c
941
if (irqchip_in_kernel(vcpu->kvm)) {
arch/arm64/kvm/pmu-emul.c
949
if (!vgic_initialized(vcpu->kvm))
arch/arm64/kvm/pmu-emul.c
953
if (!vgic_is_v5(vcpu->kvm))
arch/arm64/kvm/pmu-emul.c
978
static bool pmu_irq_is_valid(struct kvm *kvm, int irq)
arch/arm64/kvm/pmu-emul.c
984
if (vgic_is_v5(kvm) && irq != KVM_ARMV8_PMU_GICV5_IRQ)
arch/arm64/kvm/pmu-emul.c
987
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/arm64/kvm/pmu-emul.c
991
if (irq_is_ppi(vcpu->kvm, irq)) {
arch/arm64/kvm/psci.c
128
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/psci.c
149
kvm_for_each_vcpu(i, tmp, kvm) {
arch/arm64/kvm/psci.c
178
kvm_for_each_vcpu(i, tmp, vcpu->kvm) {
arch/arm64/kvm/psci.c
183
kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP);
arch/arm64/kvm/psci.c
325
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/psci.c
359
if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags))
arch/arm64/kvm/psci.c
383
if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) {
arch/arm64/kvm/psci.c
63
struct kvm *kvm = source_vcpu->kvm;
arch/arm64/kvm/psci.c
72
vcpu = kvm_mpidr_to_vcpu(kvm, cpu_id);
arch/arm64/kvm/ptdump.c
144
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
arch/arm64/kvm/ptdump.c
158
write_lock(&kvm->mmu_lock);
arch/arm64/kvm/ptdump.c
160
write_unlock(&kvm->mmu_lock);
arch/arm64/kvm/ptdump.c
168
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
arch/arm64/kvm/ptdump.c
172
if (!kvm_get_kvm_safe(kvm))
arch/arm64/kvm/ptdump.c
187
kvm_put_kvm(kvm);
arch/arm64/kvm/ptdump.c
193
struct kvm *kvm = kvm_s2_mmu_to_kvm(m->i_private);
arch/arm64/kvm/ptdump.c
197
kvm_put_kvm(kvm);
arch/arm64/kvm/ptdump.c
229
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
arch/arm64/kvm/ptdump.c
233
if (!kvm_get_kvm_safe(kvm))
arch/arm64/kvm/ptdump.c
240
kvm_put_kvm(kvm);
arch/arm64/kvm/ptdump.c
256
struct kvm *kvm = kvm_s2_mmu_to_kvm(m->i_private);
arch/arm64/kvm/ptdump.c
258
kvm_put_kvm(kvm);
arch/arm64/kvm/ptdump.c
298
void kvm_s2_ptdump_create_debugfs(struct kvm *kvm)
arch/arm64/kvm/ptdump.c
300
debugfs_create_file("stage2_page_tables", 0400, kvm->debugfs_dentry,
arch/arm64/kvm/ptdump.c
301
&kvm->arch.mmu, &kvm_ptdump_guest_fops);
arch/arm64/kvm/ptdump.c
302
debugfs_create_file("ipa_range", 0400, kvm->debugfs_dentry,
arch/arm64/kvm/ptdump.c
303
&kvm->arch.mmu, &kvm_pgtable_range_fops);
arch/arm64/kvm/ptdump.c
304
debugfs_create_file("stage2_levels", 0400, kvm->debugfs_dentry,
arch/arm64/kvm/ptdump.c
305
&kvm->arch.mmu, &kvm_pgtable_levels_fops);
arch/arm64/kvm/ptdump.c
307
kvm->arch.debugfs_nv_dentry = debugfs_create_dir("nested", kvm->debugfs_dentry);
arch/arm64/kvm/pvtime.c
15
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/pvtime.c
25
idx = srcu_read_lock(&kvm->srcu);
arch/arm64/kvm/pvtime.c
26
if (!kvm_get_guest(kvm, base + offset, steal)) {
arch/arm64/kvm/pvtime.c
30
kvm_put_guest(kvm, base + offset, cpu_to_le64(steal));
arch/arm64/kvm/pvtime.c
32
srcu_read_unlock(&kvm->srcu, idx);
arch/arm64/kvm/pvtime.c
54
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/pvtime.c
65
kvm_write_guest_lock(kvm, base, &init_values, sizeof(init_values));
arch/arm64/kvm/pvtime.c
79
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/pvtime.c
96
idx = srcu_read_lock(&kvm->srcu);
arch/arm64/kvm/pvtime.c
97
if (kvm_is_error_hva(gfn_to_hva(kvm, ipa >> PAGE_SHIFT)))
arch/arm64/kvm/pvtime.c
99
srcu_read_unlock(&kvm->srcu, idx);
arch/arm64/kvm/reset.c
275
u32 kvm_get_pa_bits(struct kvm *kvm)
arch/arm64/kvm/reset.c
88
set_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &vcpu->kvm->arch.flags);
arch/arm64/kvm/sys_regs.c
1015
u8 n = vcpu->kvm->arch.nr_pmu_counters;
arch/arm64/kvm/sys_regs.c
1041
__vcpu_rmw_sys_reg(vcpu, r->reg, &=, kvm_pmu_evtyper_mask(vcpu->kvm));
arch/arm64/kvm/sys_regs.c
1436
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/sys_regs.c
1438
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/sys_regs.c
1445
if (!kvm_vm_has_ran_once(kvm) &&
arch/arm64/kvm/sys_regs.c
1447
new_n <= kvm_arm_pmu_get_max_counters(kvm))
arch/arm64/kvm/sys_regs.c
1448
kvm->arch.nr_pmu_counters = new_n;
arch/arm64/kvm/sys_regs.c
1450
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/sys_regs.c
1706
if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags))
arch/arm64/kvm/sys_regs.c
1710
if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags))
arch/arm64/kvm/sys_regs.c
1909
val = limit_nv_id_reg(vcpu->kvm, id, val);
arch/arm64/kvm/sys_regs.c
1922
return kvm_read_vm_id_reg(vcpu->kvm, reg_to_encoding(r));
arch/arm64/kvm/sys_regs.c
2031
if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SME, IMP))
arch/arm64/kvm/sys_regs.c
2040
if (kvm_has_fpmr(vcpu->kvm))
arch/arm64/kvm/sys_regs.c
2089
if (!kvm_has_mte(vcpu->kvm)) {
arch/arm64/kvm/sys_regs.c
2116
if (!kvm_has_mte(vcpu->kvm)) {
arch/arm64/kvm/sys_regs.c
2408
if (kvm_vm_has_ran_once(vcpu->kvm)) {
arch/arm64/kvm/sys_regs.c
2413
mutex_lock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/sys_regs.c
2415
mutex_unlock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/sys_regs.c
2426
mutex_lock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/sys_regs.c
2432
if (kvm_vm_has_ran_once(vcpu->kvm)) {
arch/arm64/kvm/sys_regs.c
2438
mutex_unlock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/sys_regs.c
2444
kvm_set_vm_id_reg(vcpu->kvm, id, val);
arch/arm64/kvm/sys_regs.c
2446
mutex_unlock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/sys_regs.c
2460
void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val)
arch/arm64/kvm/sys_regs.c
2462
u64 *p = __vm_id_reg(&kvm->arch, reg);
arch/arm64/kvm/sys_regs.c
2464
lockdep_assert_held(&kvm->arch.config_lock);
arch/arm64/kvm/sys_regs.c
2466
if (KVM_BUG_ON(kvm_vm_has_ran_once(kvm) || !p, kvm))
arch/arm64/kvm/sys_regs.c
2491
p->regval = kvm_read_vm_id_reg(vcpu->kvm, SYS_CTR_EL0);
arch/arm64/kvm/sys_regs.c
2553
if (kvm_has_mte(vcpu->kvm))
arch/arm64/kvm/sys_regs.c
2606
if (kvm_has_mte(vcpu->kvm))
arch/arm64/kvm/sys_regs.c
2837
kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
arch/arm64/kvm/sys_regs.c
2846
if (kvm_has_sctlr2(vcpu->kvm))
arch/arm64/kvm/sys_regs.c
2926
if (kvm_has_s1poe(vcpu->kvm))
arch/arm64/kvm/sys_regs.c
2941
if (kvm_has_tcr2(vcpu->kvm))
arch/arm64/kvm/sys_regs.c
2957
kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, FGT, FGT2))
arch/arm64/kvm/sys_regs.c
2967
kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, FGT, IMP))
arch/arm64/kvm/sys_regs.c
2976
if (kvm_has_s1pie(vcpu->kvm))
arch/arm64/kvm/sys_regs.c
3017
if (hpmn > vcpu->kvm->arch.nr_pmu_counters) {
arch/arm64/kvm/sys_regs.c
3018
hpmn = vcpu->kvm->arch.nr_pmu_counters;
arch/arm64/kvm/sys_regs.c
3038
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/sys_regs.c
3046
if (!(kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, V1P1) ||
arch/arm64/kvm/sys_regs.c
3047
(kvm_has_feat_enum(kvm, ID_AA64PFR0_EL1, RAS, IMP) &&
arch/arm64/kvm/sys_regs.c
3048
kvm_has_feat(kvm, ID_AA64PFR1_EL1, RAS_frac, RASv1p1)))) {
arch/arm64/kvm/sys_regs.c
3054
if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) {
arch/arm64/kvm/sys_regs.c
3089
if (test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &vcpu->kvm->arch.flags))
arch/arm64/kvm/sys_regs.c
3131
KVM_BUG_ON(1, vcpu->kvm);
arch/arm64/kvm/sys_regs.c
3139
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/sys_regs.c
3142
guard(mutex)(&kvm->arch.config_lock);
arch/arm64/kvm/sys_regs.c
3148
if (!test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags))
arch/arm64/kvm/sys_regs.c
3155
if (kvm_vm_has_ran_once(kvm))
arch/arm64/kvm/sys_regs.c
3165
kvm_set_vm_id_reg(kvm, reg_to_encoding(r), val);
arch/arm64/kvm/sys_regs.c
3180
__vcpu_assign_sys_reg(vcpu, r->reg, vcpu->kvm->arch.nr_pmu_counters);
arch/arm64/kvm/sys_regs.c
3181
return vcpu->kvm->arch.nr_pmu_counters;
arch/arm64/kvm/sys_regs.c
3900
!kvm_has_feat(vcpu->kvm, ID_AA64ISAR2_EL1, ATS1A, IMP)) {
arch/arm64/kvm/sys_regs.c
3924
struct kvm *kvm = vpcu->kvm;
arch/arm64/kvm/sys_regs.c
3928
!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))
arch/arm64/kvm/sys_regs.c
3932
!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
arch/arm64/kvm/sys_regs.c
3946
write_lock(&vcpu->kvm->mmu_lock);
arch/arm64/kvm/sys_regs.c
3952
kvm_nested_s2_unmap(vcpu->kvm, true);
arch/arm64/kvm/sys_regs.c
3954
write_unlock(&vcpu->kvm->mmu_lock);
arch/arm64/kvm/sys_regs.c
3961
struct kvm *kvm = vpcu->kvm;
arch/arm64/kvm/sys_regs.c
3966
!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))
arch/arm64/kvm/sys_regs.c
3970
!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
arch/arm64/kvm/sys_regs.c
3974
!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
arch/arm64/kvm/sys_regs.c
3978
!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
arch/arm64/kvm/sys_regs.c
4040
limit = BIT_ULL(kvm_get_pa_bits(vcpu->kvm));
arch/arm64/kvm/sys_regs.c
4042
kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr),
arch/arm64/kvm/sys_regs.c
4071
kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr),
arch/arm64/kvm/sys_regs.c
4118
kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr),
arch/arm64/kvm/sys_regs.c
4182
kvm_s2_mmu_iterate_by_vmid(vcpu->kvm,
arch/arm64/kvm/sys_regs.c
4388
u64 dfr = kvm_read_vm_id_reg(vcpu->kvm, SYS_ID_AA64DFR0_EL1);
arch/arm64/kvm/sys_regs.c
4389
u32 el3 = kvm_has_feat(vcpu->kvm, ID_AA64PFR0_EL1, EL3, IMP);
arch/arm64/kvm/sys_regs.c
5098
static const struct sys_reg_desc *idregs_debug_find(struct kvm *kvm, loff_t pos)
arch/arm64/kvm/sys_regs.c
5117
struct kvm *kvm = s->private;
arch/arm64/kvm/sys_regs.c
5119
if (!test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags))
arch/arm64/kvm/sys_regs.c
5122
return (void *)idregs_debug_find(kvm, *pos);
arch/arm64/kvm/sys_regs.c
5127
struct kvm *kvm = s->private;
arch/arm64/kvm/sys_regs.c
5131
return (void *)idregs_debug_find(kvm, *pos);
arch/arm64/kvm/sys_regs.c
5141
struct kvm *kvm = s->private;
arch/arm64/kvm/sys_regs.c
5147
desc->name, kvm_read_vm_id_reg(kvm, reg_to_encoding(desc)));
arch/arm64/kvm/sys_regs.c
5161
static const struct sys_reg_desc *sr_resx_find(struct kvm *kvm, loff_t pos)
arch/arm64/kvm/sys_regs.c
5180
struct kvm *kvm = s->private;
arch/arm64/kvm/sys_regs.c
5182
if (!kvm->arch.sysreg_masks)
arch/arm64/kvm/sys_regs.c
5185
return (void *)sr_resx_find(kvm, *pos);
arch/arm64/kvm/sys_regs.c
5190
struct kvm *kvm = s->private;
arch/arm64/kvm/sys_regs.c
5194
return (void *)sr_resx_find(kvm, *pos);
arch/arm64/kvm/sys_regs.c
5204
struct kvm *kvm = s->private;
arch/arm64/kvm/sys_regs.c
5210
resx = kvm_get_sysreg_resx(kvm, desc->reg);
arch/arm64/kvm/sys_regs.c
5227
void kvm_sys_regs_create_debugfs(struct kvm *kvm)
arch/arm64/kvm/sys_regs.c
5229
debugfs_create_file("idregs", 0444, kvm->debugfs_dentry, kvm,
arch/arm64/kvm/sys_regs.c
5231
debugfs_create_file("resx", 0444, kvm->debugfs_dentry, kvm,
arch/arm64/kvm/sys_regs.c
5238
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/sys_regs.c
5240
if (test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags))
arch/arm64/kvm/sys_regs.c
5243
kvm_set_vm_id_reg(kvm, id, reg->reset(vcpu, reg));
arch/arm64/kvm/sys_regs.c
526
if (!kvm_has_mte(vcpu->kvm))
arch/arm64/kvm/sys_regs.c
5264
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/sys_regs.c
5284
set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags);
arch/arm64/kvm/sys_regs.c
5655
int kvm_vm_ioctl_get_reg_writable_masks(struct kvm *kvm, struct reg_mask_range *range)
arch/arm64/kvm/sys_regs.c
5692
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/sys_regs.c
5708
kvm_read_vm_id_reg(kvm, SYS_CTR_EL0) == read_sanitised_ftr_reg(SYS_CTR_EL0))
arch/arm64/kvm/sys_regs.c
5716
if (kvm_has_mte(vcpu->kvm))
arch/arm64/kvm/sys_regs.c
5726
if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
arch/arm64/kvm/sys_regs.c
5732
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/sys_regs.c
5734
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/sys_regs.c
5739
if (test_bit(KVM_ARCH_FLAG_FGU_INITIALIZED, &kvm->arch.flags))
arch/arm64/kvm/sys_regs.c
5742
compute_fgu(kvm, HFGRTR_GROUP);
arch/arm64/kvm/sys_regs.c
5743
compute_fgu(kvm, HFGITR_GROUP);
arch/arm64/kvm/sys_regs.c
5744
compute_fgu(kvm, HDFGRTR_GROUP);
arch/arm64/kvm/sys_regs.c
5745
compute_fgu(kvm, HAFGRTR_GROUP);
arch/arm64/kvm/sys_regs.c
5746
compute_fgu(kvm, HFGRTR2_GROUP);
arch/arm64/kvm/sys_regs.c
5747
compute_fgu(kvm, HFGITR2_GROUP);
arch/arm64/kvm/sys_regs.c
5748
compute_fgu(kvm, HDFGRTR2_GROUP);
arch/arm64/kvm/sys_regs.c
5749
compute_fgu(kvm, ICH_HFGRTR_GROUP);
arch/arm64/kvm/sys_regs.c
5750
compute_fgu(kvm, ICH_HFGITR_GROUP);
arch/arm64/kvm/sys_regs.c
5752
set_bit(KVM_ARCH_FLAG_FGU_INITIALIZED, &kvm->arch.flags);
arch/arm64/kvm/sys_regs.c
5754
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/sys_regs.c
5767
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/sys_regs.c
5769
guard(mutex)(&kvm->arch.config_lock);
arch/arm64/kvm/sys_regs.c
5777
if (kvm_vm_has_ran_once(kvm))
arch/arm64/kvm/sys_regs.c
5784
if (!irqchip_in_kernel(kvm)) {
arch/arm64/kvm/sys_regs.c
5787
val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1) & ~ID_AA64PFR0_EL1_GIC;
arch/arm64/kvm/sys_regs.c
5788
kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, val);
arch/arm64/kvm/sys_regs.c
5789
val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR2_EL1) & ~ID_AA64PFR2_EL1_GCIE;
arch/arm64/kvm/sys_regs.c
5790
kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR2_EL1, val);
arch/arm64/kvm/sys_regs.c
5791
val = kvm_read_vm_id_reg(kvm, SYS_ID_PFR1_EL1) & ~ID_PFR1_EL1_GIC;
arch/arm64/kvm/sys_regs.c
5792
kvm_set_vm_id_reg(kvm, SYS_ID_PFR1_EL1, val);
arch/arm64/kvm/sys_regs.c
5822
kvm_vgic_finalize_idregs(kvm);
arch/arm64/kvm/sys_regs.c
608
if (!kvm_has_gicv3(vcpu->kvm))
arch/arm64/kvm/sys_regs.c
654
if (!kvm_has_gicv3(vcpu->kvm))
arch/arm64/kvm/sys_regs.c
673
if (!kvm_has_gicv3(vcpu->kvm))
arch/arm64/kvm/sys_regs.c
725
unsigned long *mask = vcpu->kvm->arch.vgic.gicv5_vm.vgic_ppi_mask;
arch/arm64/kvm/sys_regs.c
751
for_each_visible_v5_ppi(i, vcpu->kvm) {
arch/arm64/kvm/sys_regs.c
760
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/sys_regs.c
788
if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, LO, IMP))
arch/arm64/kvm/sys_regs.c
901
KVM_BUG_ON(1, vcpu->kvm);
arch/arm64/kvm/trace_arm.h
10
#define TRACE_SYSTEM kvm
arch/arm64/kvm/trace_handle_exit.h
9
#define TRACE_SYSTEM kvm
arch/arm64/kvm/vgic/trace.h
33
#define TRACE_INCLUDE_PATH ../../arch/arm64/kvm/vgic
arch/arm64/kvm/vgic/trace.h
8
#define TRACE_SYSTEM kvm
arch/arm64/kvm/vgic/vgic-debug.c
104
struct kvm *kvm = s->private;
arch/arm64/kvm/vgic/vgic-debug.c
111
iter_init(kvm, iter, *pos);
arch/arm64/kvm/vgic/vgic-debug.c
123
struct kvm *kvm = s->private;
arch/arm64/kvm/vgic/vgic-debug.c
127
iter_next(kvm, iter);
arch/arm64/kvm/vgic/vgic-debug.c
149
struct kvm *kvm = s->private;
arch/arm64/kvm/vgic/vgic-debug.c
156
seq_printf(s, "nr_lpis:\t%d\n", vgic_count_lpis(kvm));
arch/arm64/kvm/vgic/vgic-debug.c
236
struct kvm *kvm = s->private;
arch/arm64/kvm/vgic/vgic-debug.c
243
print_dist_state(s, &kvm->arch.vgic, iter);
arch/arm64/kvm/vgic/vgic-debug.c
247
if (!kvm->arch.vgic.initialized)
arch/arm64/kvm/vgic/vgic-debug.c
251
vcpu = kvm_get_vcpu(kvm, iter->vcpu_id);
arch/arm64/kvm/vgic/vgic-debug.c
256
irq = vgic_get_irq(kvm, iter->intid);
arch/arm64/kvm/vgic/vgic-debug.c
265
vgic_put_irq(kvm, irq);
arch/arm64/kvm/vgic/vgic-debug.c
278
void vgic_debug_init(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-debug.c
280
debugfs_create_file("vgic-state", 0444, kvm->debugfs_dentry, kvm,
arch/arm64/kvm/vgic/vgic-debug.c
284
void vgic_debug_destroy(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-debug.c
33
static void iter_next(struct kvm *kvm, struct vgic_state_iter *iter)
arch/arm64/kvm/vgic/vgic-debug.c
35
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-debug.c
505
debugfs_create_file(name, 0444, dev->kvm->debugfs_dentry, its, &vgic_its_debug_fops);
arch/arm64/kvm/vgic/vgic-debug.c
64
static int vgic_count_lpis(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-debug.c
66
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-debug.c
79
static void iter_init(struct kvm *kvm, struct vgic_state_iter *iter,
arch/arm64/kvm/vgic/vgic-debug.c
82
int nr_cpus = atomic_read(&kvm->online_vcpus);
arch/arm64/kvm/vgic/vgic-debug.c
87
iter->nr_spis = kvm->arch.vgic.nr_spis;
arch/arm64/kvm/vgic/vgic-debug.c
91
iter_next(kvm, iter);
arch/arm64/kvm/vgic/vgic-init.c
102
if (kvm_trylock_all_vcpus(kvm))
arch/arm64/kvm/vgic/vgic-init.c
109
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
118
if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
arch/arm64/kvm/vgic/vgic-init.c
121
if (irqchip_in_kernel(kvm)) {
arch/arm64/kvm/vgic/vgic-init.c
126
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/arm64/kvm/vgic/vgic-init.c
133
kvm->max_vcpus = VGIC_V2_MAX_CPUS;
arch/arm64/kvm/vgic/vgic-init.c
135
kvm->max_vcpus = VGIC_V3_MAX_CPUS;
arch/arm64/kvm/vgic/vgic-init.c
137
kvm->max_vcpus = min(VGIC_V5_MAX_CPUS,
arch/arm64/kvm/vgic/vgic-init.c
140
if (atomic_read(&kvm->online_vcpus) > kvm->max_vcpus) {
arch/arm64/kvm/vgic/vgic-init.c
145
kvm->arch.vgic.in_kernel = true;
arch/arm64/kvm/vgic/vgic-init.c
146
kvm->arch.vgic.vgic_model = type;
arch/arm64/kvm/vgic/vgic-init.c
147
kvm->arch.vgic.implementation_rev = KVM_VGIC_IMP_REV_LATEST;
arch/arm64/kvm/vgic/vgic-init.c
148
kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
arch/arm64/kvm/vgic/vgic-init.c
152
kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
arch/arm64/kvm/vgic/vgic-init.c
155
INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
arch/arm64/kvm/vgic/vgic-init.c
163
kvm_vgic_finalize_idregs(kvm);
arch/arm64/kvm/vgic/vgic-init.c
165
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/arm64/kvm/vgic/vgic-init.c
172
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/arm64/kvm/vgic/vgic-init.c
178
kvm->arch.vgic.vgic_model = 0;
arch/arm64/kvm/vgic/vgic-init.c
183
kvm->arch.vgic.nassgicap = system_supports_direct_sgis();
arch/arm64/kvm/vgic/vgic-init.c
192
kvm_timer_init_vm(kvm);
arch/arm64/kvm/vgic/vgic-init.c
195
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
196
kvm_unlock_all_vcpus(kvm);
arch/arm64/kvm/vgic/vgic-init.c
207
static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis)
arch/arm64/kvm/vgic/vgic-init.c
209
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-init.c
210
struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
arch/arm64/kvm/vgic/vgic-init.c
260
guard(mutex)(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
267
if (vcpu->kvm->arch.vgic.mi_intid == 0)
arch/arm64/kvm/vgic/vgic-init.c
268
vcpu->kvm->arch.vgic.mi_intid = DEFAULT_MI_INTID;
arch/arm64/kvm/vgic/vgic-init.c
269
ret = kvm_vgic_set_owner(vcpu, vcpu->kvm->arch.vgic.mi_intid, vcpu);
arch/arm64/kvm/vgic/vgic-init.c
322
lockdep_assert_held(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
327
if (vgic_is_v5(vcpu->kvm))
arch/arm64/kvm/vgic/vgic-init.c
348
if (vgic_is_v5(vcpu->kvm))
arch/arm64/kvm/vgic/vgic-init.c
361
mutex_lock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
363
mutex_unlock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
380
struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-init.c
389
if (!irqchip_in_kernel(vcpu->kvm))
arch/arm64/kvm/vgic/vgic-init.c
401
mutex_lock(&vcpu->kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-init.c
403
mutex_unlock(&vcpu->kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-init.c
410
const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-init.c
429
int vgic_init(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-init.c
431
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-init.c
436
lockdep_assert_held(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
438
if (vgic_initialized(kvm))
arch/arm64/kvm/vgic/vgic-init.c
442
if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
arch/arm64/kvm/vgic/vgic-init.c
445
if (!vgic_is_v5(kvm)) {
arch/arm64/kvm/vgic/vgic-init.c
450
ret = kvm_vgic_dist_init(kvm, dist->nr_spis);
arch/arm64/kvm/vgic/vgic-init.c
458
if (vgic_supports_direct_irqs(kvm)) {
arch/arm64/kvm/vgic/vgic-init.c
459
ret = vgic_v4_init(kvm);
arch/arm64/kvm/vgic/vgic-init.c
464
ret = vgic_v5_init(kvm);
arch/arm64/kvm/vgic/vgic-init.c
469
kvm_for_each_vcpu(idx, vcpu, kvm)
arch/arm64/kvm/vgic/vgic-init.c
472
ret = kvm_vgic_setup_default_irq_routing(kvm);
arch/arm64/kvm/vgic/vgic-init.c
476
vgic_debug_init(kvm);
arch/arm64/kvm/vgic/vgic-init.c
482
static void kvm_vgic_dist_destroy(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-init.c
484
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-init.c
497
vgic_v3_free_redist_region(kvm, rdreg);
arch/arm64/kvm/vgic/vgic-init.c
503
if (vgic_supports_direct_irqs(kvm))
arch/arm64/kvm/vgic/vgic-init.c
504
vgic_v4_teardown(kvm);
arch/arm64/kvm/vgic/vgic-init.c
52
void kvm_vgic_early_init(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-init.c
523
if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
arch/arm64/kvm/vgic/vgic-init.c
54
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-init.c
540
if (kvm_get_vcpu_by_id(vcpu->kvm, vcpu->vcpu_id) != vcpu)
arch/arm64/kvm/vgic/vgic-init.c
549
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/vgic/vgic-init.c
551
mutex_lock(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-init.c
553
mutex_unlock(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-init.c
556
void kvm_vgic_destroy(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-init.c
561
mutex_lock(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-init.c
562
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
564
vgic_debug_destroy(kvm);
arch/arm64/kvm/vgic/vgic-init.c
566
kvm_for_each_vcpu(i, vcpu, kvm)
arch/arm64/kvm/vgic/vgic-init.c
569
kvm_vgic_dist_destroy(kvm);
arch/arm64/kvm/vgic/vgic-init.c
571
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
573
if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
arch/arm64/kvm/vgic/vgic-init.c
574
kvm_for_each_vcpu(i, vcpu, kvm)
arch/arm64/kvm/vgic/vgic-init.c
577
mutex_unlock(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-init.c
586
int vgic_lazy_init(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-init.c
590
if (unlikely(!vgic_initialized(kvm))) {
arch/arm64/kvm/vgic/vgic-init.c
597
if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2)
arch/arm64/kvm/vgic/vgic-init.c
600
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
601
ret = vgic_init(kvm);
arch/arm64/kvm/vgic/vgic-init.c
602
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
620
int kvm_vgic_map_resources(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-init.c
622
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-init.c
631
mutex_lock(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-init.c
632
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
636
if (!irqchip_in_kernel(kvm))
arch/arm64/kvm/vgic/vgic-init.c
640
ret = vgic_v2_map_resources(kvm);
arch/arm64/kvm/vgic/vgic-init.c
643
ret = vgic_v3_map_resources(kvm);
arch/arm64/kvm/vgic/vgic-init.c
646
ret = vgic_v5_map_resources(kvm);
arch/arm64/kvm/vgic/vgic-init.c
656
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
658
ret = vgic_register_dist_iodev(kvm, dist_base, type);
arch/arm64/kvm/vgic/vgic-init.c
664
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
670
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-init.c
673
kvm_vm_dead(kvm);
arch/arm64/kvm/vgic/vgic-init.c
675
mutex_unlock(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-init.c
680
void kvm_vgic_finalize_idregs(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-init.c
682
u32 type = kvm->arch.vgic.vgic_model;
arch/arm64/kvm/vgic/vgic-init.c
685
aa64pfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1) & ~ID_AA64PFR0_EL1_GIC;
arch/arm64/kvm/vgic/vgic-init.c
686
aa64pfr2 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR2_EL1) & ~ID_AA64PFR2_EL1_GCIE;
arch/arm64/kvm/vgic/vgic-init.c
687
pfr1 = kvm_read_vm_id_reg(kvm, SYS_ID_PFR1_EL1) & ~ID_PFR1_EL1_GIC;
arch/arm64/kvm/vgic/vgic-init.c
704
kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, aa64pfr0);
arch/arm64/kvm/vgic/vgic-init.c
705
kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR2_EL1, aa64pfr2);
arch/arm64/kvm/vgic/vgic-init.c
706
kvm_set_vm_id_reg(kvm, SYS_ID_PFR1_EL1, pfr1);
arch/arm64/kvm/vgic/vgic-init.c
71
int kvm_vgic_create(struct kvm *kvm, u32 type)
arch/arm64/kvm/vgic/vgic-init.c
95
lockdep_assert_held(&kvm->lock);
arch/arm64/kvm/vgic/vgic-irqfd.c
105
return vgic_its_inject_msi(kvm, &msi);
arch/arm64/kvm/vgic/vgic-irqfd.c
112
struct kvm *kvm, int irq_source_id, int level,
arch/arm64/kvm/vgic/vgic-irqfd.c
122
if (!vgic_has_its(kvm))
arch/arm64/kvm/vgic/vgic-irqfd.c
126
return vgic_its_inject_cached_translation(kvm, &msi);
arch/arm64/kvm/vgic/vgic-irqfd.c
134
if (unlikely(!vgic_initialized(kvm)))
arch/arm64/kvm/vgic/vgic-irqfd.c
136
return vgic_irqfd_set_irq(e, kvm, irq_source_id, 1, line_status);
arch/arm64/kvm/vgic/vgic-irqfd.c
142
int kvm_vgic_setup_default_irq_routing(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-irqfd.c
145
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-irqfd.c
159
ret = kvm_set_irq_routing(kvm, entries, nr, 0);
arch/arm64/kvm/vgic/vgic-irqfd.c
19
struct kvm *kvm, int irq_source_id,
arch/arm64/kvm/vgic/vgic-irqfd.c
25
if (!vgic_valid_spi(kvm, spi_id))
arch/arm64/kvm/vgic/vgic-irqfd.c
28
ret = vgic_lazy_init(kvm);
arch/arm64/kvm/vgic/vgic-irqfd.c
32
return kvm_vgic_inject_irq(kvm, NULL, spi_id, level, NULL);
arch/arm64/kvm/vgic/vgic-irqfd.c
44
int kvm_set_routing_entry(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-irqfd.c
93
struct kvm *kvm, int irq_source_id,
arch/arm64/kvm/vgic/vgic-irqfd.c
98
if (!vgic_has_its(kvm))
arch/arm64/kvm/vgic/vgic-its.c
1036
static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1061
lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser))
arch/arm64/kvm/vgic/vgic-its.c
1089
vcpu = collection_to_vcpu(kvm, collection);
arch/arm64/kvm/vgic/vgic-its.c
1091
irq = vgic_add_lpi(kvm, lpi_nr, vcpu);
arch/arm64/kvm/vgic/vgic-its.c
1095
its_free_ite(kvm, ite);
arch/arm64/kvm/vgic/vgic-its.c
1104
static void vgic_its_free_device(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1115
its_free_ite(kvm, ite);
arch/arm64/kvm/vgic/vgic-its.c
1124
static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)
arch/arm64/kvm/vgic/vgic-its.c
1129
vgic_its_free_device(kvm, its, cur);
arch/arm64/kvm/vgic/vgic-its.c
1133
static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)
arch/arm64/kvm/vgic/vgic-its.c
1165
static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1189
vgic_its_free_device(kvm, its, device);
arch/arm64/kvm/vgic/vgic-its.c
1208
static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1224
vcpu = kvm_get_vcpu_by_id(kvm, its_cmd_get_target_addr(its_cmd));
arch/arm64/kvm/vgic/vgic-its.c
1244
update_affinity_collection(kvm, its, collection);
arch/arm64/kvm/vgic/vgic-its.c
1255
static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1276
int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq)
arch/arm64/kvm/vgic/vgic-its.c
1278
return update_lpi_config(kvm, irq, NULL, true);
arch/arm64/kvm/vgic/vgic-its.c
1285
static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1297
return vgic_its_inv_lpi(kvm, ite->irq);
arch/arm64/kvm/vgic/vgic-its.c
1310
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/vgic/vgic-its.c
1311
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-its.c
1316
irq = vgic_get_irq(kvm, intid);
arch/arm64/kvm/vgic/vgic-its.c
1320
update_lpi_config(kvm, irq, vcpu, false);
arch/arm64/kvm/vgic/vgic-its.c
1321
vgic_put_irq(kvm, irq);
arch/arm64/kvm/vgic/vgic-its.c
1338
static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1349
vcpu = collection_to_vcpu(kvm, collection);
arch/arm64/kvm/vgic/vgic-its.c
1363
static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1366
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-its.c
1372
vcpu1 = kvm_get_vcpu_by_id(kvm, its_cmd_get_target_addr(its_cmd));
arch/arm64/kvm/vgic/vgic-its.c
1373
vcpu2 = kvm_get_vcpu_by_id(kvm, its_cmd_mask_field(its_cmd, 3, 16, 32));
arch/arm64/kvm/vgic/vgic-its.c
1382
irq = vgic_get_irq(kvm, intid);
arch/arm64/kvm/vgic/vgic-its.c
1388
vgic_put_irq(kvm, irq);
arch/arm64/kvm/vgic/vgic-its.c
1400
static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1406
return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
arch/arm64/kvm/vgic/vgic-its.c
1413
static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1421
ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
arch/arm64/kvm/vgic/vgic-its.c
1424
ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
arch/arm64/kvm/vgic/vgic-its.c
1427
ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
arch/arm64/kvm/vgic/vgic-its.c
1430
ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
arch/arm64/kvm/vgic/vgic-its.c
1433
ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
arch/arm64/kvm/vgic/vgic-its.c
1436
ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
arch/arm64/kvm/vgic/vgic-its.c
1439
ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
arch/arm64/kvm/vgic/vgic-its.c
1442
ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
arch/arm64/kvm/vgic/vgic-its.c
1445
ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
arch/arm64/kvm/vgic/vgic-its.c
1448
ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
arch/arm64/kvm/vgic/vgic-its.c
1451
ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
arch/arm64/kvm/vgic/vgic-its.c
148
ret = update_lpi_config(kvm, irq, NULL, false);
arch/arm64/kvm/vgic/vgic-its.c
1499
static unsigned long vgic_mmio_read_its_cbaser(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-its.c
150
vgic_put_irq(kvm, irq);
arch/arm64/kvm/vgic/vgic-its.c
1506
static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1531
static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
arch/arm64/kvm/vgic/vgic-its.c
154
ret = vgic_v3_lpi_sync_pending_status(kvm, irq);
arch/arm64/kvm/vgic/vgic-its.c
1543
int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr,
arch/arm64/kvm/vgic/vgic-its.c
1553
vgic_its_handle_command(kvm, its, cmd_buf);
arch/arm64/kvm/vgic/vgic-its.c
156
vgic_put_irq(kvm, irq);
arch/arm64/kvm/vgic/vgic-its.c
1567
static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1586
vgic_its_process_commands(kvm, its);
arch/arm64/kvm/vgic/vgic-its.c
1591
static unsigned long vgic_mmio_read_its_cwriter(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-its.c
1598
static unsigned long vgic_mmio_read_its_creadr(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-its.c
1605
static int vgic_mmio_uaccess_write_its_creadr(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-its.c
1633
static unsigned long vgic_mmio_read_its_baser(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-its.c
1655
static void vgic_mmio_write_its_baser(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-its.c
1699
vgic_its_free_device_list(kvm, its);
arch/arm64/kvm/vgic/vgic-its.c
1702
vgic_its_free_collection_list(kvm, its);
arch/arm64/kvm/vgic/vgic-its.c
1709
static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
arch/arm64/kvm/vgic/vgic-its.c
1725
static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1749
vgic_its_process_commands(kvm, its);
arch/arm64/kvm/vgic/vgic-its.c
1774
static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1816
static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
1822
mutex_lock(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-its.c
1836
ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, iodev->base_addr,
arch/arm64/kvm/vgic/vgic-its.c
1839
mutex_unlock(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-its.c
1867
mutex_lock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-its.c
1869
if (vgic_initialized(dev->kvm)) {
arch/arm64/kvm/vgic/vgic-its.c
1870
ret = vgic_v4_init(dev->kvm);
arch/arm64/kvm/vgic/vgic-its.c
1872
mutex_unlock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-its.c
1895
dev->kvm->arch.vgic.msis_require_devid = true;
arch/arm64/kvm/vgic/vgic-its.c
1896
dev->kvm->arch.vgic.has_its = true;
arch/arm64/kvm/vgic/vgic-its.c
1904
dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE;
arch/arm64/kvm/vgic/vgic-its.c
1908
mutex_unlock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-its.c
1914
struct kvm *kvm = kvm_dev->kvm;
arch/arm64/kvm/vgic/vgic-its.c
1921
vgic_its_free_device_list(kvm, its);
arch/arm64/kvm/vgic/vgic-its.c
1922
vgic_its_free_collection_list(kvm, its);
arch/arm64/kvm/vgic/vgic-its.c
1980
mutex_lock(&dev->kvm->lock);
arch/arm64/kvm/vgic/vgic-its.c
1982
if (kvm_trylock_all_vcpus(dev->kvm)) {
arch/arm64/kvm/vgic/vgic-its.c
1983
mutex_unlock(&dev->kvm->lock);
arch/arm64/kvm/vgic/vgic-its.c
1987
mutex_lock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-its.c
2008
ret = region->uaccess_its_write(dev->kvm, its, addr,
arch/arm64/kvm/vgic/vgic-its.c
2011
region->its_write(dev->kvm, its, addr, len, *reg);
arch/arm64/kvm/vgic/vgic-its.c
2013
*reg = region->its_read(dev->kvm, its, addr, len);
arch/arm64/kvm/vgic/vgic-its.c
2016
mutex_unlock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-its.c
2017
kvm_unlock_all_vcpus(dev->kvm);
arch/arm64/kvm/vgic/vgic-its.c
2018
mutex_unlock(&dev->kvm->lock);
arch/arm64/kvm/vgic/vgic-its.c
2081
struct kvm *kvm = its->dev->kvm;
arch/arm64/kvm/vgic/vgic-its.c
2094
ret = kvm_read_guest_lock(kvm, gpa, entry, esz);
arch/arm64/kvm/vgic/vgic-its.c
2144
struct kvm *kvm = its->dev->kvm;
arch/arm64/kvm/vgic/vgic-its.c
2182
vcpu = kvm_get_vcpu_by_id(kvm, collection->target_addr);
arch/arm64/kvm/vgic/vgic-its.c
2184
irq = vgic_add_lpi(kvm, lpi_id, vcpu);
arch/arm64/kvm/vgic/vgic-its.c
2186
its_free_ite(kvm, ite);
arch/arm64/kvm/vgic/vgic-its.c
2334
vgic_its_free_device(its->dev->kvm, its, dev);
arch/arm64/kvm/vgic/vgic-its.c
2456
vgic_its_free_device_list(its->dev->kvm, its);
arch/arm64/kvm/vgic/vgic-its.c
2483
struct kvm *kvm = its->dev->kvm;
arch/arm64/kvm/vgic/vgic-its.c
2499
!kvm_get_vcpu_by_id(kvm, target_addr))
arch/arm64/kvm/vgic/vgic-its.c
2585
vgic_its_free_collection_list(its->dev->kvm, its);
arch/arm64/kvm/vgic/vgic-its.c
2620
vgic_its_free_collection_list(its->dev->kvm, its);
arch/arm64/kvm/vgic/vgic-its.c
2639
static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)
arch/arm64/kvm/vgic/vgic-its.c
2648
vgic_its_free_device_list(kvm, its);
arch/arm64/kvm/vgic/vgic-its.c
2649
vgic_its_free_collection_list(kvm, its);
arch/arm64/kvm/vgic/vgic-its.c
2680
static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
arch/arm64/kvm/vgic/vgic-its.c
2688
mutex_lock(&kvm->lock);
arch/arm64/kvm/vgic/vgic-its.c
2690
if (kvm_trylock_all_vcpus(kvm)) {
arch/arm64/kvm/vgic/vgic-its.c
2691
mutex_unlock(&kvm->lock);
arch/arm64/kvm/vgic/vgic-its.c
2695
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-its.c
2700
vgic_its_reset(kvm, its);
arch/arm64/kvm/vgic/vgic-its.c
2714
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-its.c
2715
kvm_unlock_all_vcpus(kvm);
arch/arm64/kvm/vgic/vgic-its.c
2716
mutex_unlock(&kvm->lock);
arch/arm64/kvm/vgic/vgic-its.c
2730
bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-its.c
2732
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-its.c
2755
ret = vgic_check_iorange(dev->kvm, its->vgic_its_base,
arch/arm64/kvm/vgic/vgic-its.c
2760
ret = vgic_register_its_iodev(dev->kvm, its, addr);
arch/arm64/kvm/vgic/vgic-its.c
2768
return vgic_its_ctrl(dev->kvm, its, attr->attr);
arch/arm64/kvm/vgic/vgic-its.c
289
static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
arch/arm64/kvm/vgic/vgic-its.c
292
u64 propbase = GICR_PROPBASER_ADDRESS(kvm->arch.vgic.propbaser);
arch/arm64/kvm/vgic/vgic-its.c
297
ret = kvm_read_guest_lock(kvm, propbase + irq->intid - GIC_LPI_OFFSET,
arch/arm64/kvm/vgic/vgic-its.c
31
static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
arch/arm64/kvm/vgic/vgic-its.c
310
vgic_queue_irq_unlock(kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-its.c
345
static struct kvm_vcpu *collection_to_vcpu(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-its.c
348
return kvm_get_vcpu_by_id(kvm, col->target_addr);
arch/arm64/kvm/vgic/vgic-its.c
357
static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite)
arch/arm64/kvm/vgic/vgic-its.c
364
vcpu = collection_to_vcpu(kvm, ite->collection);
arch/arm64/kvm/vgic/vgic-its.c
37
struct kvm *__k = (i)->dev->kvm; \
arch/arm64/kvm/vgic/vgic-its.c
372
static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
382
update_affinity_ite(kvm, ite);
arch/arm64/kvm/vgic/vgic-its.c
401
struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-its.c
419
ret = kvm_read_guest_lock(vcpu->kvm,
arch/arm64/kvm/vgic/vgic-its.c
428
irq = vgic_get_irq(vcpu->kvm, intid);
arch/arm64/kvm/vgic/vgic-its.c
435
vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-its.c
436
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-its.c
442
static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-its.c
464
static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-its.c
475
static int vgic_mmio_uaccess_write_its_iidr(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-its.c
488
static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-its.c
515
static struct vgic_its *__vgic_doorbell_to_its(struct kvm *kvm, gpa_t db)
arch/arm64/kvm/vgic/vgic-its.c
520
guard(srcu)(&kvm->srcu);
arch/arm64/kvm/vgic/vgic-its.c
522
kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, db);
arch/arm64/kvm/vgic/vgic-its.c
54
struct kvm *__k = (i)->dev->kvm; \
arch/arm64/kvm/vgic/vgic-its.c
542
static struct vgic_irq *vgic_its_check_cache(struct kvm *kvm, phys_addr_t db,
arch/arm64/kvm/vgic/vgic-its.c
552
its = __vgic_doorbell_to_its(kvm, db);
arch/arm64/kvm/vgic/vgic-its.c
567
static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
592
vgic_put_irq(kvm, irq);
arch/arm64/kvm/vgic/vgic-its.c
602
vgic_put_irq(kvm, old);
arch/arm64/kvm/vgic/vgic-its.c
607
struct kvm *kvm = its->dev->kvm;
arch/arm64/kvm/vgic/vgic-its.c
615
vgic_put_irq(kvm, irq);
arch/arm64/kvm/vgic/vgic-its.c
619
void vgic_its_invalidate_all_caches(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-its.c
626
list_for_each_entry_rcu(dev, &kvm->devices, vm_node) {
arch/arm64/kvm/vgic/vgic-its.c
637
int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
650
vcpu = collection_to_vcpu(kvm, ite->collection);
arch/arm64/kvm/vgic/vgic-its.c
657
vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);
arch/arm64/kvm/vgic/vgic-its.c
663
struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi)
arch/arm64/kvm/vgic/vgic-its.c
667
if (!vgic_has_its(kvm))
arch/arm64/kvm/vgic/vgic-its.c
675
return __vgic_doorbell_to_its(kvm, address);
arch/arm64/kvm/vgic/vgic-its.c
685
static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
692
err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq);
arch/arm64/kvm/vgic/vgic-its.c
702
vgic_queue_irq_unlock(kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-its.c
707
int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi)
arch/arm64/kvm/vgic/vgic-its.c
714
irq = vgic_its_check_cache(kvm, db, msi->devid, msi->data);
arch/arm64/kvm/vgic/vgic-its.c
720
vgic_queue_irq_unlock(kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-its.c
721
vgic_put_irq(kvm, irq);
arch/arm64/kvm/vgic/vgic-its.c
732
int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
arch/arm64/kvm/vgic/vgic-its.c
737
if (!vgic_its_inject_cached_translation(kvm, msi))
arch/arm64/kvm/vgic/vgic-its.c
740
its = vgic_msi_to_its(kvm, msi);
arch/arm64/kvm/vgic/vgic-its.c
745
ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data);
arch/arm64/kvm/vgic/vgic-its.c
76
static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
arch/arm64/kvm/vgic/vgic-its.c
763
static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
arch/arm64/kvm/vgic/vgic-its.c
777
vgic_put_irq(kvm, ite->irq);
arch/arm64/kvm/vgic/vgic-its.c
79
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-its.c
80
struct vgic_irq *irq = vgic_get_irq(kvm, intid), *oldirq;
arch/arm64/kvm/vgic/vgic-its.c
802
static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
821
its_free_ite(kvm, ite);
arch/arm64/kvm/vgic/vgic-its.c
833
static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-its.c
855
vcpu = collection_to_vcpu(kvm, collection);
arch/arm64/kvm/vgic/vgic-its.c
868
idx = srcu_read_lock(&its->dev->kvm->srcu);
arch/arm64/kvm/vgic/vgic-its.c
869
ret = kvm_is_visible_gfn(its->dev->kvm, gfn);
arch/arm64/kvm/vgic/vgic-its.c
870
srcu_read_unlock(&its->dev->kvm->srcu, idx);
arch/arm64/kvm/vgic/vgic-its.c
924
if (kvm_read_guest_lock(its->dev->kvm,
arch/arm64/kvm/vgic/vgic-kvm-device.c
111
mutex_lock(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
114
r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
arch/arm64/kvm/vgic/vgic-kvm-device.c
120
r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
arch/arm64/kvm/vgic/vgic-kvm-device.c
126
r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V3);
arch/arm64/kvm/vgic/vgic-kvm-device.c
134
r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V3);
arch/arm64/kvm/vgic/vgic-kvm-device.c
138
r = vgic_v3_set_redist_base(kvm, 0, addr, 0);
arch/arm64/kvm/vgic/vgic-kvm-device.c
154
r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V3);
arch/arm64/kvm/vgic/vgic-kvm-device.c
168
r = vgic_v3_set_redist_base(kvm, index,
arch/arm64/kvm/vgic/vgic-kvm-device.c
173
rdreg = vgic_v3_rdist_region_from_index(kvm, index);
arch/arm64/kvm/vgic/vgic-kvm-device.c
18
int vgic_check_iorange(struct kvm *kvm, phys_addr_t ioaddr,
arch/arm64/kvm/vgic/vgic-kvm-device.c
191
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
193
r = vgic_check_iorange(kvm, *addr_ptr, addr, alignment, size);
arch/arm64/kvm/vgic/vgic-kvm-device.c
199
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
202
mutex_unlock(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
217
r = kvm_vgic_addr(dev->kvm, attr, true);
arch/arm64/kvm/vgic/vgic-kvm-device.c
238
mutex_lock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
245
if (dev->kvm->arch.vgic.nr_spis)
arch/arm64/kvm/vgic/vgic-kvm-device.c
248
dev->kvm->arch.vgic.nr_spis =
arch/arm64/kvm/vgic/vgic-kvm-device.c
251
mutex_unlock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
258
mutex_lock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
259
r = vgic_init(dev->kvm);
arch/arm64/kvm/vgic/vgic-kvm-device.c
260
mutex_unlock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
268
if (vgic_check_type(dev->kvm, KVM_DEV_TYPE_ARM_VGIC_V3))
arch/arm64/kvm/vgic/vgic-kvm-device.c
270
mutex_lock(&dev->kvm->lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
272
if (kvm_trylock_all_vcpus(dev->kvm)) {
arch/arm64/kvm/vgic/vgic-kvm-device.c
273
mutex_unlock(&dev->kvm->lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
277
mutex_lock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
278
r = vgic_v3_save_pending_tables(dev->kvm);
arch/arm64/kvm/vgic/vgic-kvm-device.c
279
mutex_unlock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
280
kvm_unlock_all_vcpus(dev->kvm);
arch/arm64/kvm/vgic/vgic-kvm-device.c
281
mutex_unlock(&dev->kvm->lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
298
r = kvm_vgic_addr(dev->kvm, attr, false);
arch/arm64/kvm/vgic/vgic-kvm-device.c
303
r = put_user(dev->kvm->arch.vgic.nr_spis +
arch/arm64/kvm/vgic/vgic-kvm-device.c
31
if (addr & ~kvm_phys_mask(&kvm->arch.mmu) ||
arch/arm64/kvm/vgic/vgic-kvm-device.c
314
return kvm_vgic_create(dev->kvm, type);
arch/arm64/kvm/vgic/vgic-kvm-device.c
32
(addr + size) > kvm_phys_size(&kvm->arch.mmu))
arch/arm64/kvm/vgic/vgic-kvm-device.c
354
reg_attr->vcpu = kvm_get_vcpu_by_id(dev->kvm, cpuid);
arch/arm64/kvm/vgic/vgic-kvm-device.c
38
static int vgic_check_type(struct kvm *kvm, int type_needed)
arch/arm64/kvm/vgic/vgic-kvm-device.c
390
mutex_lock(&dev->kvm->lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
392
if (kvm_trylock_all_vcpus(dev->kvm)) {
arch/arm64/kvm/vgic/vgic-kvm-device.c
393
mutex_unlock(&dev->kvm->lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
397
mutex_lock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
399
ret = vgic_init(dev->kvm);
arch/arm64/kvm/vgic/vgic-kvm-device.c
40
if (kvm->arch.vgic.vgic_model != type_needed)
arch/arm64/kvm/vgic/vgic-kvm-device.c
416
mutex_unlock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
417
kvm_unlock_all_vcpus(dev->kvm);
arch/arm64/kvm/vgic/vgic-kvm-device.c
418
mutex_unlock(&dev->kvm->lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
46
int kvm_set_legacy_vgic_v2_addr(struct kvm *kvm, struct kvm_arm_device_addr *dev_addr)
arch/arm64/kvm/vgic/vgic-kvm-device.c
48
struct vgic_dist *vgic = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-kvm-device.c
498
reg_attr->vcpu = kvm_mpidr_to_vcpu(dev->kvm, mpidr_reg);
arch/arm64/kvm/vgic/vgic-kvm-device.c
500
reg_attr->vcpu = kvm_get_vcpu(dev->kvm, 0);
arch/arm64/kvm/vgic/vgic-kvm-device.c
51
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
54
r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
arch/arm64/kvm/vgic/vgic-kvm-device.c
56
r = vgic_check_iorange(kvm, vgic->vgic_dist_base, dev_addr->addr,
arch/arm64/kvm/vgic/vgic-kvm-device.c
569
mutex_lock(&dev->kvm->lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
571
if (kvm_trylock_all_vcpus(dev->kvm)) {
arch/arm64/kvm/vgic/vgic-kvm-device.c
572
mutex_unlock(&dev->kvm->lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
576
mutex_lock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
578
if (!(vgic_initialized(dev->kvm) || reg_allowed_pre_init(attr))) {
arch/arm64/kvm/vgic/vgic-kvm-device.c
614
mutex_unlock(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
615
kvm_unlock_all_vcpus(dev->kvm);
arch/arm64/kvm/vgic/vgic-kvm-device.c
616
mutex_unlock(&dev->kvm->lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
62
r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
arch/arm64/kvm/vgic/vgic-kvm-device.c
64
r = vgic_check_iorange(kvm, vgic->vgic_cpu_base, dev_addr->addr,
arch/arm64/kvm/vgic/vgic-kvm-device.c
642
guard(mutex)(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
643
if (vgic_initialized(dev->kvm))
arch/arm64/kvm/vgic/vgic-kvm-device.c
646
if (!irq_is_ppi(dev->kvm, val))
arch/arm64/kvm/vgic/vgic-kvm-device.c
649
dev->kvm->arch.vgic.mi_intid = val;
arch/arm64/kvm/vgic/vgic-kvm-device.c
669
guard(mutex)(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
670
return put_user(dev->kvm->arch.vgic.mi_intid, uaddr);
arch/arm64/kvm/vgic/vgic-kvm-device.c
726
struct vgic_v5_vm *gicv5_vm = &dev->kvm->arch.vgic.gicv5_vm;
arch/arm64/kvm/vgic/vgic-kvm-device.c
73
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
730
guard(mutex)(&dev->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-kvm-device.c
93
static int kvm_vgic_addr(struct kvm *kvm, struct kvm_device_attr *attr, bool write)
arch/arm64/kvm/vgic/vgic-kvm-device.c
96
struct vgic_dist *vgic = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v2.c
114
if (vcpu->kvm->arch.vgic.v2_groups_user_writable)
arch/arm64/kvm/vgic/vgic-mmio-v2.c
124
int nr_vcpus = atomic_read(&source_vcpu->kvm->online_vcpus);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
145
kvm_for_each_vcpu(c, vcpu, source_vcpu->kvm) {
arch/arm64/kvm/vgic/vgic-mmio-v2.c
157
vgic_queue_irq_unlock(source_vcpu->kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
158
vgic_put_irq(source_vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
174
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
185
u8 cpu_mask = GENMASK(atomic_read(&vcpu->kvm->online_vcpus) - 1, 0);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
194
struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, intid + i);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
201
irq->target_vcpu = kvm_get_vcpu(vcpu->kvm, target);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
204
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
220
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
243
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
264
vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
268
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
28
struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v2.c
38
value |= (atomic_read(&vcpu->kvm->online_vcpus) - 1) << 5;
arch/arm64/kvm/vgic/vgic-mmio-v2.c
56
struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v2.c
63
vgic_kick_vcpus(vcpu->kvm);
arch/arm64/kvm/vgic/vgic-mmio-v2.c
76
struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v2.c
98
vcpu->kvm->arch.vgic.v2_groups_user_writable = true;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
106
if (vgic_supports_direct_sgis(vcpu->kvm))
arch/arm64/kvm/vgic/vgic-mmio-v3.c
1068
vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
1082
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
1103
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
1114
kvm_for_each_vcpu(c, c_vcpu, kvm) {
arch/arm64/kvm/vgic/vgic-mmio-v3.c
1132
c_vcpu = kvm_mpidr_to_vcpu(kvm, mpidr | aff0);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
125
struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
131
mutex_lock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
139
if (!vgic_supports_direct_sgis(vcpu->kvm))
arch/arm64/kvm/vgic/vgic-mmio-v3.c
151
vgic_v4_configure_vsgis(vcpu->kvm);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
153
if (vgic_supports_direct_sgis(vcpu->kvm) &&
arch/arm64/kvm/vgic/vgic-mmio-v3.c
155
kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_RELOAD_GICv4);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
157
vgic_kick_vcpus(vcpu->kvm);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
159
mutex_unlock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
174
struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
183
if (vgic_initialized(vcpu->kvm))
arch/arm64/kvm/vgic/vgic-mmio-v3.c
208
if (!vgic_supports_direct_sgis(vcpu->kvm))
arch/arm64/kvm/vgic/vgic-mmio-v3.c
224
struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, intid);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
234
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
250
irq = vgic_get_irq(vcpu->kvm, intid);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
259
irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
262
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
292
if (!vgic_has_its(vcpu->kvm))
arch/arm64/kvm/vgic/vgic-mmio-v3.c
307
vgic_its_invalidate_all_caches(vcpu->kvm);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
321
struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
356
if (vgic_has_its(vcpu->kvm))
arch/arm64/kvm/vgic/vgic-mmio-v3.c
41
bool vgic_has_its(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-mmio-v3.c
43
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
482
struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
491
struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
51
bool vgic_supports_direct_msis(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-mmio-v3.c
578
irq = vgic_get_irq(vcpu->kvm, intid);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
58
if (system_supports_direct_sgis() && !vgic_supports_direct_sgis(kvm))
arch/arm64/kvm/vgic/vgic-mmio-v3.c
580
vgic_its_inv_lpi(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
581
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
61
return kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
69
bool vgic_supports_direct_sgis(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-mmio-v3.c
71
return kvm->arch.vgic.nassgicap;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
780
struct kvm *kvm = vcpu->kvm;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
781
struct vgic_dist *vgic = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
788
lockdep_assert_held(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
789
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
804
if (!vgic_v3_check_base(kvm)) {
arch/arm64/kvm/vgic/vgic-mmio-v3.c
821
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
823
ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base,
arch/arm64/kvm/vgic/vgic-mmio-v3.c
833
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
84
struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
841
kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
844
static int vgic_register_all_redist_iodevs(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-mmio-v3.c
850
lockdep_assert_held(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
852
kvm_for_each_vcpu(c, vcpu, kvm) {
arch/arm64/kvm/vgic/vgic-mmio-v3.c
863
vcpu = kvm_get_vcpu(kvm, i);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
886
static int vgic_v3_alloc_redist_region(struct kvm *kvm, uint32_t index,
arch/arm64/kvm/vgic/vgic-mmio-v3.c
889
struct vgic_dist *d = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio-v3.c
892
int nr_vcpus = atomic_read(&kvm->online_vcpus);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
925
vgic_dist_overlap(kvm, base, size))
arch/arm64/kvm/vgic/vgic-mmio-v3.c
929
if (vgic_v3_rdist_overlap(kvm, base, size))
arch/arm64/kvm/vgic/vgic-mmio-v3.c
938
ret = vgic_check_iorange(kvm, rdreg->base, base, SZ_64K, size);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
954
void vgic_v3_free_redist_region(struct kvm *kvm, struct vgic_redist_region *rdreg)
arch/arm64/kvm/vgic/vgic-mmio-v3.c
959
lockdep_assert_held(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
962
kvm_for_each_vcpu(c, vcpu, kvm) {
arch/arm64/kvm/vgic/vgic-mmio-v3.c
971
int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count)
arch/arm64/kvm/vgic/vgic-mmio-v3.c
975
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
976
ret = vgic_v3_alloc_redist_region(kvm, index, addr, count);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
977
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
98
if (vgic_has_its(vcpu->kvm)) {
arch/arm64/kvm/vgic/vgic-mmio-v3.c
985
ret = vgic_register_all_redist_iodevs(kvm);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
989
mutex_lock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
990
rdreg = vgic_v3_rdist_region_from_index(kvm, index);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
991
vgic_v3_free_redist_region(kvm, rdreg);
arch/arm64/kvm/vgic/vgic-mmio-v3.c
992
mutex_unlock(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio.c
1066
data = region->its_read(vcpu->kvm, iodev->its, addr, len);
arch/arm64/kvm/vgic/vgic-mmio.c
1096
region->its_write(vcpu->kvm, iodev->its, addr, len, data);
arch/arm64/kvm/vgic/vgic-mmio.c
110
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
1108
int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
arch/arm64/kvm/vgic/vgic-mmio.c
1111
struct vgic_io_device *io_device = &kvm->arch.vgic.dist_iodev;
arch/arm64/kvm/vgic/vgic-mmio.c
1129
return kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, dist_base_address,
arch/arm64/kvm/vgic/vgic-mmio.c
139
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
159
vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-mmio.c
161
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
183
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
200
vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-mmio.c
202
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
223
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
264
switch (vcpu->kvm->arch.vgic.vgic_model) {
arch/arm64/kvm/vgic/vgic-mmio.c
280
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
301
vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2);
arch/arm64/kvm/vgic/vgic-mmio.c
316
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
339
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
348
vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-mmio.c
349
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
402
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
425
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
436
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
473
if ((vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 &&
arch/arm64/kvm/vgic/vgic-mmio.c
476
kvm_arm_halt_guest(vcpu->kvm);
arch/arm64/kvm/vgic/vgic-mmio.c
482
if ((vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 &&
arch/arm64/kvm/vgic/vgic-mmio.c
485
kvm_arm_resume_guest(vcpu->kvm);
arch/arm64/kvm/vgic/vgic-mmio.c
506
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
518
mutex_lock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio.c
524
mutex_unlock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio.c
564
u32 model = vcpu->kvm->arch.vgic.vgic_model;
arch/arm64/kvm/vgic/vgic-mmio.c
58
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
588
vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-mmio.c
603
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
613
mutex_lock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio.c
619
mutex_unlock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio.c
640
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
650
mutex_lock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio.c
656
mutex_unlock(&vcpu->kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-mmio.c
679
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
710
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
727
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
753
irq = vgic_get_irq(vcpu->kvm, intid + i);
arch/arm64/kvm/vgic/vgic-mmio.c
762
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
770
int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
arch/arm64/kvm/vgic/vgic-mmio.c
782
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
792
int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
arch/arm64/kvm/vgic/vgic-mmio.c
813
vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-mmio.c
817
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
845
const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio.c
85
vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-mmio.c
867
const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-mmio.c
88
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-mmio.c
946
static bool check_region(const struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-mmio.c
950
int flags, nr_irqs = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
arch/arm64/kvm/vgic/vgic-mmio.c
985
if (!region || !check_region(vcpu->kvm, region, addr, len))
arch/arm64/kvm/vgic/vgic-mmio.h
16
unsigned long (*its_read)(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-mmio.h
22
void (*its_write)(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-mmio.h
31
int (*uaccess_its_write)(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic-v2.c
101
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-v2.c
211
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-v2.c
420
int vgic_v2_map_resources(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-v2.c
422
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-v2.c
441
ret = vgic_init(kvm);
arch/arm64/kvm/vgic/vgic-v2.c
452
ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, dist->vgic_cpu_base,
arch/arm64/kvm/vgic/vgic-v2.c
458
ret = kvm_phys_addr_ioremap(kvm, dist->vgic_cpu_base,
arch/arm64/kvm/vgic/vgic-v2.c
66
if (lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid))
arch/arm64/kvm/vgic/vgic-v2.c
67
kvm_notify_acked_irq(vcpu->kvm, 0,
arch/arm64/kvm/vgic/vgic-v3-nested.c
237
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-v3-nested.c
394
kvm_vgic_inject_irq(vcpu->kvm, vcpu,
arch/arm64/kvm/vgic/vgic-v3-nested.c
395
vcpu->kvm->arch.vgic.mi_intid, state, vcpu);
arch/arm64/kvm/vgic/vgic-v3-nested.c
405
kvm_vgic_inject_irq(vcpu->kvm, vcpu,
arch/arm64/kvm/vgic/vgic-v3-nested.c
406
vcpu->kvm->arch.vgic.mi_intid, level, vcpu);
arch/arm64/kvm/vgic/vgic-v3.c
127
if (deactivated && lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid)) {
arch/arm64/kvm/vgic/vgic-v3.c
128
kvm_notify_acked_irq(vcpu->kvm, 0,
arch/arm64/kvm/vgic/vgic-v3.c
130
atomic_dec_if_positive(&vcpu->kvm->arch.vgic.active_spis);
arch/arm64/kvm/vgic/vgic-v3.c
133
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-v3.c
199
u32 model = vcpu->kvm->arch.vgic.vgic_model;
arch/arm64/kvm/vgic/vgic-v3.c
219
val >= vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)
arch/arm64/kvm/vgic/vgic-v3.c
29
if (!irqchip_in_kernel(vcpu->kvm))
arch/arm64/kvm/vgic/vgic-v3.c
290
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-v3.c
306
u32 model = vcpu->kvm->arch.vgic.vgic_model;
arch/arm64/kvm/vgic/vgic-v3.c
374
u32 model = vcpu->kvm->arch.vgic.vgic_model;
arch/arm64/kvm/vgic/vgic-v3.c
413
u32 model = vcpu->kvm->arch.vgic.vgic_model;
arch/arm64/kvm/vgic/vgic-v3.c
441
u32 model = vcpu->kvm->arch.vgic.vgic_model;
arch/arm64/kvm/vgic/vgic-v3.c
489
if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
arch/arm64/kvm/vgic/vgic-v3.c
512
if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2 ||
arch/arm64/kvm/vgic/vgic-v3.c
513
!irqchip_in_kernel(vcpu->kvm))
arch/arm64/kvm/vgic/vgic-v3.c
518
int vgic_v3_lpi_sync_pending_status(struct kvm *kvm, struct vgic_irq *irq)
arch/arm64/kvm/vgic/vgic-v3.c
539
ret = kvm_read_guest_lock(kvm, ptr, &val, 1);
arch/arm64/kvm/vgic/vgic-v3.c
551
vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-v3.c
556
ret = vgic_write_guest_lock(kvm, ptr, &val, 1);
arch/arm64/kvm/vgic/vgic-v3.c
567
static void unmap_all_vpes(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-v3.c
569
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-v3.c
573
free_irq(dist->its_vm.vpes[i]->irq, kvm_get_vcpu(kvm, i));
arch/arm64/kvm/vgic/vgic-v3.c
576
static void map_all_vpes(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-v3.c
578
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-v3.c
582
WARN_ON(vgic_v4_request_vpe_irq(kvm_get_vcpu(kvm, i),
arch/arm64/kvm/vgic/vgic-v3.c
590
int vgic_v3_save_pending_tables(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-v3.c
592
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-v3.c
600
if (unlikely(!vgic_initialized(kvm)))
arch/arm64/kvm/vgic/vgic-v3.c
609
unmap_all_vpes(kvm);
arch/arm64/kvm/vgic/vgic-v3.c
631
ret = kvm_read_guest_lock(kvm, ptr, &val, 1);
arch/arm64/kvm/vgic/vgic-v3.c
64
atomic_read(&vcpu->kvm->arch.vgic.active_spis))
arch/arm64/kvm/vgic/vgic-v3.c
652
ret = vgic_write_guest_lock(kvm, ptr, &val, 1);
arch/arm64/kvm/vgic/vgic-v3.c
659
map_all_vpes(kvm);
arch/arm64/kvm/vgic/vgic-v3.c
674
bool vgic_v3_rdist_overlap(struct kvm *kvm, gpa_t base, size_t size)
arch/arm64/kvm/vgic/vgic-v3.c
676
struct vgic_dist *d = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-v3.c
681
(base < rdreg->base + vgic_v3_rd_region_size(kvm, rdreg)))
arch/arm64/kvm/vgic/vgic-v3.c
691
bool vgic_v3_check_base(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-v3.c
693
struct vgic_dist *d = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-v3.c
701
size_t sz = vgic_v3_rd_region_size(kvm, rdreg);
arch/arm64/kvm/vgic/vgic-v3.c
703
if (vgic_check_iorange(kvm, VGIC_ADDR_UNDEF,
arch/arm64/kvm/vgic/vgic-v3.c
711
return !vgic_v3_rdist_overlap(kvm, d->vgic_dist_base,
arch/arm64/kvm/vgic/vgic-v3.c
738
struct vgic_redist_region *vgic_v3_rdist_region_from_index(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-v3.c
741
struct list_head *rd_regions = &kvm->arch.vgic.rd_regions;
arch/arm64/kvm/vgic/vgic-v3.c
752
int vgic_v3_map_resources(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-v3.c
754
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-v3.c
758
kvm_for_each_vcpu(c, vcpu, kvm) {
arch/arm64/kvm/vgic/vgic-v3.c
772
if (!vgic_v3_check_base(kvm)) {
arch/arm64/kvm/vgic/vgic-v3.c
781
if (!vgic_initialized(kvm)) {
arch/arm64/kvm/vgic/vgic-v3.c
786
vgic_v4_configure_vsgis(kvm);
arch/arm64/kvm/vgic/vgic-v3.c
81
if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
arch/arm64/kvm/vgic/vgic-v4.c
154
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-v4.c
186
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-v4.c
190
void vgic_v4_configure_vsgis(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-v4.c
192
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-v4.c
196
lockdep_assert_held(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-v4.c
198
kvm_arm_halt_guest(kvm);
arch/arm64/kvm/vgic/vgic-v4.c
200
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/arm64/kvm/vgic/vgic-v4.c
207
kvm_arm_resume_guest(kvm);
arch/arm64/kvm/vgic/vgic-v4.c
242
int vgic_v4_init(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-v4.c
244
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic-v4.c
249
lockdep_assert_held(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-v4.c
257
nr_vcpus = atomic_read(&kvm->online_vcpus);
arch/arm64/kvm/vgic/vgic-v4.c
266
kvm_for_each_vcpu(i, vcpu, kvm)
arch/arm64/kvm/vgic/vgic-v4.c
278
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/arm64/kvm/vgic/vgic-v4.c
309
vgic_v4_teardown(kvm);
arch/arm64/kvm/vgic/vgic-v4.c
318
void vgic_v4_teardown(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-v4.c
320
struct its_vm *its_vm = &kvm->arch.vgic.its_vm;
arch/arm64/kvm/vgic/vgic-v4.c
323
lockdep_assert_held(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-v4.c
329
struct kvm_vcpu *vcpu = kvm_get_vcpu(kvm, i);
arch/arm64/kvm/vgic/vgic-v4.c
362
if (!vgic_supports_direct_irqs(vcpu->kvm) || !vpe->resident)
arch/arm64/kvm/vgic/vgic-v4.c
373
if (!vgic_supports_direct_irqs(vcpu->kvm) || vpe->resident)
arch/arm64/kvm/vgic/vgic-v4.c
389
err = its_make_vpe_resident(vpe, false, vcpu->kvm->arch.vgic.enabled);
arch/arm64/kvm/vgic/vgic-v4.c
416
static struct vgic_its *vgic_get_its(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic-v4.c
427
return vgic_msi_to_its(kvm, &msi);
arch/arm64/kvm/vgic/vgic-v4.c
430
int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int virq,
arch/arm64/kvm/vgic/vgic-v4.c
439
if (!vgic_supports_direct_msis(kvm))
arch/arm64/kvm/vgic/vgic-v4.c
446
its = vgic_get_its(kvm, irq_entry);
arch/arm64/kvm/vgic/vgic-v4.c
459
if (vgic_its_resolve_lpi(kvm, its, irq_entry->msi.devid,
arch/arm64/kvm/vgic/vgic-v4.c
476
.vm = &kvm->arch.vgic.its_vm,
arch/arm64/kvm/vgic/vgic-v4.c
506
vgic_queue_irq_unlock(kvm, irq, flags);
arch/arm64/kvm/vgic/vgic-v4.c
514
static struct vgic_irq *__vgic_host_irq_get_vlpi(struct kvm *kvm, int host_irq)
arch/arm64/kvm/vgic/vgic-v4.c
520
xa_for_each(&kvm->arch.vgic.lpi_xa, idx, irq) {
arch/arm64/kvm/vgic/vgic-v4.c
533
void kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int host_irq)
arch/arm64/kvm/vgic/vgic-v4.c
538
if (!vgic_supports_direct_msis(kvm))
arch/arm64/kvm/vgic/vgic-v4.c
541
irq = __vgic_host_irq_get_vlpi(kvm, host_irq);
arch/arm64/kvm/vgic/vgic-v4.c
554
vgic_put_irq(kvm, irq);
arch/arm64/kvm/vgic/vgic-v5.c
128
int vgic_v5_init(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-v5.c
133
if (vgic_initialized(kvm))
arch/arm64/kvm/vgic/vgic-v5.c
136
kvm_for_each_vcpu(idx, vcpu, kvm) {
arch/arm64/kvm/vgic/vgic-v5.c
144
bitmap_zero(kvm->arch.vgic.gicv5_vm.userspace_ppis,
arch/arm64/kvm/vgic/vgic-v5.c
146
__set_bit(GICV5_ARCH_PPI_SW_PPI, kvm->arch.vgic.gicv5_vm.userspace_ppis);
arch/arm64/kvm/vgic/vgic-v5.c
147
bitmap_and(kvm->arch.vgic.gicv5_vm.userspace_ppis,
arch/arm64/kvm/vgic/vgic-v5.c
148
kvm->arch.vgic.gicv5_vm.userspace_ppis,
arch/arm64/kvm/vgic/vgic-v5.c
154
int vgic_v5_map_resources(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-v5.c
156
if (!vgic_initialized(kvm))
arch/arm64/kvm/vgic/vgic-v5.c
162
int vgic_v5_finalize_ppi_state(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic-v5.c
167
if (!vgic_is_v5(kvm))
arch/arm64/kvm/vgic/vgic-v5.c
170
guard(mutex)(&kvm->arch.config_lock);
arch/arm64/kvm/vgic/vgic-v5.c
177
if (test_bit(GICV5_ARCH_PPI_SW_PPI, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask))
arch/arm64/kvm/vgic/vgic-v5.c
181
vcpu0 = kvm_get_vcpu(kvm, 0);
arch/arm64/kvm/vgic/vgic-v5.c
183
bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
arch/arm64/kvm/vgic/vgic-v5.c
184
bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, VGIC_V5_NR_PRIVATE_IRQS);
arch/arm64/kvm/vgic/vgic-v5.c
195
__set_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask);
arch/arm64/kvm/vgic/vgic-v5.c
196
__assign_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr,
arch/arm64/kvm/vgic/vgic-v5.c
201
vgic_put_irq(vcpu0->kvm, irq);
arch/arm64/kvm/vgic/vgic-v5.c
245
bool vgic_v5_ppi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
arch/arm64/kvm/vgic/vgic-v5.c
304
for_each_visible_v5_ppi(i, vcpu->kvm) {
arch/arm64/kvm/vgic/vgic-v5.c
326
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-v5.c
346
for_each_visible_v5_ppi(i, vcpu->kvm) {
arch/arm64/kvm/vgic/vgic-v5.c
357
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-v5.c
379
for_each_visible_v5_ppi(i, vcpu->kvm) {
arch/arm64/kvm/vgic/vgic-v5.c
393
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic-v5.c
416
for_each_visible_v5_ppi(i, vcpu->kvm) {
arch/arm64/kvm/vgic/vgic-v5.c
428
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic.c
101
if (irq_is_lpi(kvm, intid))
arch/arm64/kvm/vgic/vgic.c
102
return vgic_get_lpi(kvm, intid);
arch/arm64/kvm/vgic/vgic.c
1055
if (vgic_is_v5(vcpu->kvm))
arch/arm64/kvm/vgic/vgic.c
1066
if (vgic_is_v3(vcpu->kvm)) {
arch/arm64/kvm/vgic/vgic.c
1082
if (!vgic_is_v5(vcpu->kvm))
arch/arm64/kvm/vgic/vgic.c
1108
if (vgic_is_v5(vcpu->kvm))
arch/arm64/kvm/vgic/vgic.c
1118
if (vgic_is_v5(vcpu->kvm)) {
arch/arm64/kvm/vgic/vgic.c
114
type = vcpu->kvm->arch.vgic.vgic_model;
arch/arm64/kvm/vgic/vgic.c
1168
if (vgic_supports_direct_irqs(vcpu->kvm) && kvm_vgic_global_state.has_gicv4)
arch/arm64/kvm/vgic/vgic.c
1174
const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic.c
1176
if (unlikely(!irqchip_in_kernel(vcpu->kvm) || !vgic_initialized(vcpu->kvm))) {
arch/arm64/kvm/vgic/vgic.c
1202
const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic.c
1204
if (unlikely(!irqchip_in_kernel(vcpu->kvm) || !vgic_initialized(vcpu->kvm))) {
arch/arm64/kvm/vgic/vgic.c
1236
if (vgic_is_v5(vcpu->kvm))
arch/arm64/kvm/vgic/vgic.c
1239
if (!vcpu->kvm->arch.vgic.enabled)
arch/arm64/kvm/vgic/vgic.c
1265
void vgic_kick_vcpus(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic.c
1274
kvm_for_each_vcpu(c, vcpu, kvm) {
arch/arm64/kvm/vgic/vgic.c
1288
if (!vgic_initialized(vcpu->kvm))
arch/arm64/kvm/vgic/vgic.c
129
return vgic_get_irq(vcpu->kvm, intid);
arch/arm64/kvm/vgic/vgic.c
1295
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic.c
139
static __must_check bool __vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
arch/arm64/kvm/vgic/vgic.c
141
if (!irq_is_lpi(kvm, irq->intid))
arch/arm64/kvm/vgic/vgic.c
147
static __must_check bool vgic_put_irq_norelease(struct kvm *kvm, struct vgic_irq *irq)
arch/arm64/kvm/vgic/vgic.c
149
return __vgic_put_irq(kvm, irq);
arch/arm64/kvm/vgic/vgic.c
152
void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
arch/arm64/kvm/vgic/vgic.c
154
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic.c
162
if (IS_ENABLED(CONFIG_LOCKDEP) && irq_is_lpi(kvm, irq->intid)) {
arch/arm64/kvm/vgic/vgic.c
166
if (!irq_is_lpi(kvm, irq->intid))
arch/arm64/kvm/vgic/vgic.c
176
static void vgic_release_deleted_lpis(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic.c
178
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic.c
202
if (irq_is_lpi(vcpu->kvm, irq->intid)) {
arch/arm64/kvm/vgic/vgic.c
208
deleted |= vgic_put_irq_norelease(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic.c
215
vgic_release_deleted_lpis(vcpu->kvm);
arch/arm64/kvm/vgic/vgic.c
277
!irq->target_vcpu->kvm->arch.vgic.enabled))
arch/arm64/kvm/vgic/vgic.c
391
static bool vgic_model_needs_bcst_kick(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic.c
402
kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3);
arch/arm64/kvm/vgic/vgic.c
413
bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
arch/arm64/kvm/vgic/vgic.c
422
return irq->ops->queue_irq_unlock(kvm, irq, flags);
arch/arm64/kvm/vgic/vgic.c
496
bcast = (vgic_model_needs_bcst_kick(vcpu->kvm) &&
arch/arm64/kvm/vgic/vgic.c
497
vgic_valid_spi(vcpu->kvm, irq->intid) &&
arch/arm64/kvm/vgic/vgic.c
498
atomic_fetch_inc(&vcpu->kvm->arch.vgic.active_spis) == 0);
arch/arm64/kvm/vgic/vgic.c
507
kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_IRQ_PENDING);
arch/arm64/kvm/vgic/vgic.c
530
int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
arch/arm64/kvm/vgic/vgic.c
536
if (unlikely(!vgic_initialized(kvm)))
arch/arm64/kvm/vgic/vgic.c
539
if (!vcpu && irq_is_private(kvm, intid))
arch/arm64/kvm/vgic/vgic.c
544
if (irq_is_private(kvm, intid))
arch/arm64/kvm/vgic/vgic.c
547
irq = vgic_get_irq(kvm, intid);
arch/arm64/kvm/vgic/vgic.c
556
vgic_put_irq(kvm, irq);
arch/arm64/kvm/vgic/vgic.c
565
vgic_queue_irq_unlock(kvm, irq, flags);
arch/arm64/kvm/vgic/vgic.c
566
vgic_put_irq(kvm, irq);
arch/arm64/kvm/vgic/vgic.c
581
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic.c
640
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic.c
66
static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid)
arch/arm64/kvm/vgic/vgic.c
668
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic.c
676
if (!vgic_initialized(vcpu->kvm))
arch/arm64/kvm/vgic/vgic.c
68
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic.c
685
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic.c
701
vgic_put_irq(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic.c
721
if (!vgic_initialized(vcpu->kvm))
arch/arm64/kvm/vgic/vgic.c
725
if (!irq_is_ppi(vcpu->kvm, intid) && !vgic_valid_spi(vcpu->kvm, intid))
arch/arm64/kvm/vgic/vgic.c
784
deleted_lpis |= vgic_put_irq_norelease(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic.c
843
deleted_lpis |= vgic_put_irq_norelease(vcpu->kvm, irq);
arch/arm64/kvm/vgic/vgic.c
856
vgic_release_deleted_lpis(vcpu->kvm);
arch/arm64/kvm/vgic/vgic.c
861
if (vgic_is_v5(vcpu->kvm)) {
arch/arm64/kvm/vgic/vgic.c
87
struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid)
arch/arm64/kvm/vgic/vgic.c
90
if (vgic_is_v5(kvm))
arch/arm64/kvm/vgic/vgic.c
95
intid < (kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) {
arch/arm64/kvm/vgic/vgic.c
96
intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS);
arch/arm64/kvm/vgic/vgic.c
97
return &kvm->arch.vgic.spis[intid - VGIC_NR_PRIVATE_IRQS];
arch/arm64/kvm/vgic/vgic.h
123
return vcpu->kvm->arch.vgic.implementation_rev;
arch/arm64/kvm/vgic/vgic.h
154
static inline int vgic_write_guest_lock(struct kvm *kvm, gpa_t gpa,
arch/arm64/kvm/vgic/vgic.h
157
struct vgic_dist *dist = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic.h
161
ret = kvm_write_guest_lock(kvm, gpa, data, len);
arch/arm64/kvm/vgic/vgic.h
262
struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid);
arch/arm64/kvm/vgic/vgic.h
264
void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq);
arch/arm64/kvm/vgic/vgic.h
269
bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
arch/arm64/kvm/vgic/vgic.h
271
void vgic_kick_vcpus(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
275
int vgic_check_iorange(struct kvm *kvm, phys_addr_t ioaddr,
arch/arm64/kvm/vgic/vgic.h
293
int vgic_v2_map_resources(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
294
int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
arch/arm64/kvm/vgic/vgic.h
330
int vgic_v3_map_resources(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
331
int vgic_v3_lpi_sync_pending_status(struct kvm *kvm, struct vgic_irq *irq);
arch/arm64/kvm/vgic/vgic.h
332
int vgic_v3_save_pending_tables(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
333
int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count);
arch/arm64/kvm/vgic/vgic.h
336
bool vgic_v3_check_base(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
341
bool vgic_has_its(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
345
int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi);
arch/arm64/kvm/vgic/vgic.h
360
int vgic_lazy_init(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
361
int vgic_init(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
363
void vgic_debug_init(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
364
void vgic_debug_destroy(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
368
int vgic_v5_init(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
369
int vgic_v5_map_resources(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
412
vgic_v3_rd_region_size(struct kvm *kvm, struct vgic_redist_region *rdreg)
arch/arm64/kvm/vgic/vgic.h
415
return atomic_read(&kvm->online_vcpus) * KVM_VGIC_V3_REDIST_SIZE;
arch/arm64/kvm/vgic/vgic.h
420
struct vgic_redist_region *vgic_v3_rdist_region_from_index(struct kvm *kvm,
arch/arm64/kvm/vgic/vgic.h
422
void vgic_v3_free_redist_region(struct kvm *kvm, struct vgic_redist_region *rdreg);
arch/arm64/kvm/vgic/vgic.h
424
bool vgic_v3_rdist_overlap(struct kvm *kvm, gpa_t base, size_t size);
arch/arm64/kvm/vgic/vgic.h
426
static inline bool vgic_dist_overlap(struct kvm *kvm, gpa_t base, size_t size)
arch/arm64/kvm/vgic/vgic.h
428
struct vgic_dist *d = &kvm->arch.vgic;
arch/arm64/kvm/vgic/vgic.h
435
int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
arch/arm64/kvm/vgic/vgic.h
437
struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi);
arch/arm64/kvm/vgic/vgic.h
438
int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi);
arch/arm64/kvm/vgic/vgic.h
439
void vgic_its_invalidate_all_caches(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
442
int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq);
arch/arm64/kvm/vgic/vgic.h
445
int vgic_v4_init(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
446
void vgic_v4_teardown(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
447
void vgic_v4_configure_vsgis(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
453
static inline bool kvm_has_gicv3(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic.h
455
return kvm_has_feat(kvm, ID_AA64PFR0_EL1, GIC, IMP);
arch/arm64/kvm/vgic/vgic.h
458
static inline bool kvm_has_gicv5(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic.h
460
return kvm_has_feat(kvm, ID_AA64PFR2_EL1, GCIE, IMP);
arch/arm64/kvm/vgic/vgic.h
486
bool vgic_supports_direct_msis(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
487
bool vgic_supports_direct_sgis(struct kvm *kvm);
arch/arm64/kvm/vgic/vgic.h
489
static inline bool vgic_supports_direct_irqs(struct kvm *kvm)
arch/arm64/kvm/vgic/vgic.h
492
if (vgic_is_v5(kvm))
arch/arm64/kvm/vgic/vgic.h
495
return vgic_supports_direct_msis(kvm) || vgic_supports_direct_sgis(kvm);
arch/loongarch/include/asm/kvm_dmsintc.h
12
struct kvm *kvm;
arch/loongarch/include/asm/kvm_dmsintc.h
24
int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data, int level);
arch/loongarch/include/asm/kvm_dmsintc.h
25
int dmsintc_deliver_msi_to_vcpu(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 vector, int level);
arch/loongarch/include/asm/kvm_eiointc.h
56
struct kvm *kvm;
arch/loongarch/include/asm/kvm_host.h
323
int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, bool blockable);
arch/loongarch/include/asm/kvm_host.h
324
int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
arch/loongarch/include/asm/kvm_host.h
325
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
arch/loongarch/include/asm/kvm_host.h
346
static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {}
arch/loongarch/include/asm/kvm_host.h
350
static inline void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {}
arch/loongarch/include/asm/kvm_host.h
353
void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, const struct kvm_memory_slot *memslot);
arch/loongarch/include/asm/kvm_host.h
354
void kvm_init_vmcs(struct kvm *kvm);
arch/loongarch/include/asm/kvm_ipi.h
15
struct kvm *kvm;
arch/loongarch/include/asm/kvm_pch_pic.h
54
struct kvm *kvm;
arch/loongarch/include/asm/kvm_pch_pic.h
74
int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, int level);
arch/loongarch/include/asm/kvm_vcpu.h
106
struct kvm_vcpu *kvm_get_vcpu_by_cpuid(struct kvm *kvm, int cpuid);
arch/loongarch/include/asm/kvm_vcpu.h
152
return vcpu->kvm->arch.pv_features & BIT(feature);
arch/loongarch/kernel/asm-offsets.c
296
OFFSET(KVM_VCPU_KVM, kvm_vcpu, kvm);
arch/loongarch/kernel/asm-offsets.c
315
OFFSET(KVM_GPGD, kvm, arch.pgd);
arch/loongarch/kvm/exit.c
208
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/loongarch/kvm/exit.c
210
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/loongarch/kvm/exit.c
221
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/loongarch/kvm/exit.c
224
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/loongarch/kvm/exit.c
478
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/loongarch/kvm/exit.c
481
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/loongarch/kvm/exit.c
53
ret = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK;
arch/loongarch/kvm/exit.c
646
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/loongarch/kvm/exit.c
648
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/loongarch/kvm/exit.c
676
if (unlikely(badv >= vcpu->kvm->arch.gpa_size)) {
arch/loongarch/kvm/exit.c
842
dest = kvm_get_vcpu_by_cpuid(vcpu->kvm, cpu + min);
arch/loongarch/kvm/intc/dmsintc.c
138
struct kvm *kvm;
arch/loongarch/kvm/intc/dmsintc.c
146
kvm = dev->kvm;
arch/loongarch/kvm/intc/dmsintc.c
147
if (kvm->arch.dmsintc) {
arch/loongarch/kvm/intc/dmsintc.c
156
s->kvm = kvm;
arch/loongarch/kvm/intc/dmsintc.c
157
kvm->arch.dmsintc = s;
arch/loongarch/kvm/intc/dmsintc.c
165
if (!dev || !dev->kvm || !dev->kvm->arch.dmsintc)
arch/loongarch/kvm/intc/dmsintc.c
168
kfree(dev->kvm->arch.dmsintc);
arch/loongarch/kvm/intc/dmsintc.c
47
int dmsintc_deliver_msi_to_vcpu(struct kvm *kvm,
arch/loongarch/kvm/intc/dmsintc.c
68
int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data, int level)
arch/loongarch/kvm/intc/dmsintc.c
74
cpu = (addr >> AVEC_CPU_SHIFT) & kvm->arch.dmsintc->cpu_mask;
arch/loongarch/kvm/intc/dmsintc.c
77
vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
arch/loongarch/kvm/intc/dmsintc.c
81
return dmsintc_deliver_msi_to_vcpu(kvm, vcpu, irq, level);
arch/loongarch/kvm/intc/dmsintc.c
90
struct loongarch_dmsintc *s = dev->kvm->arch.dmsintc;
arch/loongarch/kvm/intc/eiointc.c
165
struct loongarch_eiointc *eiointc = vcpu->kvm->arch.eiointc;
arch/loongarch/kvm/intc/eiointc.c
23
vcpu = kvm_get_vcpu_by_cpuid(s->kvm, cpuid);
arch/loongarch/kvm/intc/eiointc.c
297
struct loongarch_eiointc *eiointc = vcpu->kvm->arch.eiointc;
arch/loongarch/kvm/intc/eiointc.c
345
struct loongarch_eiointc *eiointc = vcpu->kvm->arch.eiointc;
arch/loongarch/kvm/intc/eiointc.c
375
struct loongarch_eiointc *eiointc = vcpu->kvm->arch.eiointc;
arch/loongarch/kvm/intc/eiointc.c
417
struct loongarch_eiointc *s = dev->kvm->arch.eiointc;
arch/loongarch/kvm/intc/eiointc.c
468
s = dev->kvm->arch.eiointc;
arch/loongarch/kvm/intc/eiointc.c
48
vcpu = kvm_get_vcpu_by_id(s->kvm, cpu);
arch/loongarch/kvm/intc/eiointc.c
528
s = dev->kvm->arch.eiointc;
arch/loongarch/kvm/intc/eiointc.c
619
struct kvm *kvm = dev->kvm;
arch/loongarch/kvm/intc/eiointc.c
622
if (kvm->arch.eiointc)
arch/loongarch/kvm/intc/eiointc.c
630
s->kvm = kvm;
arch/loongarch/kvm/intc/eiointc.c
637
mutex_lock(&kvm->slots_lock);
arch/loongarch/kvm/intc/eiointc.c
638
ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS,
arch/loongarch/kvm/intc/eiointc.c
640
mutex_unlock(&kvm->slots_lock);
arch/loongarch/kvm/intc/eiointc.c
648
mutex_lock(&kvm->slots_lock);
arch/loongarch/kvm/intc/eiointc.c
649
ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS,
arch/loongarch/kvm/intc/eiointc.c
651
mutex_unlock(&kvm->slots_lock);
arch/loongarch/kvm/intc/eiointc.c
653
mutex_lock(&kvm->slots_lock);
arch/loongarch/kvm/intc/eiointc.c
654
kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &s->device);
arch/loongarch/kvm/intc/eiointc.c
655
mutex_unlock(&kvm->slots_lock);
arch/loongarch/kvm/intc/eiointc.c
659
kvm->arch.eiointc = s;
arch/loongarch/kvm/intc/eiointc.c
666
struct kvm *kvm;
arch/loongarch/kvm/intc/eiointc.c
669
if (!dev || !dev->kvm || !dev->kvm->arch.eiointc)
arch/loongarch/kvm/intc/eiointc.c
672
kvm = dev->kvm;
arch/loongarch/kvm/intc/eiointc.c
673
eiointc = kvm->arch.eiointc;
arch/loongarch/kvm/intc/eiointc.c
674
mutex_lock(&kvm->slots_lock);
arch/loongarch/kvm/intc/eiointc.c
675
kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &eiointc->device);
arch/loongarch/kvm/intc/eiointc.c
676
kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &eiointc->device_vext);
arch/loongarch/kvm/intc/eiointc.c
677
mutex_unlock(&kvm->slots_lock);
arch/loongarch/kvm/intc/eiointc.c
89
vcpu = kvm_get_vcpu_by_cpuid(s->kvm, cpuid);
arch/loongarch/kvm/intc/ipi.c
104
static int mail_send(struct kvm *kvm, uint64_t data)
arch/loongarch/kvm/intc/ipi.c
111
vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
arch/loongarch/kvm/intc/ipi.c
143
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/loongarch/kvm/intc/ipi.c
145
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/loongarch/kvm/intc/ipi.c
159
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/loongarch/kvm/intc/ipi.c
161
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/loongarch/kvm/intc/ipi.c
168
static int any_send(struct kvm *kvm, uint64_t data)
arch/loongarch/kvm/intc/ipi.c
174
vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
arch/loongarch/kvm/intc/ipi.c
25
static void ipi_send(struct kvm *kvm, uint64_t data)
arch/loongarch/kvm/intc/ipi.c
257
ipi_send(vcpu->kvm, data);
arch/loongarch/kvm/intc/ipi.c
260
mail_send(vcpu->kvm, data);
arch/loongarch/kvm/intc/ipi.c
263
any_send(vcpu->kvm, data);
arch/loongarch/kvm/intc/ipi.c
307
vcpu = kvm_get_vcpu_by_id(dev->kvm, cpu);
arch/loongarch/kvm/intc/ipi.c
31
vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
arch/loongarch/kvm/intc/ipi.c
397
struct kvm *kvm;
arch/loongarch/kvm/intc/ipi.c
406
kvm = dev->kvm;
arch/loongarch/kvm/intc/ipi.c
407
if (kvm->arch.ipi) {
arch/loongarch/kvm/intc/ipi.c
417
s->kvm = kvm;
arch/loongarch/kvm/intc/ipi.c
424
mutex_lock(&kvm->slots_lock);
arch/loongarch/kvm/intc/ipi.c
425
ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, IOCSR_IPI_BASE, IOCSR_IPI_SIZE, device);
arch/loongarch/kvm/intc/ipi.c
426
mutex_unlock(&kvm->slots_lock);
arch/loongarch/kvm/intc/ipi.c
432
kvm->arch.ipi = s;
arch/loongarch/kvm/intc/ipi.c
442
struct kvm *kvm;
arch/loongarch/kvm/intc/ipi.c
445
if (!dev || !dev->kvm || !dev->kvm->arch.ipi)
arch/loongarch/kvm/intc/ipi.c
448
kvm = dev->kvm;
arch/loongarch/kvm/intc/ipi.c
449
ipi = kvm->arch.ipi;
arch/loongarch/kvm/intc/ipi.c
450
mutex_lock(&kvm->slots_lock);
arch/loongarch/kvm/intc/ipi.c
451
kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &ipi->device);
arch/loongarch/kvm/intc/ipi.c
452
mutex_unlock(&kvm->slots_lock);
arch/loongarch/kvm/intc/pch_pic.c
151
struct loongarch_pch_pic *s = vcpu->kvm->arch.pch_pic;
arch/loongarch/kvm/intc/pch_pic.c
25
eiointc_set_irq(s->kvm->arch.eiointc, irq, level);
arch/loongarch/kvm/intc/pch_pic.c
250
struct loongarch_pch_pic *s = vcpu->kvm->arch.pch_pic;
arch/loongarch/kvm/intc/pch_pic.c
277
struct kvm *kvm = dev->kvm;
arch/loongarch/kvm/intc/pch_pic.c
279
struct loongarch_pch_pic *s = dev->kvm->arch.pch_pic;
arch/loongarch/kvm/intc/pch_pic.c
285
mutex_lock(&kvm->slots_lock);
arch/loongarch/kvm/intc/pch_pic.c
287
ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, addr, PCH_PIC_SIZE, device);
arch/loongarch/kvm/intc/pch_pic.c
288
mutex_unlock(&kvm->slots_lock);
arch/loongarch/kvm/intc/pch_pic.c
304
s = dev->kvm->arch.pch_pic;
arch/loongarch/kvm/intc/pch_pic.c
31
eiointc_set_irq(s->kvm->arch.eiointc, irq, level);
arch/loongarch/kvm/intc/pch_pic.c
392
if (!dev->kvm->arch.pch_pic) {
arch/loongarch/kvm/intc/pch_pic.c
410
static int kvm_setup_default_irq_routing(struct kvm *kvm)
arch/loongarch/kvm/intc/pch_pic.c
426
ret = kvm_set_irq_routing(kvm, entries, nr, 0);
arch/loongarch/kvm/intc/pch_pic.c
435
struct kvm *kvm = dev->kvm;
arch/loongarch/kvm/intc/pch_pic.c
439
if (kvm->arch.pch_pic)
arch/loongarch/kvm/intc/pch_pic.c
442
ret = kvm_setup_default_irq_routing(kvm);
arch/loongarch/kvm/intc/pch_pic.c
467
s->kvm = kvm;
arch/loongarch/kvm/intc/pch_pic.c
468
kvm->arch.pch_pic = s;
arch/loongarch/kvm/intc/pch_pic.c
475
struct kvm *kvm;
arch/loongarch/kvm/intc/pch_pic.c
478
if (!dev || !dev->kvm || !dev->kvm->arch.pch_pic)
arch/loongarch/kvm/intc/pch_pic.c
481
kvm = dev->kvm;
arch/loongarch/kvm/intc/pch_pic.c
482
s = kvm->arch.pch_pic;
arch/loongarch/kvm/intc/pch_pic.c
484
mutex_lock(&kvm->slots_lock);
arch/loongarch/kvm/intc/pch_pic.c
485
kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &s->device);
arch/loongarch/kvm/intc/pch_pic.c
486
mutex_unlock(&kvm->slots_lock);
arch/loongarch/kvm/intc/pch_pic.c
71
int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, int level)
arch/loongarch/kvm/intc/pch_pic.c
75
if (cpu_has_msgint && kvm->arch.dmsintc &&
arch/loongarch/kvm/intc/pch_pic.c
76
msg_addr >= kvm->arch.dmsintc->msg_addr_base &&
arch/loongarch/kvm/intc/pch_pic.c
77
msg_addr < (kvm->arch.dmsintc->msg_addr_base + kvm->arch.dmsintc->msg_addr_size)) {
arch/loongarch/kvm/intc/pch_pic.c
78
return dmsintc_set_irq(kvm, msg_addr, e->msi.data, level);
arch/loongarch/kvm/intc/pch_pic.c
81
eiointc_set_irq(kvm->arch.eiointc, e->msi.data, level);
arch/loongarch/kvm/irqfd.c
11
struct kvm *kvm, int irq_source_id, int level, bool line_status)
arch/loongarch/kvm/irqfd.c
14
pch_pic_set_irq(kvm->arch.pch_pic, e->irqchip.pin, level);
arch/loongarch/kvm/irqfd.c
27
struct kvm *kvm, int irq_source_id, int level, bool line_status)
arch/loongarch/kvm/irqfd.c
32
return pch_msi_set_irq(kvm, e, level);
arch/loongarch/kvm/irqfd.c
44
int kvm_set_routing_entry(struct kvm *kvm,
arch/loongarch/kvm/irqfd.c
71
struct kvm *kvm, int irq_source_id, int level, bool line_status)
arch/loongarch/kvm/irqfd.c
78
pch_pic_set_irq(kvm->arch.pch_pic, e->irqchip.pin, level);
arch/loongarch/kvm/irqfd.c
81
return pch_msi_set_irq(kvm, e, level);
arch/loongarch/kvm/irqfd.c
87
bool kvm_arch_intc_initialized(struct kvm *kvm)
arch/loongarch/kvm/irqfd.c
89
return kvm_arch_irqchip_in_kernel(kvm);
arch/loongarch/kvm/main.c
211
context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
arch/loongarch/kvm/main.c
240
context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
arch/loongarch/kvm/main.c
282
void kvm_init_vmcs(struct kvm *kvm)
arch/loongarch/kvm/main.c
284
kvm->arch.vmcs = vmcs;
arch/loongarch/kvm/mmu.c
149
static kvm_pte_t *kvm_populate_gpa(struct kvm *kvm,
arch/loongarch/kvm/mmu.c
156
kvm_ptw_prepare(kvm, &ctx);
arch/loongarch/kvm/mmu.c
157
child = kvm->arch.pgd;
arch/loongarch/kvm/mmu.c
26
static inline void kvm_ptw_prepare(struct kvm *kvm, kvm_ptw_ctx *ctx)
arch/loongarch/kvm/mmu.c
28
ctx->level = kvm->arch.root_level;
arch/loongarch/kvm/mmu.c
292
static void kvm_flush_range(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn, int lock)
arch/loongarch/kvm/mmu.c
30
ctx->invalid_ptes = kvm->arch.invalid_ptes;
arch/loongarch/kvm/mmu.c
300
kvm_ptw_prepare(kvm, &ctx);
arch/loongarch/kvm/mmu.c
304
spin_lock(&kvm->mmu_lock);
arch/loongarch/kvm/mmu.c
305
ret = kvm_ptw_top(kvm->arch.pgd, start_gfn << PAGE_SHIFT,
arch/loongarch/kvm/mmu.c
307
spin_unlock(&kvm->mmu_lock);
arch/loongarch/kvm/mmu.c
309
ret = kvm_ptw_top(kvm->arch.pgd, start_gfn << PAGE_SHIFT,
arch/loongarch/kvm/mmu.c
31
ctx->pte_shifts = kvm->arch.pte_shifts;
arch/loongarch/kvm/mmu.c
314
kvm_flush_remote_tlbs(kvm);
arch/loongarch/kvm/mmu.c
34
ctx->opaque = kvm;
arch/loongarch/kvm/mmu.c
341
static int kvm_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn)
arch/loongarch/kvm/mmu.c
347
kvm_ptw_prepare(kvm, &ctx);
arch/loongarch/kvm/mmu.c
348
return kvm_ptw_top(kvm->arch.pgd, start_gfn << PAGE_SHIFT, end_gfn << PAGE_SHIFT, &ctx);
arch/loongarch/kvm/mmu.c
362
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
arch/loongarch/kvm/mmu.c
374
kvm_ptw_prepare(kvm, &ctx);
arch/loongarch/kvm/mmu.c
376
kvm_ptw_top(kvm->arch.pgd, start << PAGE_SHIFT, end << PAGE_SHIFT, &ctx);
arch/loongarch/kvm/mmu.c
379
int kvm_arch_prepare_memory_region(struct kvm *kvm, const struct kvm_memory_slot *old,
arch/loongarch/kvm/mmu.c
392
if ((new->base_gfn + new->npages) > (kvm->arch.gpa_size >> PAGE_SHIFT))
arch/loongarch/kvm/mmu.c
442
void kvm_arch_commit_memory_region(struct kvm *kvm,
arch/loongarch/kvm/mmu.c
474
if (kvm_dirty_log_manual_protect_and_init_set(kvm))
arch/loongarch/kvm/mmu.c
477
spin_lock(&kvm->mmu_lock);
arch/loongarch/kvm/mmu.c
479
needs_flush = kvm_mkclean_gpa_pt(kvm, new->base_gfn,
arch/loongarch/kvm/mmu.c
481
spin_unlock(&kvm->mmu_lock);
arch/loongarch/kvm/mmu.c
483
kvm_flush_remote_tlbs(kvm);
arch/loongarch/kvm/mmu.c
487
void kvm_arch_flush_shadow_all(struct kvm *kvm)
arch/loongarch/kvm/mmu.c
489
kvm_flush_range(kvm, 0, kvm->arch.gpa_size >> PAGE_SHIFT, 0);
arch/loongarch/kvm/mmu.c
492
void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
arch/loongarch/kvm/mmu.c
498
kvm_flush_range(kvm, slot->base_gfn, slot->base_gfn + slot->npages, 1);
arch/loongarch/kvm/mmu.c
501
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
arch/loongarch/kvm/mmu.c
507
kvm_ptw_prepare(kvm, &ctx);
arch/loongarch/kvm/mmu.c
510
return kvm_ptw_top(kvm->arch.pgd, range->start << PAGE_SHIFT,
arch/loongarch/kvm/mmu.c
514
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/loongarch/kvm/mmu.c
520
kvm_ptw_prepare(kvm, &ctx);
arch/loongarch/kvm/mmu.c
522
return kvm_ptw_top(kvm->arch.pgd, range->start << PAGE_SHIFT,
arch/loongarch/kvm/mmu.c
526
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/loongarch/kvm/mmu.c
529
kvm_pte_t *ptep = kvm_populate_gpa(kvm, NULL, gpa, 0);
arch/loongarch/kvm/mmu.c
557
struct kvm *kvm = vcpu->kvm;
arch/loongarch/kvm/mmu.c
560
spin_lock(&kvm->mmu_lock);
arch/loongarch/kvm/mmu.c
563
ptep = kvm_populate_gpa(kvm, NULL, gpa, 0);
arch/loongarch/kvm/mmu.c
582
slot = gfn_to_memslot(kvm, gfn);
arch/loongarch/kvm/mmu.c
597
spin_unlock(&kvm->mmu_lock);
arch/loongarch/kvm/mmu.c
600
mark_page_dirty(kvm, gfn);
arch/loongarch/kvm/mmu.c
604
spin_unlock(&kvm->mmu_lock);
arch/loongarch/kvm/mmu.c
666
static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn,
arch/loongarch/kvm/mmu.c
701
pgd = pgdp_get(pgd_offset(kvm->mm, hva));
arch/loongarch/kvm/mmu.c
732
struct kvm *kvm = vcpu->kvm;
arch/loongarch/kvm/mmu.c
747
kvm->stat.hugepages--;
arch/loongarch/kvm/mmu.c
748
kvm->stat.pages += PTRS_PER_PTE;
arch/loongarch/kvm/mmu.c
780
struct kvm *kvm = vcpu->kvm;
arch/loongarch/kvm/mmu.c
786
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/loongarch/kvm/mmu.c
791
memslot = gfn_to_memslot(kvm, gfn);
arch/loongarch/kvm/mmu.c
808
mmu_seq = kvm->mmu_invalidate_seq;
arch/loongarch/kvm/mmu.c
830
spin_lock(&kvm->mmu_lock);
arch/loongarch/kvm/mmu.c
831
if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) {
arch/loongarch/kvm/mmu.c
837
spin_unlock(&kvm->mmu_lock);
arch/loongarch/kvm/mmu.c
868
level = host_pfn_mapping_level(kvm, gfn, memslot);
arch/loongarch/kvm/mmu.c
875
ptep = kvm_populate_gpa(kvm, NULL, gpa, 0);
arch/loongarch/kvm/mmu.c
887
ptep = kvm_populate_gpa(kvm, memcache, gpa, level);
arch/loongarch/kvm/mmu.c
897
++kvm->stat.hugepages;
arch/loongarch/kvm/mmu.c
90
struct kvm *kvm;
arch/loongarch/kvm/mmu.c
901
++kvm->stat.pages;
arch/loongarch/kvm/mmu.c
904
kvm_release_faultin_page(kvm, page, false, writeable);
arch/loongarch/kvm/mmu.c
905
spin_unlock(&kvm->mmu_lock);
arch/loongarch/kvm/mmu.c
908
mark_page_dirty_in_slot(kvm, memslot, gfn);
arch/loongarch/kvm/mmu.c
911
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/loongarch/kvm/mmu.c
92
kvm = ctx->opaque;
arch/loongarch/kvm/mmu.c
939
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
arch/loongarch/kvm/mmu.c
94
kvm->stat.hugepages--;
arch/loongarch/kvm/mmu.c
943
void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
arch/loongarch/kvm/mmu.c
946
kvm_flush_remote_tlbs(kvm);
arch/loongarch/kvm/mmu.c
96
kvm->stat.pages--;
arch/loongarch/kvm/trace.h
13
#define TRACE_SYSTEM kvm
arch/loongarch/kvm/trace.h
216
#define TRACE_INCLUDE_PATH ../../arch/loongarch/kvm
arch/loongarch/kvm/vcpu.c
1103
val = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK;
arch/loongarch/kvm/vcpu.c
1156
struct kvm *kvm = vcpu->kvm;
arch/loongarch/kvm/vcpu.c
1168
if ((kvm->arch.pv_features & LOONGARCH_PV_FEAT_UPDATED)
arch/loongarch/kvm/vcpu.c
1169
&& ((kvm->arch.pv_features & valid) != val))
arch/loongarch/kvm/vcpu.c
1171
kvm->arch.pv_features = val | LOONGARCH_PV_FEAT_UPDATED;
arch/loongarch/kvm/vcpu.c
1183
struct kvm *kvm = vcpu->kvm;
arch/loongarch/kvm/vcpu.c
1201
idx = srcu_read_lock(&kvm->srcu);
arch/loongarch/kvm/vcpu.c
1202
if (kvm_is_error_hva(gfn_to_hva(kvm, gpa >> PAGE_SHIFT)))
arch/loongarch/kvm/vcpu.c
1204
srcu_read_unlock(&kvm->srcu, idx);
arch/loongarch/kvm/vcpu.c
1505
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
arch/loongarch/kvm/vcpu.c
1522
vcpu->arch.kvm_pgd = __pa(vcpu->kvm->arch.pgd);
arch/loongarch/kvm/vcpu.c
1528
vcpu->arch.host_pgd = (unsigned long)vcpu->kvm->mm->pgd;
arch/loongarch/kvm/vcpu.c
1587
context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
arch/loongarch/kvm/vcpu.c
1609
context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
arch/loongarch/kvm/vcpu.c
1624
write_csr_gcntc((ulong)vcpu->kvm->arch.time_offset);
arch/loongarch/kvm/vcpu.c
167
slots = kvm_memslots(vcpu->kvm);
arch/loongarch/kvm/vcpu.c
1689
if (vcpu->kvm->created_vcpus > 1)
arch/loongarch/kvm/vcpu.c
169
if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st))) {
arch/loongarch/kvm/vcpu.c
1805
if (unlikely(current->mm != vcpu->kvm->mm))
arch/loongarch/kvm/vcpu.c
1810
slots = kvm_memslots(vcpu->kvm);
arch/loongarch/kvm/vcpu.c
1812
if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st))) {
arch/loongarch/kvm/vcpu.c
1822
mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
arch/loongarch/kvm/vcpu.c
1835
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/loongarch/kvm/vcpu.c
1837
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/loongarch/kvm/vcpu.c
198
mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
arch/loongarch/kvm/vcpu.c
275
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/loongarch/kvm/vcpu.c
277
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/loongarch/kvm/vcpu.c
45
context = this_cpu_ptr(vcpu->kvm->arch.vmcs);
arch/loongarch/kvm/vcpu.c
514
map = vcpu->kvm->arch.phyid_map;
arch/loongarch/kvm/vcpu.c
517
spin_lock(&vcpu->kvm->arch.phyid_map_lock);
arch/loongarch/kvm/vcpu.c
521
spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
arch/loongarch/kvm/vcpu.c
529
spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
arch/loongarch/kvm/vcpu.c
536
spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
arch/loongarch/kvm/vcpu.c
544
spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
arch/loongarch/kvm/vcpu.c
551
spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
arch/loongarch/kvm/vcpu.c
562
map = vcpu->kvm->arch.phyid_map;
arch/loongarch/kvm/vcpu.c
568
spin_lock(&vcpu->kvm->arch.phyid_map_lock);
arch/loongarch/kvm/vcpu.c
574
spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
arch/loongarch/kvm/vcpu.c
577
struct kvm_vcpu *kvm_get_vcpu_by_cpuid(struct kvm *kvm, int cpuid)
arch/loongarch/kvm/vcpu.c
587
map = kvm->arch.phyid_map;
arch/loongarch/kvm/vcpu.c
60
context = this_cpu_ptr(vcpu->kvm->arch.vmcs);
arch/loongarch/kvm/vcpu.c
862
*v = get_cycles() + vcpu->kvm->arch.time_offset;
arch/loongarch/kvm/vcpu.c
957
vcpu->kvm->arch.time_offset = (signed long)(v - get_cycles());
arch/loongarch/kvm/vm.c
103
void kvm_arch_destroy_vm(struct kvm *kvm)
arch/loongarch/kvm/vm.c
105
kvm_destroy_vcpus(kvm);
arch/loongarch/kvm/vm.c
106
free_page((unsigned long)kvm->arch.pgd);
arch/loongarch/kvm/vm.c
107
kvm->arch.pgd = NULL;
arch/loongarch/kvm/vm.c
108
kvfree(kvm->arch.phyid_map);
arch/loongarch/kvm/vm.c
109
kvm->arch.phyid_map = NULL;
arch/loongarch/kvm/vm.c
112
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
arch/loongarch/kvm/vm.c
147
static int kvm_vm_feature_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
arch/loongarch/kvm/vm.c
161
if (kvm_vm_support(&kvm->arch, attr->attr))
arch/loongarch/kvm/vm.c
169
static int kvm_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
arch/loongarch/kvm/vm.c
173
return kvm_vm_feature_has_attr(kvm, attr);
arch/loongarch/kvm/vm.c
182
struct kvm *kvm = filp->private_data;
arch/loongarch/kvm/vm.c
192
return kvm_vm_has_attr(kvm, &attr);
arch/loongarch/kvm/vm.c
198
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event, bool line_status)
arch/loongarch/kvm/vm.c
200
if (!kvm_arch_irqchip_in_kernel(kvm))
arch/loongarch/kvm/vm.c
203
irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
arch/loongarch/kvm/vm.c
209
bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
arch/loongarch/kvm/vm.c
211
return (kvm->arch.ipi && kvm->arch.eiointc && kvm->arch.pch_pic);
arch/loongarch/kvm/vm.c
28
static void kvm_vm_init_features(struct kvm *kvm)
arch/loongarch/kvm/vm.c
33
kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_LSX);
arch/loongarch/kvm/vm.c
35
kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_LASX);
arch/loongarch/kvm/vm.c
37
kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_X86BT);
arch/loongarch/kvm/vm.c
39
kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_ARMBT);
arch/loongarch/kvm/vm.c
41
kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_MIPSBT);
arch/loongarch/kvm/vm.c
43
kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_PTW);
arch/loongarch/kvm/vm.c
45
kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_MSGINT);
arch/loongarch/kvm/vm.c
49
kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_PMU);
arch/loongarch/kvm/vm.c
52
kvm->arch.pv_features |= BIT(KVM_FEATURE_IPI);
arch/loongarch/kvm/vm.c
53
kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_PV_IPI);
arch/loongarch/kvm/vm.c
55
kvm->arch.pv_features |= BIT(KVM_FEATURE_PREEMPT);
arch/loongarch/kvm/vm.c
56
kvm->arch.pv_features |= BIT(KVM_FEATURE_STEAL_TIME);
arch/loongarch/kvm/vm.c
57
kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_PV_PREEMPT);
arch/loongarch/kvm/vm.c
58
kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_PV_STEALTIME);
arch/loongarch/kvm/vm.c
62
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
arch/loongarch/kvm/vm.c
67
kvm->arch.pgd = kvm_pgd_alloc();
arch/loongarch/kvm/vm.c
68
if (!kvm->arch.pgd)
arch/loongarch/kvm/vm.c
71
kvm->arch.phyid_map = kvzalloc_obj(struct kvm_phyid_map,
arch/loongarch/kvm/vm.c
73
if (!kvm->arch.phyid_map) {
arch/loongarch/kvm/vm.c
74
free_page((unsigned long)kvm->arch.pgd);
arch/loongarch/kvm/vm.c
75
kvm->arch.pgd = NULL;
arch/loongarch/kvm/vm.c
78
spin_lock_init(&kvm->arch.phyid_map_lock);
arch/loongarch/kvm/vm.c
80
kvm_init_vmcs(kvm);
arch/loongarch/kvm/vm.c
81
kvm_vm_init_features(kvm);
arch/loongarch/kvm/vm.c
87
kvm->arch.gpa_size = BIT(cpu_vabits);
arch/loongarch/kvm/vm.c
88
kvm->arch.root_level = CONFIG_PGTABLE_LEVELS - 1;
arch/loongarch/kvm/vm.c
89
kvm->arch.invalid_ptes[0] = 0;
arch/loongarch/kvm/vm.c
90
kvm->arch.invalid_ptes[1] = (unsigned long)invalid_pte_table;
arch/loongarch/kvm/vm.c
92
kvm->arch.invalid_ptes[2] = (unsigned long)invalid_pmd_table;
arch/loongarch/kvm/vm.c
95
kvm->arch.invalid_ptes[3] = (unsigned long)invalid_pud_table;
arch/loongarch/kvm/vm.c
97
for (i = 0; i <= kvm->arch.root_level; i++)
arch/loongarch/kvm/vm.c
98
kvm->arch.pte_shifts[i] = PAGE_SHIFT + i * (PAGE_SHIFT - 3);
arch/mips/include/asm/kvm_host.h
169
struct kvm *kvm;
arch/mips/include/asm/kvm_host.h
733
int (*check_extension)(struct kvm *kvm, long ext);
arch/mips/include/asm/kvm_host.h
737
void (*prepare_flush_shadow)(struct kvm *kvm);
arch/mips/include/asm/kvm_host.h
808
bool kvm_mips_flush_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn);
arch/mips/include/asm/kvm_host.h
809
int kvm_mips_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn);
arch/mips/include/asm/kvm_host.h
885
extern unsigned long kvm_mips_get_ramsize(struct kvm *kvm);
arch/mips/include/asm/kvm_host.h
889
static inline void kvm_arch_free_memslot(struct kvm *kvm,
arch/mips/include/asm/kvm_host.h
891
static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {}
arch/mips/kvm/entry.c
264
UASM_i_LW(&p, GPR_S0, (int)offsetof(struct kvm_vcpu, kvm) -
arch/mips/kvm/entry.c
266
UASM_i_LW(&p, GPR_A0, offsetof(struct kvm, arch.gpa_mm.pgd), GPR_S0);
arch/mips/kvm/entry.c
309
offsetof(struct kvm, arch.gpa_mm.context.asid));
arch/mips/kvm/interrupt.h
42
extern void kvm_init_loongson_ipi(struct kvm *kvm);
arch/mips/kvm/loongson_ipi.c
106
struct kvm *kvm = ipi->kvm;
arch/mips/kvm/loongson_ipi.c
125
kvm_vcpu_ioctl_interrupt(kvm_get_vcpu(kvm, id), &irq);
arch/mips/kvm/loongson_ipi.c
133
kvm_vcpu_ioctl_interrupt(kvm_get_vcpu(kvm, id), &irq);
arch/mips/kvm/loongson_ipi.c
192
void kvm_init_loongson_ipi(struct kvm *kvm)
arch/mips/kvm/loongson_ipi.c
199
s = &kvm->arch.ipi;
arch/mips/kvm/loongson_ipi.c
200
s->kvm = kvm;
arch/mips/kvm/loongson_ipi.c
210
mutex_lock(&kvm->slots_lock);
arch/mips/kvm/loongson_ipi.c
211
kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, addr, 0x400, device);
arch/mips/kvm/loongson_ipi.c
212
mutex_unlock(&kvm->slots_lock);
arch/mips/kvm/mips.c
1030
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
arch/mips/kvm/mips.c
1070
r = kvm_mips_callbacks->check_extension(kvm, ext);
arch/mips/kvm/mips.c
138
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
arch/mips/kvm/mips.c
151
kvm->arch.gpa_mm.pgd = kvm_pgd_alloc();
arch/mips/kvm/mips.c
152
if (!kvm->arch.gpa_mm.pgd)
arch/mips/kvm/mips.c
156
kvm_init_loongson_ipi(kvm);
arch/mips/kvm/mips.c
162
static void kvm_mips_free_gpa_pt(struct kvm *kvm)
arch/mips/kvm/mips.c
165
WARN_ON(!kvm_mips_flush_gpa_pt(kvm, 0, ~0));
arch/mips/kvm/mips.c
166
pgd_free(NULL, kvm->arch.gpa_mm.pgd);
arch/mips/kvm/mips.c
169
void kvm_arch_destroy_vm(struct kvm *kvm)
arch/mips/kvm/mips.c
171
kvm_destroy_vcpus(kvm);
arch/mips/kvm/mips.c
172
kvm_mips_free_gpa_pt(kvm);
arch/mips/kvm/mips.c
181
void kvm_arch_flush_shadow_all(struct kvm *kvm)
arch/mips/kvm/mips.c
184
kvm_mips_flush_gpa_pt(kvm, 0, ~0);
arch/mips/kvm/mips.c
185
kvm_flush_remote_tlbs(kvm);
arch/mips/kvm/mips.c
188
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
arch/mips/kvm/mips.c
196
spin_lock(&kvm->mmu_lock);
arch/mips/kvm/mips.c
198
kvm_mips_flush_gpa_pt(kvm, slot->base_gfn,
arch/mips/kvm/mips.c
200
kvm_flush_remote_tlbs_memslot(kvm, slot);
arch/mips/kvm/mips.c
201
spin_unlock(&kvm->mmu_lock);
arch/mips/kvm/mips.c
204
int kvm_arch_prepare_memory_region(struct kvm *kvm,
arch/mips/kvm/mips.c
212
void kvm_arch_commit_memory_region(struct kvm *kvm,
arch/mips/kvm/mips.c
231
spin_lock(&kvm->mmu_lock);
arch/mips/kvm/mips.c
233
needs_flush = kvm_mips_mkclean_gpa_pt(kvm, new->base_gfn,
arch/mips/kvm/mips.c
236
kvm_flush_remote_tlbs_memslot(kvm, new);
arch/mips/kvm/mips.c
237
spin_unlock(&kvm->mmu_lock);
arch/mips/kvm/mips.c
273
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
arch/mips/kvm/mips.c
285
vcpu->kvm, vcpu->vcpu_id, vcpu);
arch/mips/kvm/mips.c
495
dvcpu = kvm_get_vcpu(vcpu->kvm, irq->cpu);
arch/mips/kvm/mips.c
876
if (!kvm_vm_ioctl_check_extension(vcpu->kvm, cap->cap))
arch/mips/kvm/mips.c
976
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
arch/mips/kvm/mips.c
981
int kvm_arch_flush_remote_tlbs(struct kvm *kvm)
arch/mips/kvm/mips.c
983
kvm_mips_callbacks->prepare_flush_shadow(kvm);
arch/mips/kvm/mmu.c
142
static pte_t *kvm_mips_pte_for_gpa(struct kvm *kvm,
arch/mips/kvm/mmu.c
146
return kvm_mips_walk_pgd(kvm->arch.gpa_mm.pgd, cache, addr);
arch/mips/kvm/mmu.c
270
bool kvm_mips_flush_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn)
arch/mips/kvm/mmu.c
272
return kvm_mips_flush_gpa_pgd(kvm->arch.gpa_mm.pgd,
arch/mips/kvm/mmu.c
396
int kvm_mips_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn)
arch/mips/kvm/mmu.c
398
return kvm_mips_mkclean_pgd(kvm->arch.gpa_mm.pgd,
arch/mips/kvm/mmu.c
414
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
arch/mips/kvm/mmu.c
422
kvm_mips_mkclean_gpa_pt(kvm, start, end);
arch/mips/kvm/mmu.c
433
static int kvm_mips_mkold_gpa_pt(struct kvm *kvm, gfn_t start_gfn,
arch/mips/kvm/mmu.c
436
return kvm_mips_mkold_pgd(kvm->arch.gpa_mm.pgd,
arch/mips/kvm/mmu.c
441
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
arch/mips/kvm/mmu.c
443
kvm_mips_flush_gpa_pt(kvm, range->start, range->end);
arch/mips/kvm/mmu.c
447
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/mips/kvm/mmu.c
449
return kvm_mips_mkold_gpa_pt(kvm, range->start, range->end);
arch/mips/kvm/mmu.c
452
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/mips/kvm/mmu.c
455
pte_t *gpa_pte = kvm_mips_pte_for_gpa(kvm, NULL, gpa);
arch/mips/kvm/mmu.c
484
struct kvm *kvm = vcpu->kvm;
arch/mips/kvm/mmu.c
489
spin_lock(&kvm->mmu_lock);
arch/mips/kvm/mmu.c
492
ptep = kvm_mips_pte_for_gpa(kvm, NULL, gpa);
arch/mips/kvm/mmu.c
510
mark_page_dirty(kvm, gfn);
arch/mips/kvm/mmu.c
519
spin_unlock(&kvm->mmu_lock);
arch/mips/kvm/mmu.c
551
struct kvm *kvm = vcpu->kvm;
arch/mips/kvm/mmu.c
563
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/mips/kvm/mmu.c
579
mmu_seq = kvm->mmu_invalidate_seq;
arch/mips/kvm/mmu.c
600
spin_lock(&kvm->mmu_lock);
arch/mips/kvm/mmu.c
602
if (mmu_invalidate_retry(kvm, mmu_seq)) {
arch/mips/kvm/mmu.c
608
spin_unlock(&kvm->mmu_lock);
arch/mips/kvm/mmu.c
614
ptep = kvm_mips_pte_for_gpa(kvm, memcache, gpa);
arch/mips/kvm/mmu.c
622
mark_page_dirty(kvm, gfn);
arch/mips/kvm/mmu.c
636
kvm_release_faultin_page(kvm, page, false, writeable);
arch/mips/kvm/mmu.c
637
spin_unlock(&kvm->mmu_lock);
arch/mips/kvm/mmu.c
639
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/mips/kvm/tlb.c
42
struct mm_struct *gpa_mm = &vcpu->kvm->arch.gpa_mm;
arch/mips/kvm/trace.h
16
#define TRACE_SYSTEM kvm
arch/mips/kvm/vz.c
2504
struct kvm *kvm = vcpu->kvm;
arch/mips/kvm/vz.c
2505
struct mm_struct *gpa_mm = &kvm->arch.gpa_mm;
arch/mips/kvm/vz.c
2556
if (cpumask_test_and_clear_cpu(cpu, &kvm->arch.asid_flush_mask))
arch/mips/kvm/vz.c
2699
if (vcpu->kvm->created_vcpus > 1)
arch/mips/kvm/vz.c
3024
static int kvm_vz_check_extension(struct kvm *kvm, long ext)
arch/mips/kvm/vz.c
3218
static void kvm_vz_prepare_flush_shadow(struct kvm *kvm)
arch/mips/kvm/vz.c
3230
cpumask_setall(&kvm->arch.asid_flush_mask);
arch/powerpc/include/asm/kvm_book3s.h
161
extern long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr,
arch/powerpc/include/asm/kvm_book3s.h
175
extern int kvmppc_book3s_hcall_implemented(struct kvm *kvm, unsigned long hc);
arch/powerpc/include/asm/kvm_book3s.h
194
extern void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr,
arch/powerpc/include/asm/kvm_book3s.h
196
extern void kvmppc_unmap_pte(struct kvm *kvm, pte_t *pte, unsigned long gpa,
arch/powerpc/include/asm/kvm_book3s.h
200
extern bool kvmppc_hv_handle_set_rc(struct kvm *kvm, bool nested,
arch/powerpc/include/asm/kvm_book3s.h
208
extern int kvmppc_init_vm_radix(struct kvm *kvm);
arch/powerpc/include/asm/kvm_book3s.h
209
extern void kvmppc_free_radix(struct kvm *kvm);
arch/powerpc/include/asm/kvm_book3s.h
210
extern void kvmppc_free_pgtable_radix(struct kvm *kvm, pgd_t *pgd,
arch/powerpc/include/asm/kvm_book3s.h
214
extern void kvm_unmap_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
arch/powerpc/include/asm/kvm_book3s.h
216
extern bool kvm_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
arch/powerpc/include/asm/kvm_book3s.h
218
extern bool kvm_test_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
arch/powerpc/include/asm/kvm_book3s.h
220
extern long kvmppc_hv_get_dirty_log_radix(struct kvm *kvm,
arch/powerpc/include/asm/kvm_book3s.h
222
extern void kvmppc_radix_flush_memslot(struct kvm *kvm,
arch/powerpc/include/asm/kvm_book3s.h
224
extern int kvmhv_get_rmmu_info(struct kvm *kvm, struct kvm_ppc_rmmu_info *info);
arch/powerpc/include/asm/kvm_book3s.h
239
extern void kvmppc_add_revmap_chain(struct kvm *kvm, struct revmap_entry *rev,
arch/powerpc/include/asm/kvm_book3s.h
243
extern void kvmppc_invalidate_hpte(struct kvm *kvm, __be64 *hptep,
arch/powerpc/include/asm/kvm_book3s.h
245
void kvmppc_clear_ref_hpte(struct kvm *kvm, __be64 *hptep,
arch/powerpc/include/asm/kvm_book3s.h
247
extern void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long addr,
arch/powerpc/include/asm/kvm_book3s.h
249
extern void kvmppc_unpin_guest_page(struct kvm *kvm, void *addr,
arch/powerpc/include/asm/kvm_book3s.h
251
extern long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
arch/powerpc/include/asm/kvm_book3s.h
254
extern long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags,
arch/powerpc/include/asm/kvm_book3s.h
257
extern long kvmppc_hv_get_dirty_log_hpt(struct kvm *kvm,
arch/powerpc/include/asm/kvm_book3s.h
262
extern unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm,
arch/powerpc/include/asm/kvm_book3s.h
264
extern void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr,
arch/powerpc/include/asm/kvm_book3s.h
277
extern void kvmppc_pr_init_default_hcalls(struct kvm *kvm);
arch/powerpc/include/asm/kvm_book3s.h
302
void kvmhv_vm_nested_init(struct kvm *kvm);
arch/powerpc/include/asm/kvm_book3s.h
307
void kvmhv_release_all_nested(struct kvm *kvm);
arch/powerpc/include/asm/kvm_book3s.h
630
static inline bool is_kvmppc_hv_enabled(struct kvm *kvm);
arch/powerpc/include/asm/kvm_book3s.h
634
return !is_kvmppc_hv_enabled(vcpu->kvm);
arch/powerpc/include/asm/kvm_book3s.h
684
static inline u32 kvmppc_pack_vcpu_id(struct kvm *kvm, u32 id)
arch/powerpc/include/asm/kvm_book3s.h
687
int stride = kvm->arch.emul_smt_mode;
arch/powerpc/include/asm/kvm_book3s.h
87
struct kvm *kvm;
arch/powerpc/include/asm/kvm_book3s_64.h
101
int kvmhv_nested_next_lpid(struct kvm *kvm, int lpid);
arch/powerpc/include/asm/kvm_book3s_64.h
126
static inline bool kvm_is_radix(struct kvm *kvm)
arch/powerpc/include/asm/kvm_book3s_64.h
128
return kvm->arch.radix;
arch/powerpc/include/asm/kvm_book3s_64.h
138
radix = kvm_is_radix(vcpu->kvm);
arch/powerpc/include/asm/kvm_book3s_64.h
24
struct kvm *l1_host; /* L1 VM that owns this nested guest */
arch/powerpc/include/asm/kvm_book3s_64.h
525
static inline void note_hpte_modification(struct kvm *kvm,
arch/powerpc/include/asm/kvm_book3s_64.h
528
if (atomic_read(&kvm->arch.hpte_mod_interest))
arch/powerpc/include/asm/kvm_book3s_64.h
538
static inline struct kvm_memslots *kvm_memslots_raw(struct kvm *kvm)
arch/powerpc/include/asm/kvm_book3s_64.h
540
return rcu_dereference_raw_check(kvm->memslots[0]);
arch/powerpc/include/asm/kvm_book3s_64.h
543
extern void kvmppc_mmu_debugfs_init(struct kvm *kvm);
arch/powerpc/include/asm/kvm_book3s_64.h
544
extern void kvmhv_radix_debugfs_init(struct kvm *kvm);
arch/powerpc/include/asm/kvm_book3s_64.h
625
extern int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,
arch/powerpc/include/asm/kvm_book3s_64.h
629
extern void kvmhv_insert_nest_rmap(struct kvm *kvm, unsigned long *rmapp,
arch/powerpc/include/asm/kvm_book3s_64.h
631
extern void kvmhv_update_nest_rmap_rc_list(struct kvm *kvm, unsigned long *rmapp,
arch/powerpc/include/asm/kvm_book3s_64.h
634
extern void kvmhv_remove_nest_rmap_range(struct kvm *kvm,
arch/powerpc/include/asm/kvm_book3s_64.h
640
find_kvm_secondary_pte_unlocked(struct kvm *kvm, unsigned long ea,
arch/powerpc/include/asm/kvm_book3s_64.h
645
pte = __find_linux_pte(kvm->arch.pgtable, ea, NULL, hshift);
arch/powerpc/include/asm/kvm_book3s_64.h
649
static inline pte_t *find_kvm_secondary_pte(struct kvm *kvm, unsigned long ea,
arch/powerpc/include/asm/kvm_book3s_64.h
654
VM_WARN(!spin_is_locked(&kvm->mmu_lock),
arch/powerpc/include/asm/kvm_book3s_64.h
656
pte = __find_linux_pte(kvm->arch.pgtable, ea, NULL, hshift);
arch/powerpc/include/asm/kvm_book3s_64.h
661
static inline pte_t *find_kvm_host_pte(struct kvm *kvm, unsigned long mmu_seq,
arch/powerpc/include/asm/kvm_book3s_64.h
666
VM_WARN(!spin_is_locked(&kvm->mmu_lock),
arch/powerpc/include/asm/kvm_book3s_64.h
669
if (mmu_invalidate_retry(kvm, mmu_seq))
arch/powerpc/include/asm/kvm_book3s_64.h
672
pte = __find_linux_pte(kvm->mm->pgd, ea, NULL, hshift);
arch/powerpc/include/asm/kvm_book3s_64.h
677
extern pte_t *find_kvm_nested_guest_pte(struct kvm *kvm, unsigned long lpid,
arch/powerpc/include/asm/kvm_book3s_64.h
98
struct kvm_nested_guest *kvmhv_get_nested(struct kvm *kvm, int l1_lpid,
arch/powerpc/include/asm/kvm_book3s_uvmem.h
10
void kvmppc_uvmem_slot_free(struct kvm *kvm,
arch/powerpc/include/asm/kvm_book3s_uvmem.h
12
unsigned long kvmppc_h_svm_page_in(struct kvm *kvm,
arch/powerpc/include/asm/kvm_book3s_uvmem.h
16
unsigned long kvmppc_h_svm_page_out(struct kvm *kvm,
arch/powerpc/include/asm/kvm_book3s_uvmem.h
20
unsigned long kvmppc_h_svm_init_start(struct kvm *kvm);
arch/powerpc/include/asm/kvm_book3s_uvmem.h
21
unsigned long kvmppc_h_svm_init_done(struct kvm *kvm);
arch/powerpc/include/asm/kvm_book3s_uvmem.h
22
int kvmppc_send_page_to_uv(struct kvm *kvm, unsigned long gfn);
arch/powerpc/include/asm/kvm_book3s_uvmem.h
23
unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm);
arch/powerpc/include/asm/kvm_book3s_uvmem.h
25
struct kvm *kvm, bool skip_page_out);
arch/powerpc/include/asm/kvm_book3s_uvmem.h
26
int kvmppc_uvmem_memslot_create(struct kvm *kvm,
arch/powerpc/include/asm/kvm_book3s_uvmem.h
28
void kvmppc_uvmem_memslot_delete(struct kvm *kvm,
arch/powerpc/include/asm/kvm_book3s_uvmem.h
44
kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot)
arch/powerpc/include/asm/kvm_book3s_uvmem.h
50
kvmppc_uvmem_slot_free(struct kvm *kvm, const struct kvm_memory_slot *slot) { }
arch/powerpc/include/asm/kvm_book3s_uvmem.h
53
kvmppc_h_svm_page_in(struct kvm *kvm, unsigned long gra,
arch/powerpc/include/asm/kvm_book3s_uvmem.h
60
kvmppc_h_svm_page_out(struct kvm *kvm, unsigned long gra,
arch/powerpc/include/asm/kvm_book3s_uvmem.h
66
static inline unsigned long kvmppc_h_svm_init_start(struct kvm *kvm)
arch/powerpc/include/asm/kvm_book3s_uvmem.h
71
static inline unsigned long kvmppc_h_svm_init_done(struct kvm *kvm)
arch/powerpc/include/asm/kvm_book3s_uvmem.h
76
static inline unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm)
arch/powerpc/include/asm/kvm_book3s_uvmem.h
81
static inline int kvmppc_send_page_to_uv(struct kvm *kvm, unsigned long gfn)
arch/powerpc/include/asm/kvm_book3s_uvmem.h
88
struct kvm *kvm, bool skip_page_out) { }
arch/powerpc/include/asm/kvm_book3s_uvmem.h
9
int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot);
arch/powerpc/include/asm/kvm_book3s_uvmem.h
90
static inline int kvmppc_uvmem_memslot_create(struct kvm *kvm,
arch/powerpc/include/asm/kvm_book3s_uvmem.h
96
static inline void kvmppc_uvmem_memslot_delete(struct kvm *kvm,
arch/powerpc/include/asm/kvm_host.h
191
struct kvm *kvm;
arch/powerpc/include/asm/kvm_host.h
905
static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {}
arch/powerpc/include/asm/kvm_host.h
906
static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {}
arch/powerpc/include/asm/kvm_ppc.h
169
extern void kvmppc_set_hpt(struct kvm *kvm, struct kvm_hpt_info *info);
arch/powerpc/include/asm/kvm_ppc.h
170
extern int kvmppc_alloc_reset_hpt(struct kvm *kvm, int order);
arch/powerpc/include/asm/kvm_ppc.h
172
extern void kvmppc_rmap_reset(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
176
extern long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd,
arch/powerpc/include/asm/kvm_ppc.h
178
extern void kvm_spapr_tce_release_iommu_group(struct kvm *kvm,
arch/powerpc/include/asm/kvm_ppc.h
180
extern int kvmppc_switch_mmu_to_hpt(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
181
extern int kvmppc_switch_mmu_to_radix(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
182
extern void kvmppc_setup_partition_table(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
184
extern int kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
arch/powerpc/include/asm/kvm_ppc.h
202
extern int kvmppc_core_init_vm(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
203
extern void kvmppc_core_destroy_vm(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
204
extern void kvmppc_core_free_memslot(struct kvm *kvm,
arch/powerpc/include/asm/kvm_ppc.h
206
extern int kvmppc_core_prepare_memory_region(struct kvm *kvm,
arch/powerpc/include/asm/kvm_ppc.h
210
extern void kvmppc_core_commit_memory_region(struct kvm *kvm,
arch/powerpc/include/asm/kvm_ppc.h
214
extern int kvm_vm_ioctl_get_smmu_info(struct kvm *kvm,
arch/powerpc/include/asm/kvm_ppc.h
216
extern void kvmppc_core_flush_memslot(struct kvm *kvm,
arch/powerpc/include/asm/kvm_ppc.h
224
extern int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *);
arch/powerpc/include/asm/kvm_ppc.h
225
extern int kvm_vm_ioctl_resize_hpt_prepare(struct kvm *kvm,
arch/powerpc/include/asm/kvm_ppc.h
227
extern int kvm_vm_ioctl_resize_hpt_commit(struct kvm *kvm,
arch/powerpc/include/asm/kvm_ppc.h
232
extern int kvm_vm_ioctl_rtas_define_token(struct kvm *kvm, void __user *argp);
arch/powerpc/include/asm/kvm_ppc.h
234
extern void kvmppc_rtas_tokens_free(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
236
extern int kvmppc_xics_set_xive(struct kvm *kvm, u32 irq, u32 server,
arch/powerpc/include/asm/kvm_ppc.h
238
extern int kvmppc_xics_get_xive(struct kvm *kvm, u32 irq, u32 *server,
arch/powerpc/include/asm/kvm_ppc.h
240
extern int kvmppc_xics_int_on(struct kvm *kvm, u32 irq);
arch/powerpc/include/asm/kvm_ppc.h
241
extern int kvmppc_xics_int_off(struct kvm *kvm, u32 irq);
arch/powerpc/include/asm/kvm_ppc.h
277
int (*get_dirty_log)(struct kvm *kvm, struct kvm_dirty_log *log);
arch/powerpc/include/asm/kvm_ppc.h
278
void (*flush_memslot)(struct kvm *kvm, struct kvm_memory_slot *memslot);
arch/powerpc/include/asm/kvm_ppc.h
279
int (*prepare_memory_region)(struct kvm *kvm,
arch/powerpc/include/asm/kvm_ppc.h
283
void (*commit_memory_region)(struct kvm *kvm,
arch/powerpc/include/asm/kvm_ppc.h
287
bool (*unmap_gfn_range)(struct kvm *kvm, struct kvm_gfn_range *range);
arch/powerpc/include/asm/kvm_ppc.h
288
bool (*age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range);
arch/powerpc/include/asm/kvm_ppc.h
289
bool (*test_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range);
arch/powerpc/include/asm/kvm_ppc.h
291
int (*init_vm)(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
292
void (*destroy_vm)(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
293
int (*get_smmu_info)(struct kvm *kvm, struct kvm_ppc_smmu_info *info);
arch/powerpc/include/asm/kvm_ppc.h
306
int (*configure_mmu)(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg);
arch/powerpc/include/asm/kvm_ppc.h
307
int (*get_rmmu_info)(struct kvm *kvm, struct kvm_ppc_rmmu_info *info);
arch/powerpc/include/asm/kvm_ppc.h
308
int (*set_smt_mode)(struct kvm *kvm, unsigned long mode,
arch/powerpc/include/asm/kvm_ppc.h
311
int (*enable_nested)(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
316
int (*enable_svm)(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
317
int (*svm_off)(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
318
int (*enable_dawr1)(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
320
int (*create_vm_debugfs)(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
365
static inline bool is_kvmppc_hv_enabled(struct kvm *kvm)
arch/powerpc/include/asm/kvm_ppc.h
367
return kvm->arch.kvm_ops == kvmppc_hv_ops;
arch/powerpc/include/asm/kvm_ppc.h
566
vcpu->kvm->arch.kvm_ops->fast_vcpu_kick(vcpu);
arch/powerpc/include/asm/kvm_ppc.h
573
extern void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu);
arch/powerpc/include/asm/kvm_ppc.h
662
struct kvm *kvm)
arch/powerpc/include/asm/kvm_ppc.h
664
if (kvm && kvm_irq_bypass)
arch/powerpc/include/asm/kvm_ppc.h
665
return kvm->arch.pimap;
arch/powerpc/include/asm/kvm_ppc.h
671
extern void kvmppc_free_pimap(struct kvm *kvm);
arch/powerpc/include/asm/kvm_ppc.h
681
extern void kvmppc_xics_set_mapped(struct kvm *kvm, unsigned long guest_irq,
arch/powerpc/include/asm/kvm_ppc.h
683
extern void kvmppc_xics_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
arch/powerpc/include/asm/kvm_ppc.h
690
extern int kvmppc_xics_set_irq(struct kvm *kvm, int irq_source_id, u32 irq,
arch/powerpc/include/asm/kvm_ppc.h
696
struct kvm *kvm)
arch/powerpc/include/asm/kvm_ppc.h
700
static inline void kvmppc_free_pimap(struct kvm *kvm) {}
arch/powerpc/include/asm/kvm_ppc.h
721
extern int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,
arch/powerpc/include/asm/kvm_ppc.h
723
extern int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server,
arch/powerpc/include/asm/kvm_ppc.h
725
extern int kvmppc_xive_int_on(struct kvm *kvm, u32 irq);
arch/powerpc/include/asm/kvm_ppc.h
726
extern int kvmppc_xive_int_off(struct kvm *kvm, u32 irq);
arch/powerpc/include/asm/kvm_ppc.h
731
extern int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq,
arch/powerpc/include/asm/kvm_ppc.h
733
extern int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
arch/powerpc/include/asm/kvm_ppc.h
738
extern int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq,
arch/powerpc/include/asm/kvm_ppc.h
759
static inline int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,
arch/powerpc/include/asm/kvm_ppc.h
761
static inline int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server,
arch/powerpc/include/asm/kvm_ppc.h
763
static inline int kvmppc_xive_int_on(struct kvm *kvm, u32 irq) { return -1; }
arch/powerpc/include/asm/kvm_ppc.h
764
static inline int kvmppc_xive_int_off(struct kvm *kvm, u32 irq) { return -1; }
arch/powerpc/include/asm/kvm_ppc.h
769
static inline int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq,
arch/powerpc/include/asm/kvm_ppc.h
771
static inline int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
arch/powerpc/include/asm/kvm_ppc.h
776
static inline int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq,
arch/powerpc/include/asm/kvm_types.h
10
#define KVM_SUB_MODULES kvm-hv
arch/powerpc/include/asm/kvm_types.h
6
#define KVM_SUB_MODULES kvm-pr,kvm-hv
arch/powerpc/include/asm/kvm_types.h
8
#define KVM_SUB_MODULES kvm-pr
arch/powerpc/kernel/asm-offsets.c
415
OFFSET(VCPU_KVM, kvm_vcpu, kvm);
arch/powerpc/kernel/asm-offsets.c
416
OFFSET(KVM_LPID, kvm, arch.lpid);
arch/powerpc/kernel/asm-offsets.c
420
OFFSET(KVM_SDR1, kvm, arch.sdr1);
arch/powerpc/kernel/asm-offsets.c
421
OFFSET(KVM_HOST_LPID, kvm, arch.host_lpid);
arch/powerpc/kernel/asm-offsets.c
422
OFFSET(KVM_HOST_LPCR, kvm, arch.host_lpcr);
arch/powerpc/kernel/asm-offsets.c
423
OFFSET(KVM_HOST_SDR1, kvm, arch.host_sdr1);
arch/powerpc/kernel/asm-offsets.c
424
OFFSET(KVM_ENABLED_HCALLS, kvm, arch.enabled_hcalls);
arch/powerpc/kernel/asm-offsets.c
425
OFFSET(KVM_VRMA_SLB_V, kvm, arch.vrma_slb_v);
arch/powerpc/kernel/asm-offsets.c
426
OFFSET(KVM_SECURE_GUEST, kvm, arch.secure_guest);
arch/powerpc/kernel/asm-offsets.c
485
OFFSET(VCORE_KVM, kvmppc_vcore, kvm);
arch/powerpc/kvm/book3s.c
1006
srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/book3s.c
1008
srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s.c
101
if (is_kvmppc_hv_enabled(vcpu->kvm))
arch/powerpc/kvm/book3s.c
1016
int kvmppc_book3s_hcall_implemented(struct kvm *kvm, unsigned long hcall)
arch/powerpc/kvm/book3s.c
1018
return kvm->arch.kvm_ops->hcall_implemented(hcall);
arch/powerpc/kvm/book3s.c
1022
int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
arch/powerpc/kvm/book3s.c
1026
return kvmppc_xive_set_irq(kvm, irq_source_id, irq, level,
arch/powerpc/kvm/book3s.c
1029
return kvmppc_xics_set_irq(kvm, irq_source_id, irq, level,
arch/powerpc/kvm/book3s.c
1034
struct kvm *kvm, int irq_source_id,
arch/powerpc/kvm/book3s.c
1037
return kvm_set_irq(kvm, irq_source_id, irq_entry->gsi,
arch/powerpc/kvm/book3s.c
1041
struct kvm *kvm, int irq_source_id, int level,
arch/powerpc/kvm/book3s.c
1044
return kvm_set_irq(kvm, irq_source_id, e->gsi, level, line_status);
arch/powerpc/kvm/book3s.c
1047
int kvm_irq_map_gsi(struct kvm *kvm,
arch/powerpc/kvm/book3s.c
1058
int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin)
arch/powerpc/kvm/book3s.c
115
if (is_kvmppc_hv_enabled(vcpu->kvm))
arch/powerpc/kvm/book3s.c
137
vcpu->kvm->arch.kvm_ops->inject_interrupt(vcpu, vec, flags);
arch/powerpc/kvm/book3s.c
535
ret = vcpu->kvm->arch.kvm_ops->get_sregs(vcpu, sregs);
arch/powerpc/kvm/book3s.c
547
ret = vcpu->kvm->arch.kvm_ops->set_sregs(vcpu, sregs);
arch/powerpc/kvm/book3s.c
624
r = vcpu->kvm->arch.kvm_ops->get_one_reg(vcpu, id, val);
arch/powerpc/kvm/book3s.c
712
r = vcpu->kvm->arch.kvm_ops->set_one_reg(vcpu, id, val);
arch/powerpc/kvm/book3s.c
793
vcpu->kvm->arch.kvm_ops->vcpu_load(vcpu, cpu);
arch/powerpc/kvm/book3s.c
798
vcpu->kvm->arch.kvm_ops->vcpu_put(vcpu);
arch/powerpc/kvm/book3s.c
803
vcpu->kvm->arch.kvm_ops->set_msr(vcpu, msr);
arch/powerpc/kvm/book3s.c
809
return vcpu->kvm->arch.kvm_ops->vcpu_run(vcpu);
arch/powerpc/kvm/book3s.c
835
return vcpu->kvm->arch.kvm_ops->vcpu_create(vcpu);
arch/powerpc/kvm/book3s.c
840
vcpu->kvm->arch.kvm_ops->vcpu_free(vcpu);
arch/powerpc/kvm/book3s.c
845
return vcpu->kvm->arch.kvm_ops->check_requests(vcpu);
arch/powerpc/kvm/book3s.c
848
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
arch/powerpc/kvm/book3s.c
853
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
arch/powerpc/kvm/book3s.c
855
return kvm->arch.kvm_ops->get_dirty_log(kvm, log);
arch/powerpc/kvm/book3s.c
858
void kvmppc_core_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
arch/powerpc/kvm/book3s.c
860
kvm->arch.kvm_ops->free_memslot(slot);
arch/powerpc/kvm/book3s.c
863
void kvmppc_core_flush_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot)
arch/powerpc/kvm/book3s.c
865
kvm->arch.kvm_ops->flush_memslot(kvm, memslot);
arch/powerpc/kvm/book3s.c
868
int kvmppc_core_prepare_memory_region(struct kvm *kvm,
arch/powerpc/kvm/book3s.c
873
return kvm->arch.kvm_ops->prepare_memory_region(kvm, old, new, change);
arch/powerpc/kvm/book3s.c
876
void kvmppc_core_commit_memory_region(struct kvm *kvm,
arch/powerpc/kvm/book3s.c
881
kvm->arch.kvm_ops->commit_memory_region(kvm, old, new, change);
arch/powerpc/kvm/book3s.c
884
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/book3s.c
886
return kvm->arch.kvm_ops->unmap_gfn_range(kvm, range);
arch/powerpc/kvm/book3s.c
889
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/book3s.c
891
return kvm->arch.kvm_ops->age_gfn(kvm, range);
arch/powerpc/kvm/book3s.c
894
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/book3s.c
896
return kvm->arch.kvm_ops->test_age_gfn(kvm, range);
arch/powerpc/kvm/book3s.c
899
int kvmppc_core_init_vm(struct kvm *kvm)
arch/powerpc/kvm/book3s.c
903
INIT_LIST_HEAD_RCU(&kvm->arch.spapr_tce_tables);
arch/powerpc/kvm/book3s.c
904
INIT_LIST_HEAD(&kvm->arch.rtas_tokens);
arch/powerpc/kvm/book3s.c
905
mutex_init(&kvm->arch.rtas_token_lock);
arch/powerpc/kvm/book3s.c
908
return kvm->arch.kvm_ops->init_vm(kvm);
arch/powerpc/kvm/book3s.c
911
void kvmppc_core_destroy_vm(struct kvm *kvm)
arch/powerpc/kvm/book3s.c
913
kvm->arch.kvm_ops->destroy_vm(kvm);
arch/powerpc/kvm/book3s.c
916
kvmppc_rtas_tokens_free(kvm);
arch/powerpc/kvm/book3s.c
917
WARN_ON(!list_empty(&kvm->arch.spapr_tce_tables));
arch/powerpc/kvm/book3s.c
925
kfree(kvm->arch.xive_devices.native);
arch/powerpc/kvm/book3s.c
926
kvm->arch.xive_devices.native = NULL;
arch/powerpc/kvm/book3s.c
927
kfree(kvm->arch.xive_devices.xics_on_xive);
arch/powerpc/kvm/book3s.c
928
kvm->arch.xive_devices.xics_on_xive = NULL;
arch/powerpc/kvm/book3s.c
929
kfree(kvm->arch.xics_device);
arch/powerpc/kvm/book3s.c
930
kvm->arch.xics_device = NULL;
arch/powerpc/kvm/book3s.c
945
srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/book3s.c
947
srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s.h
10
extern void kvmppc_core_flush_memslot_hv(struct kvm *kvm,
arch/powerpc/kvm/book3s.h
12
extern bool kvm_unmap_gfn_range_hv(struct kvm *kvm, struct kvm_gfn_range *range);
arch/powerpc/kvm/book3s.h
13
extern bool kvm_age_gfn_hv(struct kvm *kvm, struct kvm_gfn_range *range);
arch/powerpc/kvm/book3s.h
14
extern bool kvm_test_age_gfn_hv(struct kvm *kvm, struct kvm_gfn_range *range);
arch/powerpc/kvm/book3s_32_mmu.c
115
r = gfn_to_hva(vcpu->kvm, pteg >> PAGE_SHIFT);
arch/powerpc/kvm/book3s_32_mmu.c
344
kvm_for_each_vcpu(i, v, vcpu->kvm)
arch/powerpc/kvm/book3s_32_mmu_host.c
199
mark_page_dirty(vcpu->kvm, orig_pte->raddr >> PAGE_SHIFT);
arch/powerpc/kvm/book3s_64_mmu.c
148
r = gfn_to_hva(vcpu->kvm, pteg >> PAGE_SHIFT);
arch/powerpc/kvm/book3s_64_mmu.c
244
mutex_lock(&vcpu->kvm->arch.hpt_mutex);
arch/powerpc/kvm/book3s_64_mmu.c
348
mutex_unlock(&vcpu->kvm->arch.hpt_mutex);
arch/powerpc/kvm/book3s_64_mmu.c
355
mutex_unlock(&vcpu->kvm->arch.hpt_mutex);
arch/powerpc/kvm/book3s_64_mmu.c
558
kvm_for_each_vcpu(i, v, vcpu->kvm)
arch/powerpc/kvm/book3s_64_mmu_host.c
128
mark_page_dirty(vcpu->kvm, gfn);
arch/powerpc/kvm/book3s_64_mmu_host.c
151
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_host.c
152
if (!cpte || mmu_invalidate_retry(kvm, mmu_seq)) {
arch/powerpc/kvm/book3s_64_mmu_host.c
204
kvm_release_faultin_page(kvm, page, false,
arch/powerpc/kvm/book3s_64_mmu_host.c
206
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_host.c
87
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_64_mmu_host.c
94
mmu_seq = kvm->mmu_invalidate_seq;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1005
static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
arch/powerpc/kvm/book3s_64_mmu_hv.c
1007
struct revmap_entry *rev = kvm->arch.hpt.rev;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1024
hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
arch/powerpc/kvm/book3s_64_mmu_hv.c
1043
(!hpte_is_writable(hptep1) || vcpus_running(kvm)))
arch/powerpc/kvm/book3s_64_mmu_hv.c
1062
kvmppc_invalidate_hpte(kvm, hptep, i);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1069
note_hpte_modification(kvm, &rev[i]);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1104
long kvmppc_hv_get_dirty_log_hpt(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_mmu_hv.c
1113
int npages = kvm_test_clear_dirty_npages(kvm, rmapp);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1127
void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa,
arch/powerpc/kvm/book3s_64_mmu_hv.c
1137
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1138
memslot = gfn_to_memslot(kvm, gfn);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1146
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1154
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1158
void kvmppc_unpin_guest_page(struct kvm *kvm, void *va, unsigned long gpa,
arch/powerpc/kvm/book3s_64_mmu_hv.c
1173
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1174
memslot = gfn_to_memslot(kvm, gfn);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1177
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_64_mmu_hv.c
118
void kvmppc_set_hpt(struct kvm *kvm, struct kvm_hpt_info *info)
arch/powerpc/kvm/book3s_64_mmu_hv.c
120
atomic64_set(&kvm->arch.mmio_update, 0);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1200
struct kvm *kvm = resize->kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1201
struct kvm_hpt_info *old = &kvm->arch.hpt;
arch/powerpc/kvm/book3s_64_mmu_hv.c
121
kvm->arch.hpt = *info;
arch/powerpc/kvm/book3s_64_mmu_hv.c
122
kvm->arch.sdr1 = __pa(info->virt) | (info->order - 18);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1248
int srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_64_mmu_hv.c
125
info->virt, (long)info->order, kvm->arch.lpid);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1250
__gfn_to_memslot(kvm_memslots(kvm), gfn);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1257
kvmppc_unmap_hpte(kvm, idx, memslot, rmapp, gfn);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1261
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_64_mmu_hv.c
128
int kvmppc_alloc_reset_hpt(struct kvm *kvm, int order)
arch/powerpc/kvm/book3s_64_mmu_hv.c
133
mutex_lock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
134
if (kvm->arch.mmu_ready) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
135
kvm->arch.mmu_ready = 0;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1354
struct kvm *kvm = resize->kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1358
for (i = 0; i < kvmppc_hpt_npte(&kvm->arch.hpt); i++) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
1369
struct kvm *kvm = resize->kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1377
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
138
if (atomic_read(&kvm->arch.vcpus_running)) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
1380
hpt_tmp = kvm->arch.hpt;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1381
kvmppc_set_hpt(kvm, &resize->hpt);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1384
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1386
synchronize_srcu_expedited(&kvm->srcu);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1389
kvmppc_setup_partition_table(kvm);
arch/powerpc/kvm/book3s_64_mmu_hv.c
139
kvm->arch.mmu_ready = 1;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1394
static void resize_hpt_release(struct kvm *kvm, struct kvm_resize_hpt *resize)
arch/powerpc/kvm/book3s_64_mmu_hv.c
1396
if (WARN_ON(!mutex_is_locked(&kvm->arch.mmu_setup_lock)))
arch/powerpc/kvm/book3s_64_mmu_hv.c
1408
if (kvm->arch.resize_hpt == resize)
arch/powerpc/kvm/book3s_64_mmu_hv.c
1409
kvm->arch.resize_hpt = NULL;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1417
struct kvm *kvm = resize->kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1423
mutex_lock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1426
if (kvm->arch.resize_hpt == resize) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
143
if (kvm_is_radix(kvm)) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
1430
mutex_unlock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
144
err = kvmppc_switch_mmu_to_hpt(kvm);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1443
mutex_lock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1451
if (kvm->arch.resize_hpt != resize)
arch/powerpc/kvm/book3s_64_mmu_hv.c
1452
resize_hpt_release(kvm, resize);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1454
mutex_unlock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1457
int kvm_vm_ioctl_resize_hpt_prepare(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_mmu_hv.c
1465
if (flags != 0 || kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_64_mmu_hv.c
1471
mutex_lock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1473
resize = kvm->arch.resize_hpt;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1482
resize_hpt_release(kvm, resize);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1488
resize_hpt_release(kvm, resize);
arch/powerpc/kvm/book3s_64_mmu_hv.c
149
if (kvm->arch.hpt.order == order) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
1505
resize->kvm = kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1507
kvm->arch.resize_hpt = resize;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1514
mutex_unlock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1523
int kvm_vm_ioctl_resize_hpt_commit(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_mmu_hv.c
153
memset((void *)kvm->arch.hpt.virt, 0, 1ul << order);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1531
if (flags != 0 || kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_64_mmu_hv.c
1537
mutex_lock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1539
resize = kvm->arch.resize_hpt;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1543
if (WARN_ON(!kvm->arch.mmu_ready))
arch/powerpc/kvm/book3s_64_mmu_hv.c
1547
kvm->arch.mmu_ready = 0;
arch/powerpc/kvm/book3s_64_mmu_hv.c
157
kvmppc_rmap_reset(kvm);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1570
kvm->arch.mmu_ready = 1;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1573
resize_hpt_release(kvm, resize);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1574
mutex_unlock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1597
struct kvm *kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
162
if (kvm->arch.hpt.virt) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
163
kvmppc_free_hpt(&kvm->arch.hpt);
arch/powerpc/kvm/book3s_64_mmu_hv.c
164
kvmppc_rmap_reset(kvm);
arch/powerpc/kvm/book3s_64_mmu_hv.c
170
kvmppc_set_hpt(kvm, &info);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1700
struct kvm *kvm = ctx->kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1713
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_64_mmu_hv.c
1720
hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));
arch/powerpc/kvm/book3s_64_mmu_hv.c
1721
revp = kvm->arch.hpt.rev + i;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1736
while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&
arch/powerpc/kvm/book3s_64_mmu_hv.c
1746
while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&
arch/powerpc/kvm/book3s_64_mmu_hv.c
175
cpumask_setall(&kvm->arch.need_tlb_flush);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1762
while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&
arch/powerpc/kvm/book3s_64_mmu_hv.c
177
mutex_unlock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1783
if (i >= kvmppc_hpt_npte(&kvm->arch.hpt)) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
1799
struct kvm *kvm = ctx->kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1813
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_64_mmu_hv.c
1817
mutex_lock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1818
mmu_ready = kvm->arch.mmu_ready;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1820
kvm->arch.mmu_ready = 0; /* temporarily */
arch/powerpc/kvm/book3s_64_mmu_hv.c
1823
if (atomic_read(&kvm->arch.vcpus_running)) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
1824
kvm->arch.mmu_ready = 1;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1825
mutex_unlock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1845
if (i >= kvmppc_hpt_npte(&kvm->arch.hpt) ||
arch/powerpc/kvm/book3s_64_mmu_hv.c
1846
i + hdr.n_valid + hdr.n_invalid > kvmppc_hpt_npte(&kvm->arch.hpt))
arch/powerpc/kvm/book3s_64_mmu_hv.c
1849
hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));
arch/powerpc/kvm/book3s_64_mmu_hv.c
1871
kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1873
ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r,
arch/powerpc/kvm/book3s_64_mmu_hv.c
1883
kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
arch/powerpc/kvm/book3s_64_mmu_hv.c
1887
kvmppc_update_lpcr(kvm, lpcr,
arch/powerpc/kvm/book3s_64_mmu_hv.c
1890
kvmppc_setup_partition_table(kvm);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1900
kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1910
kvm->arch.mmu_ready = mmu_ready;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1911
mutex_unlock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1924
atomic_dec(&ctx->kvm->arch.hpte_mod_interest);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1925
kvm_put_kvm(ctx->kvm);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1937
int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf)
arch/powerpc/kvm/book3s_64_mmu_hv.c
1949
kvm_get_kvm(kvm);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1950
ctx->kvm = kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1959
kvm_put_kvm_no_destroy(kvm);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1964
mutex_lock(&kvm->slots_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1965
atomic_inc(&kvm->arch.hpte_mod_interest);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1967
synchronize_srcu_expedited(&kvm->srcu);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1968
mutex_unlock(&kvm->slots_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1975
struct kvm *kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1985
struct kvm *kvm = inode->i_private;
arch/powerpc/kvm/book3s_64_mmu_hv.c
1992
kvm_get_kvm(kvm);
arch/powerpc/kvm/book3s_64_mmu_hv.c
1993
p->kvm = kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
2004
kvm_put_kvm(p->kvm);
arch/powerpc/kvm/book3s_64_mmu_hv.c
2016
struct kvm *kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
2019
kvm = p->kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
2020
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_64_mmu_hv.c
2046
hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));
arch/powerpc/kvm/book3s_64_mmu_hv.c
2047
for (; len != 0 && i < kvmppc_hpt_npte(&kvm->arch.hpt);
arch/powerpc/kvm/book3s_64_mmu_hv.c
2058
gr = kvm->arch.hpt.rev[i].guest_rpte;
arch/powerpc/kvm/book3s_64_mmu_hv.c
2106
void kvmppc_mmu_debugfs_init(struct kvm *kvm)
arch/powerpc/kvm/book3s_64_mmu_hv.c
2108
debugfs_create_file("htab", 0400, kvm->debugfs_dentry, kvm,
arch/powerpc/kvm/book3s_64_mmu_hv.c
217
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
226
if (npages > kvmppc_hpt_mask(&kvm->arch.hpt) + 1)
arch/powerpc/kvm/book3s_64_mmu_hv.c
227
npages = kvmppc_hpt_mask(&kvm->arch.hpt) + 1;
arch/powerpc/kvm/book3s_64_mmu_hv.c
238
& kvmppc_hpt_mask(&kvm->arch.hpt);
arch/powerpc/kvm/book3s_64_mmu_hv.c
248
ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, hash, hp_v, hp_r,
arch/powerpc/kvm/book3s_64_mmu_hv.c
292
static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
arch/powerpc/kvm/book3s_64_mmu_hv.c
299
ret = kvmppc_do_h_enter(kvm, flags, pte_index, pteh, ptel,
arch/powerpc/kvm/book3s_64_mmu_hv.c
300
kvm->mm->pgd, false, pte_idx_ret);
arch/powerpc/kvm/book3s_64_mmu_hv.c
344
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
353
if (kvm_is_radix(vcpu->kvm))
arch/powerpc/kvm/book3s_64_mmu_hv.c
364
slb_v = vcpu->kvm->arch.vrma_slb_v;
arch/powerpc/kvm/book3s_64_mmu_hv.c
369
index = kvmppc_hv_find_lock_hpte(kvm, eaddr, slb_v,
arch/powerpc/kvm/book3s_64_mmu_hv.c
375
hptep = (__be64 *)(kvm->arch.hpt.virt + (index << 4));
arch/powerpc/kvm/book3s_64_mmu_hv.c
379
gr = kvm->arch.hpt.rev[index].guest_rpte;
arch/powerpc/kvm/book3s_64_mmu_hv.c
447
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/book3s_64_mmu_hv.c
450
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/powerpc/kvm/book3s_64_mmu_hv.c
47
static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
arch/powerpc/kvm/book3s_64_mmu_hv.c
509
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
528
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_64_mmu_hv.c
53
struct kvm *kvm;
arch/powerpc/kvm/book3s_64_mmu_hv.c
541
mmio_update = atomic64_read(&kvm->arch.mmio_update);
arch/powerpc/kvm/book3s_64_mmu_hv.c
554
hptep = (__be64 *)(kvm->arch.hpt.virt + (index << 4));
arch/powerpc/kvm/book3s_64_mmu_hv.c
555
rev = &kvm->arch.hpt.rev[index];
arch/powerpc/kvm/book3s_64_mmu_hv.c
579
memslot = gfn_to_memslot(kvm, gfn);
arch/powerpc/kvm/book3s_64_mmu_hv.c
596
mmu_seq = kvm->mmu_invalidate_seq;
arch/powerpc/kvm/book3s_64_mmu_hv.c
615
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
616
ptep = find_kvm_host_pte(kvm, mmu_seq, hva, &shift);
arch/powerpc/kvm/book3s_64_mmu_hv.c
620
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
679
if (!kvm->arch.mmu_ready)
arch/powerpc/kvm/book3s_64_mmu_hv.c
694
if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
707
kvmppc_invalidate_hpte(kvm, hptep, index);
arch/powerpc/kvm/book3s_64_mmu_hv.c
711
kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0);
arch/powerpc/kvm/book3s_64_mmu_hv.c
739
void kvmppc_rmap_reset(struct kvm *kvm)
arch/powerpc/kvm/book3s_64_mmu_hv.c
745
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_64_mmu_hv.c
746
slots = kvm_memslots(kvm);
arch/powerpc/kvm/book3s_64_mmu_hv.c
749
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
756
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_hv.c
758
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_64_mmu_hv.c
762
static void kvmppc_unmap_hpte(struct kvm *kvm, unsigned long i,
arch/powerpc/kvm/book3s_64_mmu_hv.c
766
__be64 *hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
arch/powerpc/kvm/book3s_64_mmu_hv.c
767
struct revmap_entry *rev = kvm->arch.hpt.rev;
arch/powerpc/kvm/book3s_64_mmu_hv.c
790
kvmppc_invalidate_hpte(kvm, hptep, i);
arch/powerpc/kvm/book3s_64_mmu_hv.c
799
note_hpte_modification(kvm, &rev[i]);
arch/powerpc/kvm/book3s_64_mmu_hv.c
804
static void kvm_unmap_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
arch/powerpc/kvm/book3s_64_mmu_hv.c
825
hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
arch/powerpc/kvm/book3s_64_mmu_hv.c
834
kvmppc_unmap_hpte(kvm, i, memslot, rmapp, gfn);
arch/powerpc/kvm/book3s_64_mmu_hv.c
840
bool kvm_unmap_gfn_range_hv(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/book3s_64_mmu_hv.c
844
if (kvm_is_radix(kvm)) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
846
kvm_unmap_radix(kvm, range->slot, gfn);
arch/powerpc/kvm/book3s_64_mmu_hv.c
849
kvm_unmap_rmapp(kvm, range->slot, gfn);
arch/powerpc/kvm/book3s_64_mmu_hv.c
855
void kvmppc_core_flush_memslot_hv(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_mmu_hv.c
864
if (kvm_is_radix(kvm)) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
865
kvmppc_radix_flush_memslot(kvm, memslot);
arch/powerpc/kvm/book3s_64_mmu_hv.c
877
kvm_unmap_rmapp(kvm, memslot, gfn);
arch/powerpc/kvm/book3s_64_mmu_hv.c
882
static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
arch/powerpc/kvm/book3s_64_mmu_hv.c
885
struct revmap_entry *rev = kvm->arch.hpt.rev;
arch/powerpc/kvm/book3s_64_mmu_hv.c
905
hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
arch/powerpc/kvm/book3s_64_mmu_hv.c
923
kvmppc_clear_ref_hpte(kvm, hptep, i);
arch/powerpc/kvm/book3s_64_mmu_hv.c
926
note_hpte_modification(kvm, &rev[i]);
arch/powerpc/kvm/book3s_64_mmu_hv.c
937
bool kvm_age_gfn_hv(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/book3s_64_mmu_hv.c
942
if (kvm_is_radix(kvm)) {
arch/powerpc/kvm/book3s_64_mmu_hv.c
944
ret |= kvm_age_radix(kvm, range->slot, gfn);
arch/powerpc/kvm/book3s_64_mmu_hv.c
947
ret |= kvm_age_rmapp(kvm, range->slot, gfn);
arch/powerpc/kvm/book3s_64_mmu_hv.c
953
static bool kvm_test_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
arch/powerpc/kvm/book3s_64_mmu_hv.c
956
struct revmap_entry *rev = kvm->arch.hpt.rev;
arch/powerpc/kvm/book3s_64_mmu_hv.c
973
hp = (unsigned long *)(kvm->arch.hpt.virt + (i << 4));
arch/powerpc/kvm/book3s_64_mmu_hv.c
986
bool kvm_test_age_gfn_hv(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/book3s_64_mmu_hv.c
990
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_64_mmu_hv.c
991
return kvm_test_age_radix(kvm, range->slot, range->start);
arch/powerpc/kvm/book3s_64_mmu_hv.c
993
return kvm_test_age_rmapp(kvm, range->slot, range->start);
arch/powerpc/kvm/book3s_64_mmu_hv.c
996
static int vcpus_running(struct kvm *kvm)
arch/powerpc/kvm/book3s_64_mmu_hv.c
998
return atomic_read(&kvm->arch.vcpus_running) != 0;
arch/powerpc/kvm/book3s_64_mmu_radix.c
1011
void kvm_unmap_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
arch/powerpc/kvm/book3s_64_mmu_radix.c
1018
if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE) {
arch/powerpc/kvm/book3s_64_mmu_radix.c
1019
uv_page_inval(kvm->arch.lpid, gpa, PAGE_SHIFT);
arch/powerpc/kvm/book3s_64_mmu_radix.c
102
int lpid = vcpu->kvm->arch.lpid;
arch/powerpc/kvm/book3s_64_mmu_radix.c
1023
ptep = find_kvm_secondary_pte(kvm, gpa, &shift);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1025
kvmppc_unmap_pte(kvm, ptep, gpa, shift, memslot,
arch/powerpc/kvm/book3s_64_mmu_radix.c
1026
kvm->arch.lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1030
bool kvm_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
arch/powerpc/kvm/book3s_64_mmu_radix.c
1039
if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
arch/powerpc/kvm/book3s_64_mmu_radix.c
1042
ptep = find_kvm_secondary_pte(kvm, gpa, &shift);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1044
old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_ACCESSED, 0,
arch/powerpc/kvm/book3s_64_mmu_radix.c
1049
kvmhv_update_nest_rmap_rc_list(kvm, rmapp, _PAGE_ACCESSED, 0,
arch/powerpc/kvm/book3s_64_mmu_radix.c
1058
bool kvm_test_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
arch/powerpc/kvm/book3s_64_mmu_radix.c
1067
if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
arch/powerpc/kvm/book3s_64_mmu_radix.c
1070
ptep = find_kvm_secondary_pte(kvm, gpa, &shift);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1077
static int kvm_radix_test_clear_dirty(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_mmu_radix.c
1087
if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
arch/powerpc/kvm/book3s_64_mmu_radix.c
1094
ptep = find_kvm_secondary_pte_unlocked(kvm, gpa, &shift);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1100
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1112
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1119
old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_DIRTY, 0,
arch/powerpc/kvm/book3s_64_mmu_radix.c
1121
kvmppc_radix_tlbie_page(kvm, gpa, shift, kvm->arch.lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1124
kvmhv_update_nest_rmap_rc_list(kvm, rmapp, _PAGE_DIRTY, 0,
arch/powerpc/kvm/book3s_64_mmu_radix.c
1127
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1132
long kvmppc_hv_get_dirty_log_radix(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_mmu_radix.c
1139
npages = kvm_radix_test_clear_dirty(kvm, memslot, i);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1157
void kvmppc_radix_flush_memslot(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_mmu_radix.c
1165
if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START)
arch/powerpc/kvm/book3s_64_mmu_radix.c
1166
kvmppc_uvmem_drop_pages(memslot, kvm, true);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1168
if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
arch/powerpc/kvm/book3s_64_mmu_radix.c
1172
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1174
ptep = find_kvm_secondary_pte(kvm, gpa, &shift);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1176
kvmppc_unmap_pte(kvm, ptep, gpa, shift, memslot,
arch/powerpc/kvm/book3s_64_mmu_radix.c
1177
kvm->arch.lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1184
kvm->mmu_invalidate_seq++;
arch/powerpc/kvm/book3s_64_mmu_radix.c
1185
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1198
int kvmhv_get_rmmu_info(struct kvm *kvm, struct kvm_ppc_rmmu_info *info)
arch/powerpc/kvm/book3s_64_mmu_radix.c
1225
int kvmppc_init_vm_radix(struct kvm *kvm)
arch/powerpc/kvm/book3s_64_mmu_radix.c
1227
kvm->arch.pgtable = pgd_alloc(kvm->mm);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1228
if (!kvm->arch.pgtable)
arch/powerpc/kvm/book3s_64_mmu_radix.c
1244
struct kvm *kvm;
arch/powerpc/kvm/book3s_64_mmu_radix.c
1256
struct kvm *kvm = inode->i_private;
arch/powerpc/kvm/book3s_64_mmu_radix.c
1263
kvm_get_kvm(kvm);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1264
p->kvm = kvm;
arch/powerpc/kvm/book3s_64_mmu_radix.c
1275
kvm_put_kvm(p->kvm);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1286
struct kvm *kvm;
arch/powerpc/kvm/book3s_64_mmu_radix.c
1298
kvm = p->kvm;
arch/powerpc/kvm/book3s_64_mmu_radix.c
1299
if (!kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_64_mmu_radix.c
1335
p->lpid = kvmhv_nested_next_lpid(kvm, p->lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1342
pgt = kvm->arch.pgtable;
arch/powerpc/kvm/book3s_64_mmu_radix.c
1344
nested = kvmhv_get_nested(kvm, p->lpid, false);
arch/powerpc/kvm/book3s_64_mmu_radix.c
1447
void kvmhv_radix_debugfs_init(struct kvm *kvm)
arch/powerpc/kvm/book3s_64_mmu_radix.c
1449
debugfs_create_file("radix", 0400, kvm->debugfs_dentry, kvm,
arch/powerpc/kvm/book3s_64_mmu_radix.c
146
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_64_mmu_radix.c
180
ret = kvm_read_guest(kvm, addr, &rpte, sizeof(rpte));
arch/powerpc/kvm/book3s_64_mmu_radix.c
242
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_64_mmu_radix.c
258
ret = kvm_read_guest(kvm, ptbl, &entry, sizeof(entry));
arch/powerpc/kvm/book3s_64_mmu_radix.c
289
vcpu->kvm->arch.process_table, pid, &pte);
arch/powerpc/kvm/book3s_64_mmu_radix.c
315
void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr,
arch/powerpc/kvm/book3s_64_mmu_radix.c
353
static void kvmppc_radix_flush_pwc(struct kvm *kvm, u64 lpid)
arch/powerpc/kvm/book3s_64_mmu_radix.c
374
static unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep,
arch/powerpc/kvm/book3s_64_mmu_radix.c
381
static void kvmppc_radix_set_pte_at(struct kvm *kvm, unsigned long addr,
arch/powerpc/kvm/book3s_64_mmu_radix.c
384
radix__set_pte_at(kvm->mm, addr, ptep, pte, 0);
arch/powerpc/kvm/book3s_64_mmu_radix.c
423
void kvmppc_unmap_pte(struct kvm *kvm, pte_t *pte, unsigned long gpa,
arch/powerpc/kvm/book3s_64_mmu_radix.c
434
old = kvmppc_radix_update_pte(kvm, pte, ~0UL, 0, gpa, shift);
arch/powerpc/kvm/book3s_64_mmu_radix.c
435
kvmppc_radix_tlbie_page(kvm, gpa, shift, lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
438
if (lpid != kvm->arch.lpid)
arch/powerpc/kvm/book3s_64_mmu_radix.c
442
memslot = gfn_to_memslot(kvm, gfn);
arch/powerpc/kvm/book3s_64_mmu_radix.c
449
kvm->stat.num_2M_pages--;
arch/powerpc/kvm/book3s_64_mmu_radix.c
451
kvm->stat.num_1G_pages--;
arch/powerpc/kvm/book3s_64_mmu_radix.c
456
kvmhv_remove_nest_rmap_range(kvm, memslot, gpa, hpa, page_size);
arch/powerpc/kvm/book3s_64_mmu_radix.c
476
static void kvmppc_unmap_free_pte(struct kvm *kvm, pte_t *pte, bool full,
arch/powerpc/kvm/book3s_64_mmu_radix.c
488
kvmppc_unmap_pte(kvm, p,
arch/powerpc/kvm/book3s_64_mmu_radix.c
497
static void kvmppc_unmap_free_pmd(struct kvm *kvm, pmd_t *pmd, bool full,
arch/powerpc/kvm/book3s_64_mmu_radix.c
511
kvmppc_unmap_pte(kvm, (pte_t *)p,
arch/powerpc/kvm/book3s_64_mmu_radix.c
519
kvmppc_unmap_free_pte(kvm, pte, full, lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
526
static void kvmppc_unmap_free_pud(struct kvm *kvm, pud_t *pud,
arch/powerpc/kvm/book3s_64_mmu_radix.c
541
kvmppc_unmap_free_pmd(kvm, pmd, true, lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
545
pud_free(kvm->mm, pud);
arch/powerpc/kvm/book3s_64_mmu_radix.c
548
void kvmppc_free_pgtable_radix(struct kvm *kvm, pgd_t *pgd, u64 lpid)
arch/powerpc/kvm/book3s_64_mmu_radix.c
559
kvmppc_unmap_free_pud(kvm, pud, lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
564
void kvmppc_free_radix(struct kvm *kvm)
arch/powerpc/kvm/book3s_64_mmu_radix.c
566
if (kvm->arch.pgtable) {
arch/powerpc/kvm/book3s_64_mmu_radix.c
567
kvmppc_free_pgtable_radix(kvm, kvm->arch.pgtable,
arch/powerpc/kvm/book3s_64_mmu_radix.c
568
kvm->arch.lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
569
pgd_free(kvm->mm, kvm->arch.pgtable);
arch/powerpc/kvm/book3s_64_mmu_radix.c
570
kvm->arch.pgtable = NULL;
arch/powerpc/kvm/book3s_64_mmu_radix.c
574
static void kvmppc_unmap_free_pmd_entry_table(struct kvm *kvm, pmd_t *pmd,
arch/powerpc/kvm/book3s_64_mmu_radix.c
585
kvmppc_radix_flush_pwc(kvm, lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
587
kvmppc_unmap_free_pte(kvm, pte, false, lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
590
static void kvmppc_unmap_free_pud_entry_table(struct kvm *kvm, pud_t *pud,
arch/powerpc/kvm/book3s_64_mmu_radix.c
601
kvmppc_radix_flush_pwc(kvm, lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
603
kvmppc_unmap_free_pmd(kvm, pmd, false, lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
615
int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,
arch/powerpc/kvm/book3s_64_mmu_radix.c
635
new_pud = pud_alloc_one(kvm->mm, gpa);
arch/powerpc/kvm/book3s_64_mmu_radix.c
647
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_radix.c
649
if (mmu_invalidate_retry(kvm, mmu_seq))
arch/powerpc/kvm/book3s_64_mmu_radix.c
657
p4d_populate(kvm->mm, p4d, new_pud);
arch/powerpc/kvm/book3s_64_mmu_radix.c
673
kvmppc_radix_update_pte(kvm, (pte_t *)pud,
arch/powerpc/kvm/book3s_64_mmu_radix.c
687
kvmppc_unmap_pte(kvm, (pte_t *)pud, hgpa, PUD_SHIFT, NULL,
arch/powerpc/kvm/book3s_64_mmu_radix.c
697
kvmppc_unmap_free_pud_entry_table(kvm, pud, gpa, lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
699
kvmppc_radix_set_pte_at(kvm, gpa, (pte_t *)pud, pte);
arch/powerpc/kvm/book3s_64_mmu_radix.c
701
kvmhv_insert_nest_rmap(kvm, rmapp, n_rmap);
arch/powerpc/kvm/book3s_64_mmu_radix.c
708
pud_populate(kvm->mm, pud, new_pmd);
arch/powerpc/kvm/book3s_64_mmu_radix.c
724
kvmppc_radix_update_pte(kvm, pmdp_ptep(pmd),
arch/powerpc/kvm/book3s_64_mmu_radix.c
739
kvmppc_unmap_pte(kvm, pmdp_ptep(pmd), lgpa, PMD_SHIFT, NULL,
arch/powerpc/kvm/book3s_64_mmu_radix.c
749
kvmppc_unmap_free_pmd_entry_table(kvm, pmd, gpa, lpid);
arch/powerpc/kvm/book3s_64_mmu_radix.c
751
kvmppc_radix_set_pte_at(kvm, gpa, pmdp_ptep(pmd), pte);
arch/powerpc/kvm/book3s_64_mmu_radix.c
753
kvmhv_insert_nest_rmap(kvm, rmapp, n_rmap);
arch/powerpc/kvm/book3s_64_mmu_radix.c
760
pmd_populate(kvm->mm, pmd, new_ptep);
arch/powerpc/kvm/book3s_64_mmu_radix.c
773
kvmppc_radix_update_pte(kvm, ptep, 0, pte_val(pte), gpa, 0);
arch/powerpc/kvm/book3s_64_mmu_radix.c
777
kvmppc_radix_set_pte_at(kvm, gpa, ptep, pte);
arch/powerpc/kvm/book3s_64_mmu_radix.c
779
kvmhv_insert_nest_rmap(kvm, rmapp, n_rmap);
arch/powerpc/kvm/book3s_64_mmu_radix.c
783
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_radix.c
785
pud_free(kvm->mm, new_pud);
arch/powerpc/kvm/book3s_64_mmu_radix.c
793
bool kvmppc_hv_handle_set_rc(struct kvm *kvm, bool nested, bool writing,
arch/powerpc/kvm/book3s_64_mmu_radix.c
810
ptep = find_kvm_nested_guest_pte(kvm, lpid, gpa, &shift);
arch/powerpc/kvm/book3s_64_mmu_radix.c
812
ptep = find_kvm_secondary_pte(kvm, gpa, &shift);
arch/powerpc/kvm/book3s_64_mmu_radix.c
815
kvmppc_radix_update_pte(kvm, ptep, 0, pgflags, gpa, shift);
arch/powerpc/kvm/book3s_64_mmu_radix.c
827
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_64_mmu_radix.c
839
mmu_seq = kvm->mmu_invalidate_seq;
arch/powerpc/kvm/book3s_64_mmu_radix.c
852
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_radix.c
853
ptep = find_kvm_host_pte(kvm, mmu_seq, hva, &shift);
arch/powerpc/kvm/book3s_64_mmu_radix.c
857
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_radix.c
902
ret = kvmppc_create_pte(kvm, kvm->arch.pgtable, pte, gpa, level,
arch/powerpc/kvm/book3s_64_mmu_radix.c
903
mmu_seq, kvm->arch.lpid, NULL, NULL);
arch/powerpc/kvm/book3s_64_mmu_radix.c
918
kvm->stat.num_2M_pages++;
arch/powerpc/kvm/book3s_64_mmu_radix.c
920
kvm->stat.num_1G_pages++;
arch/powerpc/kvm/book3s_64_mmu_radix.c
929
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_64_mmu_radix.c
956
if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
arch/powerpc/kvm/book3s_64_mmu_radix.c
957
return kvmppc_send_page_to_uv(kvm, gfn);
arch/powerpc/kvm/book3s_64_mmu_radix.c
960
memslot = gfn_to_memslot(kvm, gfn);
arch/powerpc/kvm/book3s_64_mmu_radix.c
990
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_mmu_radix.c
991
if (kvmppc_hv_handle_set_rc(kvm, false, writing,
arch/powerpc/kvm/book3s_64_mmu_radix.c
992
gpa, kvm->arch.lpid))
arch/powerpc/kvm/book3s_64_mmu_radix.c
994
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_64_vio.c
109
long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd,
arch/powerpc/kvm/book3s_64_vio.c
124
list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) {
arch/powerpc/kvm/book3s_64_vio.c
260
struct kvm *kvm = stt->kvm;
arch/powerpc/kvm/book3s_64_vio.c
262
mutex_lock(&kvm->lock);
arch/powerpc/kvm/book3s_64_vio.c
264
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/book3s_64_vio.c
274
account_locked_vm(kvm->mm,
arch/powerpc/kvm/book3s_64_vio.c
277
kvm_put_kvm(stt->kvm);
arch/powerpc/kvm/book3s_64_vio.c
289
int kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_vio.c
294
struct mm_struct *mm = kvm->mm;
arch/powerpc/kvm/book3s_64_vio.c
316
stt->kvm = kvm;
arch/powerpc/kvm/book3s_64_vio.c
320
mutex_lock(&kvm->lock);
arch/powerpc/kvm/book3s_64_vio.c
324
list_for_each_entry(siter, &kvm->arch.spapr_tce_tables, list) {
arch/powerpc/kvm/book3s_64_vio.c
331
kvm_get_kvm(kvm);
arch/powerpc/kvm/book3s_64_vio.c
337
list_add_rcu(&stt->list, &kvm->arch.spapr_tce_tables);
arch/powerpc/kvm/book3s_64_vio.c
339
kvm_put_kvm_no_destroy(kvm);
arch/powerpc/kvm/book3s_64_vio.c
341
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/book3s_64_vio.c
352
static long kvmppc_tce_to_ua(struct kvm *kvm, unsigned long tce,
arch/powerpc/kvm/book3s_64_vio.c
358
memslot = __gfn_to_memslot(kvm_memslots(kvm), gfn);
arch/powerpc/kvm/book3s_64_vio.c
36
static struct kvmppc_spapr_tce_table *kvmppc_find_table(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_vio.c
383
if (kvmppc_tce_to_ua(stt->kvm, tce, &ua))
arch/powerpc/kvm/book3s_64_vio.c
392
mem = mm_iommu_lookup(stt->kvm->mm, ua, 1ULL << shift);
arch/powerpc/kvm/book3s_64_vio.c
41
list_for_each_entry_lockless(stt, &kvm->arch.spapr_tce_tables, list)
arch/powerpc/kvm/book3s_64_vio.c
448
static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_vio.c
458
mem = mm_iommu_lookup(kvm->mm, be64_to_cpu(*pua), pgsize);
arch/powerpc/kvm/book3s_64_vio.c
469
static long kvmppc_tce_iommu_do_unmap(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_vio.c
476
if (WARN_ON_ONCE(iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa,
arch/powerpc/kvm/book3s_64_vio.c
483
ret = kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);
arch/powerpc/kvm/book3s_64_vio.c
485
iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir);
arch/powerpc/kvm/book3s_64_vio.c
490
static long kvmppc_tce_iommu_unmap(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_vio.c
499
ret = kvmppc_tce_iommu_do_unmap(kvm, tbl, io_entry + i);
arch/powerpc/kvm/book3s_64_vio.c
509
static long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,
arch/powerpc/kvm/book3s_64_vio.c
522
mem = mm_iommu_lookup(kvm->mm, ua, 1ULL << tbl->it_page_shift);
arch/powerpc/kvm/book3s_64_vio.c
533
ret = iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir);
arch/powerpc/kvm/book3s_64_vio.c
540
kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);
arch/powerpc/kvm/book3s_64_vio.c
547
static long kvmppc_tce_iommu_map(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_vio.c
559
ret = kvmppc_tce_iommu_do_map(kvm, tbl,
arch/powerpc/kvm/book3s_64_vio.c
582
stt = kvmppc_find_table(vcpu->kvm, liobn);
arch/powerpc/kvm/book3s_64_vio.c
590
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/book3s_64_vio.c
598
if ((dir != DMA_NONE) && kvmppc_tce_to_ua(vcpu->kvm, tce, &ua)) {
arch/powerpc/kvm/book3s_64_vio.c
607
ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt,
arch/powerpc/kvm/book3s_64_vio.c
610
ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, stit->tbl,
arch/powerpc/kvm/book3s_64_vio.c
615
kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry);
arch/powerpc/kvm/book3s_64_vio.c
623
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/powerpc/kvm/book3s_64_vio.c
640
stt = kvmppc_find_table(vcpu->kvm, liobn);
arch/powerpc/kvm/book3s_64_vio.c
659
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/book3s_64_vio.c
660
if (kvmppc_tce_to_ua(vcpu->kvm, tce_list, &ua)) {
arch/powerpc/kvm/book3s_64_vio.c
695
if (kvmppc_tce_to_ua(vcpu->kvm, tce, &ua)) {
arch/powerpc/kvm/book3s_64_vio.c
701
ret = kvmppc_tce_iommu_map(vcpu->kvm, stt,
arch/powerpc/kvm/book3s_64_vio.c
706
kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl,
arch/powerpc/kvm/book3s_64_vio.c
716
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/powerpc/kvm/book3s_64_vio.c
730
stt = kvmppc_find_table(vcpu->kvm, liobn);
arch/powerpc/kvm/book3s_64_vio.c
746
ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt,
arch/powerpc/kvm/book3s_64_vio.c
756
kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry + i);
arch/powerpc/kvm/book3s_64_vio.c
776
stt = kvmppc_find_table(vcpu->kvm, liobn);
arch/powerpc/kvm/book3s_64_vio.c
81
void kvm_spapr_tce_release_iommu_group(struct kvm *kvm,
arch/powerpc/kvm/book3s_64_vio.c
90
list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) {
arch/powerpc/kvm/book3s_hv.c
1006
mark_page_dirty(kvm, to >> PAGE_SHIFT);
arch/powerpc/kvm/book3s_hv.c
1028
ret = kvmppc_copy_guest(vcpu->kvm, dest, src, pg_sz);
arch/powerpc/kvm/book3s_hv.c
1032
ret = kvm_clear_guest(vcpu->kvm, dest, pg_sz);
arch/powerpc/kvm/book3s_hv.c
1115
if (!kvm_is_radix(vcpu->kvm))
arch/powerpc/kvm/book3s_hv.c
1125
if (!nesting_enabled(vcpu->kvm))
arch/powerpc/kvm/book3s_hv.c
1139
do_h_rpt_invalidate_prt(id, vcpu->kvm->arch.lpid,
arch/powerpc/kvm/book3s_hv.c
1146
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
1154
!test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
arch/powerpc/kvm/book3s_hv.c
1208
tvcpu = kvmppc_find_vcpu(kvm, target);
arch/powerpc/kvm/book3s_hv.c
1222
tvcpu = kvmppc_find_vcpu(kvm, target);
arch/powerpc/kvm/book3s_hv.c
1238
if (list_empty(&kvm->arch.rtas_tokens))
arch/powerpc/kvm/book3s_hv.c
1241
idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv.c
1243
srcu_read_unlock(&kvm->srcu, idx);
arch/powerpc/kvm/book3s_hv.c
1342
if (nesting_enabled(kvm))
arch/powerpc/kvm/book3s_hv.c
1347
if (!nesting_enabled(kvm))
arch/powerpc/kvm/book3s_hv.c
1362
if (nesting_enabled(kvm))
arch/powerpc/kvm/book3s_hv.c
1367
if (nesting_enabled(kvm))
arch/powerpc/kvm/book3s_hv.c
1378
ret = kvmppc_h_svm_page_in(kvm,
arch/powerpc/kvm/book3s_hv.c
1386
ret = kvmppc_h_svm_page_out(kvm,
arch/powerpc/kvm/book3s_hv.c
1394
ret = kvmppc_h_svm_init_start(kvm);
arch/powerpc/kvm/book3s_hv.c
1399
ret = kvmppc_h_svm_init_done(kvm);
arch/powerpc/kvm/book3s_hv.c
1409
ret = kvmppc_h_svm_init_abort(kvm);
arch/powerpc/kvm/book3s_hv.c
1507
nthreads = vcpu->kvm->arch.emul_smt_mode;
arch/powerpc/kvm/book3s_hv.c
1511
v = kvmppc_find_vcpu(vcpu->kvm, cpu);
arch/powerpc/kvm/book3s_hv.c
1538
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
1548
thr = vcpu->vcpu_id & (kvm->arch.emul_smt_mode - 1);
arch/powerpc/kvm/book3s_hv.c
1555
if (arg >= kvm->arch.emul_smt_mode)
arch/powerpc/kvm/book3s_hv.c
1557
tvcpu = kvmppc_find_vcpu(kvm, vcpu->vcpu_id - thr + arg);
arch/powerpc/kvm/book3s_hv.c
1695
if (!vcpu->kvm->arch.fwnmi_enabled) {
arch/powerpc/kvm/book3s_hv.c
1795
if (kvm_is_radix(vcpu->kvm) || !cpu_has_feature(CPU_FTR_ARCH_300)) {
arch/powerpc/kvm/book3s_hv.c
1815
vsid = vcpu->kvm->arch.vrma_slb_v;
arch/powerpc/kvm/book3s_hv.c
1840
if (kvm_is_radix(vcpu->kvm) || !cpu_has_feature(CPU_FTR_ARCH_300)) {
arch/powerpc/kvm/book3s_hv.c
1862
vsid = vcpu->kvm->arch.vrma_slb_v;
arch/powerpc/kvm/book3s_hv.c
2036
srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/book3s_hv.c
2038
srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv.c
2046
srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/book3s_hv.c
2048
srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv.c
2142
unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm, unsigned long lpcr)
arch/powerpc/kvm/book3s_hv.c
2145
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv.c
2159
if (kvm_is_radix(kvm) && cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
arch/powerpc/kvm/book3s_hv.c
2172
static void verify_lpcr(struct kvm *kvm, unsigned long lpcr)
arch/powerpc/kvm/book3s_hv.c
2174
if (lpcr != kvmppc_filter_lpcr_hv(kvm, lpcr)) {
arch/powerpc/kvm/book3s_hv.c
2176
lpcr, kvmppc_filter_lpcr_hv(kvm, lpcr));
arch/powerpc/kvm/book3s_hv.c
2183
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
2202
new_lpcr = kvmppc_filter_lpcr_hv(kvm,
arch/powerpc/kvm/book3s_hv.c
2213
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_hv.c
2474
*val = get_reg_val(id, vcpu->kvm->arch.l1_ptcr);
arch/powerpc/kvm/book3s_hv.c
2761
vcpu->kvm->arch.l1_ptcr = set_reg_val(id, *val);
arch/powerpc/kvm/book3s_hv.c
2781
static int threads_per_vcore(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
2788
static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int id)
arch/powerpc/kvm/book3s_hv.c
2801
vcore->lpcr = kvm->arch.lpcr;
arch/powerpc/kvm/book3s_hv.c
2803
vcore->kvm = kvm;
arch/powerpc/kvm/book3s_hv.c
2848
kvm_get_kvm(vcpu->kvm);
arch/powerpc/kvm/book3s_hv.c
2859
kvm_put_kvm(p->vcpu->kvm);
arch/powerpc/kvm/book3s_hv.c
2965
struct kvm *kvm;
arch/powerpc/kvm/book3s_hv.c
2968
kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
3043
mutex_lock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
3047
if (id >= (KVM_MAX_VCPUS * kvm->arch.emul_smt_mode)) {
arch/powerpc/kvm/book3s_hv.c
3051
BUG_ON(kvm->arch.smt_mode != 1);
arch/powerpc/kvm/book3s_hv.c
3052
core = kvmppc_pack_vcpu_id(kvm, id);
arch/powerpc/kvm/book3s_hv.c
3055
core = id / kvm->arch.smt_mode;
arch/powerpc/kvm/book3s_hv.c
3058
vcore = kvm->arch.vcores[core];
arch/powerpc/kvm/book3s_hv.c
3068
vcore = kvmppc_vcore_create(kvm,
arch/powerpc/kvm/book3s_hv.c
3069
id & ~(kvm->arch.smt_mode - 1));
arch/powerpc/kvm/book3s_hv.c
3070
mutex_lock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_hv.c
3071
kvm->arch.vcores[core] = vcore;
arch/powerpc/kvm/book3s_hv.c
3072
kvm->arch.online_vcores++;
arch/powerpc/kvm/book3s_hv.c
3073
mutex_unlock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_hv.c
3076
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
3095
static int kvmhv_set_smt_mode(struct kvm *kvm, unsigned long smt_mode,
arch/powerpc/kvm/book3s_hv.c
3120
mutex_lock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
3122
if (!kvm->arch.online_vcores) {
arch/powerpc/kvm/book3s_hv.c
3123
kvm->arch.smt_mode = smt_mode;
arch/powerpc/kvm/book3s_hv.c
3124
kvm->arch.emul_smt_mode = esmt;
arch/powerpc/kvm/book3s_hv.c
3127
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
3132
static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
arch/powerpc/kvm/book3s_hv.c
3135
kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
arch/powerpc/kvm/book3s_hv.c
3142
unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
arch/powerpc/kvm/book3s_hv.c
3143
unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
arch/powerpc/kvm/book3s_hv.c
3144
unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
arch/powerpc/kvm/book3s_hv.c
3237
static DEFINE_PER_CPU(struct kvm *, cpu_in_guest);
arch/powerpc/kvm/book3s_hv.c
3239
static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
arch/powerpc/kvm/book3s_hv.c
3248
need_tlb_flush = &kvm->arch.need_tlb_flush;
arch/powerpc/kvm/book3s_hv.c
3266
struct kvm *running = *per_cpu_ptr(&cpu_in_guest, i);
arch/powerpc/kvm/book3s_hv.c
3268
if (running == kvm)
arch/powerpc/kvm/book3s_hv.c
3276
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
3286
if (kvm->arch.lpcr & LPCR_GTSE)
arch/powerpc/kvm/book3s_hv.c
3295
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
3321
radix_flush_cpu(kvm, prev_cpu, vcpu);
arch/powerpc/kvm/book3s_hv.c
3451
if (vc->num_threads < threads_per_vcore(vc->kvm)) {
arch/powerpc/kvm/book3s_hv.c
3547
if (one_vm_per_core && vc->kvm != cip->vc[0]->kvm)
arch/powerpc/kvm/book3s_hv.c
3609
if (!pvc->n_runnable || !pvc->kvm->arch.mmu_ready) {
arch/powerpc/kvm/book3s_hv.c
3638
if (!vc->kvm->arch.mmu_ready)
arch/powerpc/kvm/book3s_hv.c
3818
controlled_threads = threads_per_vcore(vc->kvm);
arch/powerpc/kvm/book3s_hv.c
4011
srcu_idx = srcu_read_lock(&vc->kvm->srcu);
arch/powerpc/kvm/book3s_hv.c
4021
srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv.c
4275
rc = plpar_guest_run_vcpu(0, vcpu->kvm->arch.lpid, vcpu->vcpu_id,
arch/powerpc/kvm/book3s_hv.c
4349
hvregs.lpid = vcpu->kvm->arch.lpid;
arch/powerpc/kvm/book3s_hv.c
4416
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
4448
__this_cpu_write(cpu_in_guest, kvm);
arch/powerpc/kvm/book3s_hv.c
4455
__this_cpu_write(cpu_in_guest, kvm);
arch/powerpc/kvm/book3s_hv.c
4507
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv.c
4721
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
4723
mutex_lock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_hv.c
4724
if (!kvm->arch.mmu_ready) {
arch/powerpc/kvm/book3s_hv.c
4725
if (!kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv.c
4729
kvmppc_setup_partition_table(kvm);
arch/powerpc/kvm/book3s_hv.c
4730
kvm->arch.mmu_ready = 1;
arch/powerpc/kvm/book3s_hv.c
4733
mutex_unlock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_hv.c
4785
if (!vcpu->kvm->arch.mmu_ready) {
arch/powerpc/kvm/book3s_hv.c
4875
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
4892
if (unlikely(!kvm->arch.mmu_ready)) {
arch/powerpc/kvm/book3s_hv.c
4909
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv.c
4919
if (need_resched() || !kvm->arch.mmu_ready)
arch/powerpc/kvm/book3s_hv.c
4984
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv.c
4995
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv.c
5089
struct kvm *kvm;
arch/powerpc/kvm/book3s_hv.c
5131
kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
5132
atomic_inc(&kvm->arch.vcpus_running);
arch/powerpc/kvm/book3s_hv.c
514
vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
arch/powerpc/kvm/book3s_hv.c
5155
vcpu->arch.pgdir = kvm->mm->pgd;
arch/powerpc/kvm/book3s_hv.c
518
static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
arch/powerpc/kvm/book3s_hv.c
5185
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv.c
5188
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv.c
5199
atomic_dec(&kvm->arch.vcpus_running);
arch/powerpc/kvm/book3s_hv.c
520
return kvm_get_vcpu_by_id(kvm, id);
arch/powerpc/kvm/book3s_hv.c
5228
static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,
arch/powerpc/kvm/book3s_hv.c
5261
static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
arch/powerpc/kvm/book3s_hv.c
5271
mutex_lock(&kvm->slots_lock);
arch/powerpc/kvm/book3s_hv.c
5277
slots = kvm_memslots(kvm);
arch/powerpc/kvm/book3s_hv.c
5291
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv.c
5292
r = kvmppc_hv_get_dirty_log_radix(kvm, memslot, buf);
arch/powerpc/kvm/book3s_hv.c
5294
r = kvmppc_hv_get_dirty_log_hpt(kvm, memslot, buf);
arch/powerpc/kvm/book3s_hv.c
5310
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_hv.c
5323
mutex_unlock(&kvm->slots_lock);
arch/powerpc/kvm/book3s_hv.c
5333
static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
arch/powerpc/kvm/book3s_hv.c
5354
static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
arch/powerpc/kvm/book3s_hv.c
5366
atomic64_inc(&kvm->arch.mmio_update);
arch/powerpc/kvm/book3s_hv.c
5381
if (change == KVM_MR_FLAGS_ONLY && kvm_is_radix(kvm) &&
arch/powerpc/kvm/book3s_hv.c
5383
kvmppc_radix_flush_memslot(kvm, old);
arch/powerpc/kvm/book3s_hv.c
5387
if (!kvm->arch.secure_guest)
arch/powerpc/kvm/book3s_hv.c
5396
kvmppc_uvmem_memslot_create(kvm, new);
arch/powerpc/kvm/book3s_hv.c
5399
kvmppc_uvmem_memslot_delete(kvm, old);
arch/powerpc/kvm/book3s_hv.c
5412
void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)
arch/powerpc/kvm/book3s_hv.c
5417
if ((kvm->arch.lpcr & mask) == lpcr)
arch/powerpc/kvm/book3s_hv.c
5420
kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;
arch/powerpc/kvm/book3s_hv.c
5423
struct kvmppc_vcore *vc = kvm->arch.vcores[i];
arch/powerpc/kvm/book3s_hv.c
5429
verify_lpcr(kvm, vc->lpcr);
arch/powerpc/kvm/book3s_hv.c
5431
if (++cores_done >= kvm->arch.online_vcores)
arch/powerpc/kvm/book3s_hv.c
5438
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_hv.c
5444
void kvmppc_setup_partition_table(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
5448
if (!kvm_is_radix(kvm)) {
arch/powerpc/kvm/book3s_hv.c
5450
dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) |
arch/powerpc/kvm/book3s_hv.c
5451
((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1);
arch/powerpc/kvm/book3s_hv.c
5453
dw0 |= kvm->arch.sdr1;
arch/powerpc/kvm/book3s_hv.c
5456
dw1 = kvm->arch.process_table;
arch/powerpc/kvm/book3s_hv.c
5459
__pa(kvm->arch.pgtable) | RADIX_PGD_INDEX_SIZE;
arch/powerpc/kvm/book3s_hv.c
5460
dw1 = PATB_GR | kvm->arch.process_table;
arch/powerpc/kvm/book3s_hv.c
5462
kvmhv_set_ptbl_entry(kvm->arch.lpid, dw0, dw1);
arch/powerpc/kvm/book3s_hv.c
5472
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
5481
if (!kvm->arch.hpt.virt) {
arch/powerpc/kvm/book3s_hv.c
5497
kvmppc_set_hpt(kvm, &info);
arch/powerpc/kvm/book3s_hv.c
5501
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv.c
5502
memslot = gfn_to_memslot(kvm, 0);
arch/powerpc/kvm/book3s_hv.c
5511
mmap_read_lock(kvm->mm);
arch/powerpc/kvm/book3s_hv.c
5512
vma = vma_lookup(kvm->mm, hva);
arch/powerpc/kvm/book3s_hv.c
5518
mmap_read_unlock(kvm->mm);
arch/powerpc/kvm/book3s_hv.c
5530
kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
arch/powerpc/kvm/book3s_hv.c
5539
kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
arch/powerpc/kvm/book3s_hv.c
5546
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv.c
5551
mmap_read_unlock(kvm->mm);
arch/powerpc/kvm/book3s_hv.c
5559
int kvmppc_switch_mmu_to_hpt(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
5563
if (nesting_enabled(kvm))
arch/powerpc/kvm/book3s_hv.c
5564
kvmhv_release_all_nested(kvm);
arch/powerpc/kvm/book3s_hv.c
5565
kvmppc_rmap_reset(kvm);
arch/powerpc/kvm/book3s_hv.c
5566
kvm->arch.process_table = 0;
arch/powerpc/kvm/book3s_hv.c
5568
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv.c
5569
kvm->arch.radix = 0;
arch/powerpc/kvm/book3s_hv.c
5570
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv.c
5571
kvmppc_free_radix(kvm);
arch/powerpc/kvm/book3s_hv.c
5577
kvmppc_update_lpcr(kvm, lpcr, lpcr_mask);
arch/powerpc/kvm/book3s_hv.c
5586
int kvmppc_switch_mmu_to_radix(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
5591
err = kvmppc_init_vm_radix(kvm);
arch/powerpc/kvm/book3s_hv.c
5594
kvmppc_rmap_reset(kvm);
arch/powerpc/kvm/book3s_hv.c
5596
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv.c
5597
kvm->arch.radix = 1;
arch/powerpc/kvm/book3s_hv.c
5598
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv.c
5599
kvmppc_free_hpt(&kvm->arch.hpt);
arch/powerpc/kvm/book3s_hv.c
5606
(kvm->arch.host_lpcr & LPCR_HAIL))
arch/powerpc/kvm/book3s_hv.c
5609
kvmppc_update_lpcr(kvm, lpcr, lpcr_mask);
arch/powerpc/kvm/book3s_hv.c
565
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
5697
static int kvmppc_core_init_vm_hv(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
5702
mutex_init(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv.c
5703
INIT_LIST_HEAD(&kvm->arch.uvmem_pfns);
arch/powerpc/kvm/book3s_hv.c
5704
mutex_init(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_hv.c
5712
kvm->arch.lpid = lpid;
arch/powerpc/kvm/book3s_hv.c
5717
kvmhv_vm_nested_init(kvm);
arch/powerpc/kvm/book3s_hv.c
573
tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
arch/powerpc/kvm/book3s_hv.c
5741
kvm->arch.lpid = guest_id;
arch/powerpc/kvm/book3s_hv.c
5753
cpumask_setall(&kvm->arch.need_tlb_flush);
arch/powerpc/kvm/book3s_hv.c
5756
memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
arch/powerpc/kvm/book3s_hv.c
5757
sizeof(kvm->arch.enabled_hcalls));
arch/powerpc/kvm/book3s_hv.c
5760
kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
arch/powerpc/kvm/book3s_hv.c
5764
kvm->arch.host_lpid = mfspr(SPRN_LPID);
arch/powerpc/kvm/book3s_hv.c
5765
kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
arch/powerpc/kvm/book3s_hv.c
5776
kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
arch/powerpc/kvm/book3s_hv.c
5804
kvm->arch.radix = 1;
arch/powerpc/kvm/book3s_hv.c
5805
kvm->arch.mmu_ready = 1;
arch/powerpc/kvm/book3s_hv.c
5810
(kvm->arch.host_lpcr & LPCR_HAIL))
arch/powerpc/kvm/book3s_hv.c
5812
ret = kvmppc_init_vm_radix(kvm);
arch/powerpc/kvm/book3s_hv.c
5815
plpar_guest_delete(0, kvm->arch.lpid);
arch/powerpc/kvm/book3s_hv.c
5817
kvmppc_free_lpid(kvm->arch.lpid);
arch/powerpc/kvm/book3s_hv.c
5820
kvmppc_setup_partition_table(kvm);
arch/powerpc/kvm/book3s_hv.c
5823
verify_lpcr(kvm, lpcr);
arch/powerpc/kvm/book3s_hv.c
5824
kvm->arch.lpcr = lpcr;
arch/powerpc/kvm/book3s_hv.c
5827
kvm->arch.resize_hpt = NULL;
arch/powerpc/kvm/book3s_hv.c
5837
kvm->arch.tlb_sets = 1;
arch/powerpc/kvm/book3s_hv.c
5839
kvm->arch.tlb_sets = POWER9_TLB_SETS_RADIX; /* 128 */
arch/powerpc/kvm/book3s_hv.c
5841
kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH; /* 256 */
arch/powerpc/kvm/book3s_hv.c
5843
kvm->arch.tlb_sets = POWER8_TLB_SETS; /* 512 */
arch/powerpc/kvm/book3s_hv.c
5845
kvm->arch.tlb_sets = POWER7_TLB_SETS; /* 128 */
arch/powerpc/kvm/book3s_hv.c
585
va = kvmppc_pin_guest_page(kvm, vpa, &nb);
arch/powerpc/kvm/book3s_hv.c
5862
kvm->arch.smt_mode = threads_per_subcore;
arch/powerpc/kvm/book3s_hv.c
5864
kvm->arch.smt_mode = 1;
arch/powerpc/kvm/book3s_hv.c
5865
kvm->arch.emul_smt_mode = 1;
arch/powerpc/kvm/book3s_hv.c
5870
static int kvmppc_arch_create_vm_debugfs_hv(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
5872
kvmppc_mmu_debugfs_init(kvm);
arch/powerpc/kvm/book3s_hv.c
5874
kvmhv_radix_debugfs_init(kvm);
arch/powerpc/kvm/book3s_hv.c
5878
static void kvmppc_free_vcores(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
5883
kfree(kvm->arch.vcores[i]);
arch/powerpc/kvm/book3s_hv.c
5884
kvm->arch.online_vcores = 0;
arch/powerpc/kvm/book3s_hv.c
5887
static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
5892
kvmppc_free_vcores(kvm);
arch/powerpc/kvm/book3s_hv.c
5895
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv.c
5896
kvmppc_free_radix(kvm);
arch/powerpc/kvm/book3s_hv.c
5898
kvmppc_free_hpt(&kvm->arch.hpt);
arch/powerpc/kvm/book3s_hv.c
5902
if (nesting_enabled(kvm))
arch/powerpc/kvm/book3s_hv.c
5903
kvmhv_release_all_nested(kvm);
arch/powerpc/kvm/book3s_hv.c
5904
kvm->arch.process_table = 0;
arch/powerpc/kvm/book3s_hv.c
5905
if (kvm->arch.secure_guest)
arch/powerpc/kvm/book3s_hv.c
5906
uv_svm_terminate(kvm->arch.lpid);
arch/powerpc/kvm/book3s_hv.c
5908
kvmhv_set_ptbl_entry(kvm->arch.lpid, 0, 0);
arch/powerpc/kvm/book3s_hv.c
5912
kvmhv_flush_lpid(kvm->arch.lpid);
arch/powerpc/kvm/book3s_hv.c
5913
plpar_guest_delete(0, kvm->arch.lpid);
arch/powerpc/kvm/book3s_hv.c
5915
kvmppc_free_lpid(kvm->arch.lpid);
arch/powerpc/kvm/book3s_hv.c
5918
kvmppc_free_pimap(kvm);
arch/powerpc/kvm/book3s_hv.c
592
kvmppc_unpin_guest_page(kvm, va, vpa, false);
arch/powerpc/kvm/book3s_hv.c
5955
void kvmppc_free_pimap(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
5957
kfree(kvm->arch.pimap);
arch/powerpc/kvm/book3s_hv.c
5965
static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
arch/powerpc/kvm/book3s_hv.c
5981
mutex_lock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
5983
pimap = kvm->arch.pimap;
arch/powerpc/kvm/book3s_hv.c
5988
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
5991
kvm->arch.pimap = pimap;
arch/powerpc/kvm/book3s_hv.c
6003
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
6015
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
6023
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
6050
rc = kvmppc_xive_set_mapped(kvm, guest_gsi, host_irq);
arch/powerpc/kvm/book3s_hv.c
6052
kvmppc_xics_set_mapped(kvm, guest_gsi, irq_map->r_hwirq);
arch/powerpc/kvm/book3s_hv.c
6056
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
6061
static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
arch/powerpc/kvm/book3s_hv.c
6074
mutex_lock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
6075
if (!kvm->arch.pimap)
arch/powerpc/kvm/book3s_hv.c
6078
pimap = kvm->arch.pimap;
arch/powerpc/kvm/book3s_hv.c
6086
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
6091
rc = kvmppc_xive_clr_mapped(kvm, guest_gsi, host_irq);
arch/powerpc/kvm/book3s_hv.c
6093
kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq);
arch/powerpc/kvm/book3s_hv.c
6103
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/book3s_hv.c
6116
ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
arch/powerpc/kvm/book3s_hv.c
6138
ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
arch/powerpc/kvm/book3s_hv.c
6148
struct kvm *kvm __maybe_unused = filp->private_data;
arch/powerpc/kvm/book3s_hv.c
6166
r = kvmppc_alloc_reset_hpt(kvm, htab_order);
arch/powerpc/kvm/book3s_hv.c
6179
r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);
arch/powerpc/kvm/book3s_hv.c
6190
r = kvm_vm_ioctl_resize_hpt_prepare(kvm, &rhpt);
arch/powerpc/kvm/book3s_hv.c
6201
r = kvm_vm_ioctl_resize_hpt_commit(kvm, &rhpt);
arch/powerpc/kvm/book3s_hv.c
6257
static int kvmhv_configure_mmu(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg)
arch/powerpc/kvm/book3s_hv.c
6288
mutex_lock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_hv.c
6289
if (radix != kvm_is_radix(kvm)) {
arch/powerpc/kvm/book3s_hv.c
6290
if (kvm->arch.mmu_ready) {
arch/powerpc/kvm/book3s_hv.c
6291
kvm->arch.mmu_ready = 0;
arch/powerpc/kvm/book3s_hv.c
6294
if (atomic_read(&kvm->arch.vcpus_running)) {
arch/powerpc/kvm/book3s_hv.c
6295
kvm->arch.mmu_ready = 1;
arch/powerpc/kvm/book3s_hv.c
6301
err = kvmppc_switch_mmu_to_radix(kvm);
arch/powerpc/kvm/book3s_hv.c
6303
err = kvmppc_switch_mmu_to_hpt(kvm);
arch/powerpc/kvm/book3s_hv.c
6308
kvm->arch.process_table = cfg->process_table;
arch/powerpc/kvm/book3s_hv.c
6309
kvmppc_setup_partition_table(kvm);
arch/powerpc/kvm/book3s_hv.c
6312
kvmppc_update_lpcr(kvm, lpcr, LPCR_GTSE);
arch/powerpc/kvm/book3s_hv.c
6316
mutex_unlock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_hv.c
6320
static int kvmhv_enable_nested(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
6332
if (kvm)
arch/powerpc/kvm/book3s_hv.c
6333
kvm->arch.nested_enable = true;
arch/powerpc/kvm/book3s_hv.c
6375
static void unpin_vpa_reset(struct kvm *kvm, struct kvmppc_vpa *vpa)
arch/powerpc/kvm/book3s_hv.c
6377
unpin_vpa(kvm, vpa);
arch/powerpc/kvm/book3s_hv.c
6390
static int kvmhv_enable_svm(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
6394
if (kvm)
arch/powerpc/kvm/book3s_hv.c
6395
kvm->arch.svm_enabled = 1;
arch/powerpc/kvm/book3s_hv.c
6407
static int kvmhv_svm_off(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
6415
if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
arch/powerpc/kvm/book3s_hv.c
6418
mutex_lock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_hv.c
6419
mmu_was_ready = kvm->arch.mmu_ready;
arch/powerpc/kvm/book3s_hv.c
6420
if (kvm->arch.mmu_ready) {
arch/powerpc/kvm/book3s_hv.c
6421
kvm->arch.mmu_ready = 0;
arch/powerpc/kvm/book3s_hv.c
6424
if (atomic_read(&kvm->arch.vcpus_running)) {
arch/powerpc/kvm/book3s_hv.c
6425
kvm->arch.mmu_ready = 1;
arch/powerpc/kvm/book3s_hv.c
6431
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv.c
6432
for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
arch/powerpc/kvm/book3s_hv.c
6434
struct kvm_memslots *slots = __kvm_memslots(kvm, i);
arch/powerpc/kvm/book3s_hv.c
6441
kvmppc_uvmem_drop_pages(memslot, kvm, true);
arch/powerpc/kvm/book3s_hv.c
6442
uv_unregister_mem_slot(kvm->arch.lpid, memslot->id);
arch/powerpc/kvm/book3s_hv.c
6445
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv.c
6447
ret = uv_svm_terminate(kvm->arch.lpid);
arch/powerpc/kvm/book3s_hv.c
6463
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_hv.c
6465
unpin_vpa_reset(kvm, &vcpu->arch.dtl);
arch/powerpc/kvm/book3s_hv.c
6466
unpin_vpa_reset(kvm, &vcpu->arch.slb_shadow);
arch/powerpc/kvm/book3s_hv.c
6467
unpin_vpa_reset(kvm, &vcpu->arch.vpa);
arch/powerpc/kvm/book3s_hv.c
6471
kvmppc_setup_partition_table(kvm);
arch/powerpc/kvm/book3s_hv.c
6472
kvm->arch.secure_guest = 0;
arch/powerpc/kvm/book3s_hv.c
6473
kvm->arch.mmu_ready = mmu_was_ready;
arch/powerpc/kvm/book3s_hv.c
6475
mutex_unlock(&kvm->arch.mmu_setup_lock);
arch/powerpc/kvm/book3s_hv.c
6479
static int kvmhv_enable_dawr1(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.c
6485
if (kvm)
arch/powerpc/kvm/book3s_hv.c
6486
kvm->arch.dawr1_enabled = true;
arch/powerpc/kvm/book3s_hv.c
681
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
700
va = kvmppc_pin_guest_page(kvm, gpa, &nb);
arch/powerpc/kvm/book3s_hv.c
706
kvmppc_unpin_guest_page(kvm, va, gpa, false);
arch/powerpc/kvm/book3s_hv.c
716
kvmppc_unpin_guest_page(kvm, va, gpa, false);
arch/powerpc/kvm/book3s_hv.c
730
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv.c
744
kvmppc_unpin_guest_page(kvm, old_vpa.pinned_addr, old_vpa.gpa,
arch/powerpc/kvm/book3s_hv.c
756
kvmppc_unpin_guest_page(kvm, old_vpa.pinned_addr, old_vpa.gpa,
arch/powerpc/kvm/book3s_hv.c
764
kvmppc_unpin_guest_page(kvm, old_vpa.pinned_addr, old_vpa.gpa,
arch/powerpc/kvm/book3s_hv.c
945
if (!vcpu->kvm->arch.dawr1_enabled)
arch/powerpc/kvm/book3s_hv.c
969
static int kvmppc_copy_guest(struct kvm *kvm, gpa_t to, gpa_t from,
arch/powerpc/kvm/book3s_hv.c
978
from_memslot = gfn_to_memslot(kvm, from >> PAGE_SHIFT);
arch/powerpc/kvm/book3s_hv.c
990
to_memslot = gfn_to_memslot(kvm, to >> PAGE_SHIFT);
arch/powerpc/kvm/book3s_hv.h
30
static inline bool nesting_enabled(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv.h
32
return kvm->arch.nested_enable && kvm_is_radix(kvm);
arch/powerpc/kvm/book3s_hv_builtin.c
352
pimap = kvmppc_get_passthru_irqmap(vcpu->kvm);
arch/powerpc/kvm/book3s_hv_builtin.c
605
static void flush_guest_tlb(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_builtin.c
610
for (set = 0; set < kvm->arch.tlb_sets; ++set) {
arch/powerpc/kvm/book3s_hv_builtin.c
620
void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu)
arch/powerpc/kvm/book3s_hv_builtin.c
622
if (cpumask_test_cpu(pcpu, &kvm->arch.need_tlb_flush)) {
arch/powerpc/kvm/book3s_hv_builtin.c
623
flush_guest_tlb(kvm);
arch/powerpc/kvm/book3s_hv_builtin.c
626
cpumask_clear_cpu(pcpu, &kvm->arch.need_tlb_flush);
arch/powerpc/kvm/book3s_hv_nested.c
1001
static void kvmhv_remove_nest_rmap(struct kvm *kvm, u64 n_rmap,
arch/powerpc/kvm/book3s_hv_nested.c
1011
gp = __find_nested(kvm, lpid);
arch/powerpc/kvm/book3s_hv_nested.c
1016
ptep = find_kvm_nested_guest_pte(kvm, lpid, gpa, &shift);
arch/powerpc/kvm/book3s_hv_nested.c
1019
kvmppc_unmap_pte(kvm, ptep, gpa, shift, NULL, gp->shadow_lpid);
arch/powerpc/kvm/book3s_hv_nested.c
1022
static void kvmhv_remove_nest_rmap_list(struct kvm *kvm, unsigned long *rmapp,
arch/powerpc/kvm/book3s_hv_nested.c
1030
kvmhv_remove_nest_rmap(kvm, rmap, hpa, mask);
arch/powerpc/kvm/book3s_hv_nested.c
1036
void kvmhv_remove_nest_rmap_range(struct kvm *kvm,
arch/powerpc/kvm/book3s_hv_nested.c
1054
kvmhv_remove_nest_rmap_list(kvm, rmap, hpa, addr_mask);
arch/powerpc/kvm/book3s_hv_nested.c
1077
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_nested.c
1082
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1083
ptep = find_kvm_nested_guest_pte(kvm, gp->l1_lpid, gpa, &shift);
arch/powerpc/kvm/book3s_hv_nested.c
1087
kvmppc_unmap_pte(kvm, ptep, gpa, shift, NULL, gp->shadow_lpid);
arch/powerpc/kvm/book3s_hv_nested.c
1090
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1135
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_nested.c
1150
gp = kvmhv_get_nested(kvm, lpid, false);
arch/powerpc/kvm/book3s_hv_nested.c
1171
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_nested.c
1177
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1178
kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable,
arch/powerpc/kvm/book3s_hv_nested.c
1181
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1201
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_nested.c
1205
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1206
idr_for_each_entry(&kvm->arch.kvm_nested_guest_idr, gp, lpid) {
arch/powerpc/kvm/book3s_hv_nested.c
1207
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1209
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1211
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1217
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_nested.c
1254
gp = kvmhv_get_nested(kvm, lpid, false);
arch/powerpc/kvm/book3s_hv_nested.c
1291
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_nested.c
1294
gp = kvmhv_get_nested(kvm, lpid, false);
arch/powerpc/kvm/book3s_hv_nested.c
1361
if (!__find_nested(vcpu->kvm, lpid))
arch/powerpc/kvm/book3s_hv_nested.c
1464
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_nested.c
1476
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1478
ret = kvmppc_hv_handle_set_rc(kvm, false, writing,
arch/powerpc/kvm/book3s_hv_nested.c
1479
gpte.raddr, kvm->arch.lpid);
arch/powerpc/kvm/book3s_hv_nested.c
1486
ret = kvmppc_hv_handle_set_rc(kvm, true, writing,
arch/powerpc/kvm/book3s_hv_nested.c
1494
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1526
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_nested.c
1596
memslot = gfn_to_memslot(kvm, gfn);
arch/powerpc/kvm/book3s_hv_nested.c
1622
mmu_seq = kvm->mmu_invalidate_seq;
arch/powerpc/kvm/book3s_hv_nested.c
1627
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1628
pte_p = find_kvm_secondary_pte(kvm, gpa, &shift);
arch/powerpc/kvm/book3s_hv_nested.c
1633
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1680
ret = kvmppc_create_pte(kvm, gp->shadow_pgtable, pte, n_gpa, level,
arch/powerpc/kvm/book3s_hv_nested.c
1704
int kvmhv_nested_next_lpid(struct kvm *kvm, int lpid)
arch/powerpc/kvm/book3s_hv_nested.c
1708
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
1709
if (!idr_get_next(&kvm->arch.kvm_nested_guest_idr, &ret))
arch/powerpc/kvm/book3s_hv_nested.c
1711
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
277
*lpcr = kvmppc_filter_lpcr_hv(vcpu->kvm,
arch/powerpc/kvm/book3s_hv_nested.c
306
if (vcpu->kvm->arch.l1_ptcr == 0)
arch/powerpc/kvm/book3s_hv_nested.c
350
l2 = kvmhv_get_nested(vcpu->kvm, l2_hv.lpid, true);
arch/powerpc/kvm/book3s_hv_nested.c
570
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_nested.c
575
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv_nested.c
578
!kvm_is_visible_gfn(vcpu->kvm, (ptcr & PRTB_MASK) >> PAGE_SHIFT))
arch/powerpc/kvm/book3s_hv_nested.c
580
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv_nested.c
582
kvm->arch.l1_ptcr = ptcr;
arch/powerpc/kvm/book3s_hv_nested.c
619
gp = kvmhv_get_nested(vcpu->kvm, l1_lpid, false);
arch/powerpc/kvm/book3s_hv_nested.c
675
struct kvm *kvm = gp->l1_host;
arch/powerpc/kvm/book3s_hv_nested.c
678
ptbl_addr = (kvm->arch.l1_ptcr & PRTB_MASK) + (gp->l1_lpid << 4);
arch/powerpc/kvm/book3s_hv_nested.c
679
if (gp->l1_lpid < (1ul << ((kvm->arch.l1_ptcr & PRTS_MASK) + 12 - 4))) {
arch/powerpc/kvm/book3s_hv_nested.c
680
int srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv_nested.c
681
ret = kvm_read_guest(kvm, ptbl_addr,
arch/powerpc/kvm/book3s_hv_nested.c
683
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv_nested.c
695
void kvmhv_vm_nested_init(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_nested.c
697
idr_init(&kvm->arch.kvm_nested_guest_idr);
arch/powerpc/kvm/book3s_hv_nested.c
700
static struct kvm_nested_guest *__find_nested(struct kvm *kvm, int lpid)
arch/powerpc/kvm/book3s_hv_nested.c
702
return idr_find(&kvm->arch.kvm_nested_guest_idr, lpid);
arch/powerpc/kvm/book3s_hv_nested.c
705
static bool __prealloc_nested(struct kvm *kvm, int lpid)
arch/powerpc/kvm/book3s_hv_nested.c
707
if (idr_alloc(&kvm->arch.kvm_nested_guest_idr,
arch/powerpc/kvm/book3s_hv_nested.c
713
static void __add_nested(struct kvm *kvm, int lpid, struct kvm_nested_guest *gp)
arch/powerpc/kvm/book3s_hv_nested.c
715
if (idr_replace(&kvm->arch.kvm_nested_guest_idr, gp, lpid))
arch/powerpc/kvm/book3s_hv_nested.c
719
static void __remove_nested(struct kvm *kvm, int lpid)
arch/powerpc/kvm/book3s_hv_nested.c
721
idr_remove(&kvm->arch.kvm_nested_guest_idr, lpid);
arch/powerpc/kvm/book3s_hv_nested.c
724
static struct kvm_nested_guest *kvmhv_alloc_nested(struct kvm *kvm, unsigned int lpid)
arch/powerpc/kvm/book3s_hv_nested.c
732
gp->l1_host = kvm;
arch/powerpc/kvm/book3s_hv_nested.c
735
gp->shadow_pgtable = pgd_alloc(kvm->mm);
arch/powerpc/kvm/book3s_hv_nested.c
749
pgd_free(kvm->mm, gp->shadow_pgtable);
arch/powerpc/kvm/book3s_hv_nested.c
760
struct kvm *kvm = gp->l1_host;
arch/powerpc/kvm/book3s_hv_nested.c
768
kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable,
arch/powerpc/kvm/book3s_hv_nested.c
770
pgd_free(kvm->mm, gp->shadow_pgtable);
arch/powerpc/kvm/book3s_hv_nested.c
779
struct kvm *kvm = gp->l1_host;
arch/powerpc/kvm/book3s_hv_nested.c
783
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
784
if (gp == __find_nested(kvm, lpid)) {
arch/powerpc/kvm/book3s_hv_nested.c
785
__remove_nested(kvm, lpid);
arch/powerpc/kvm/book3s_hv_nested.c
789
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
800
void kvmhv_release_all_nested(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_nested.c
808
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
809
idr_for_each_entry(&kvm->arch.kvm_nested_guest_idr, gp, lpid) {
arch/powerpc/kvm/book3s_hv_nested.c
810
__remove_nested(kvm, lpid);
arch/powerpc/kvm/book3s_hv_nested.c
816
idr_destroy(&kvm->arch.kvm_nested_guest_idr);
arch/powerpc/kvm/book3s_hv_nested.c
818
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
824
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv_nested.c
825
kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm))
arch/powerpc/kvm/book3s_hv_nested.c
827
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv_nested.c
833
struct kvm *kvm = gp->l1_host;
arch/powerpc/kvm/book3s_hv_nested.c
835
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
836
kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable, gp->shadow_lpid);
arch/powerpc/kvm/book3s_hv_nested.c
837
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
844
struct kvm_nested_guest *kvmhv_get_nested(struct kvm *kvm, int l1_lpid,
arch/powerpc/kvm/book3s_hv_nested.c
849
if (l1_lpid >= (1ul << ((kvm->arch.l1_ptcr & PRTS_MASK) + 12 - 4)))
arch/powerpc/kvm/book3s_hv_nested.c
852
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
853
gp = __find_nested(kvm, l1_lpid);
arch/powerpc/kvm/book3s_hv_nested.c
856
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
861
newgp = kvmhv_alloc_nested(kvm, l1_lpid);
arch/powerpc/kvm/book3s_hv_nested.c
865
if (!__prealloc_nested(kvm, l1_lpid)) {
arch/powerpc/kvm/book3s_hv_nested.c
870
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
871
gp = __find_nested(kvm, l1_lpid);
arch/powerpc/kvm/book3s_hv_nested.c
873
__add_nested(kvm, l1_lpid, newgp);
arch/powerpc/kvm/book3s_hv_nested.c
879
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
889
struct kvm *kvm = gp->l1_host;
arch/powerpc/kvm/book3s_hv_nested.c
892
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
894
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/book3s_hv_nested.c
899
pte_t *find_kvm_nested_guest_pte(struct kvm *kvm, unsigned long lpid,
arch/powerpc/kvm/book3s_hv_nested.c
905
gp = __find_nested(kvm, lpid);
arch/powerpc/kvm/book3s_hv_nested.c
909
VM_WARN(!spin_is_locked(&kvm->mmu_lock),
arch/powerpc/kvm/book3s_hv_nested.c
922
void kvmhv_insert_nest_rmap(struct kvm *kvm, unsigned long *rmapp,
arch/powerpc/kvm/book3s_hv_nested.c
954
static void kvmhv_update_nest_rmap_rc(struct kvm *kvm, u64 n_rmap,
arch/powerpc/kvm/book3s_hv_nested.c
966
ptep = find_kvm_nested_guest_pte(kvm, lpid, gpa, &shift);
arch/powerpc/kvm/book3s_hv_nested.c
975
kvmppc_radix_tlbie_page(kvm, gpa, shift, lpid);
arch/powerpc/kvm/book3s_hv_nested.c
983
void kvmhv_update_nest_rmap_rc_list(struct kvm *kvm, unsigned long *rmapp,
arch/powerpc/kvm/book3s_hv_nested.c
998
kvmhv_update_nest_rmap_rc(kvm, rmap, clr, set, hpa, mask);
arch/powerpc/kvm/book3s_hv_nestedv2.c
1041
rc = plpar_guest_create_vcpu(0, vcpu->kvm->arch.lpid, vcpu->vcpu_id);
arch/powerpc/kvm/book3s_hv_nestedv2.c
651
guest_id = vcpu->kvm->arch.lpid;
arch/powerpc/kvm/book3s_hv_p9_entry.c
301
static void switch_mmu_to_guest_radix(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr)
arch/powerpc/kvm/book3s_hv_p9_entry.c
307
lpid = nested ? nested->shadow_lpid : kvm->arch.lpid;
arch/powerpc/kvm/book3s_hv_p9_entry.c
326
static void switch_mmu_to_guest_hpt(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr)
arch/powerpc/kvm/book3s_hv_p9_entry.c
332
lpid = kvm->arch.lpid;
arch/powerpc/kvm/book3s_hv_p9_entry.c
353
static void switch_mmu_to_host(struct kvm *kvm, u32 pid)
arch/powerpc/kvm/book3s_hv_p9_entry.c
355
u32 lpid = kvm->arch.host_lpid;
arch/powerpc/kvm/book3s_hv_p9_entry.c
356
u64 lpcr = kvm->arch.host_lpcr;
arch/powerpc/kvm/book3s_hv_p9_entry.c
378
static void save_clear_host_mmu(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_p9_entry.c
391
static void save_clear_guest_mmu(struct kvm *kvm, struct kvm_vcpu *vcpu)
arch/powerpc/kvm/book3s_hv_p9_entry.c
393
if (kvm_is_radix(kvm)) {
arch/powerpc/kvm/book3s_hv_p9_entry.c
419
static void flush_guest_tlb(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_p9_entry.c
424
if (kvm_is_radix(kvm)) {
arch/powerpc/kvm/book3s_hv_p9_entry.c
429
for (set = 1; set < kvm->arch.tlb_sets; ++set) {
arch/powerpc/kvm/book3s_hv_p9_entry.c
440
for (set = 0; set < kvm->arch.tlb_sets; ++set) {
arch/powerpc/kvm/book3s_hv_p9_entry.c
453
static void check_need_tlb_flush(struct kvm *kvm, int pcpu,
arch/powerpc/kvm/book3s_hv_p9_entry.c
463
need_tlb_flush = &kvm->arch.need_tlb_flush;
arch/powerpc/kvm/book3s_hv_p9_entry.c
486
flush_guest_tlb(kvm);
arch/powerpc/kvm/book3s_hv_p9_entry.c
535
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_p9_entry.c
691
if (!radix_enabled() || !kvm_is_radix(kvm) ||
arch/powerpc/kvm/book3s_hv_p9_entry.c
695
save_clear_host_mmu(kvm);
arch/powerpc/kvm/book3s_hv_p9_entry.c
697
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv_p9_entry.c
698
switch_mmu_to_guest_radix(kvm, vcpu, lpcr);
arch/powerpc/kvm/book3s_hv_p9_entry.c
700
switch_mmu_to_guest_hpt(kvm, vcpu, lpcr);
arch/powerpc/kvm/book3s_hv_p9_entry.c
703
check_need_tlb_flush(kvm, vc->pcpu, nested);
arch/powerpc/kvm/book3s_hv_p9_entry.c
862
save_clear_guest_mmu(kvm, vcpu);
arch/powerpc/kvm/book3s_hv_p9_entry.c
863
switch_mmu_to_host(kvm, host_pidr);
arch/powerpc/kvm/book3s_hv_p9_perf.c
133
save_pmu |= nesting_enabled(vcpu->kvm);
arch/powerpc/kvm/book3s_hv_ras.c
103
tlbiel_all_lpid(vcpu->kvm->arch.radix);
arch/powerpc/kvm/book3s_hv_ras.c
117
if (vcpu->kvm->arch.fwnmi_enabled) {
arch/powerpc/kvm/book3s_hv_ras.c
86
tlbiel_all_lpid(vcpu->kvm->arch.radix);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1016
void kvmppc_invalidate_hpte(struct kvm *kvm, __be64 *hptep,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1030
do_tlbies(kvm, &rb, 1, 1, true);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1034
void kvmppc_clear_ref_hpte(struct kvm *kvm, __be64 *hptep,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1051
do_tlbies(kvm, &rb, 1, 1, false);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
109
static void kvmppc_set_dirty_from_hpte(struct kvm *kvm,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1097
long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1126
hash = (vsid ^ ((eaddr & somask) >> pshift)) & kvmppc_hpt_mask(&kvm->arch.hpt);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1137
hpte = (__be64 *)(kvm->arch.hpt.virt + (hash << 7));
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1173
hash = hash ^ kvmppc_hpt_mask(&kvm->arch.hpt);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
118
memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1193
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1207
mmio_update = atomic64_read(&kvm->arch.mmio_update);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1216
index = kvmppc_hv_find_lock_hpte(kvm, addr, slb_v, valid);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1222
hpte = (__be64 *)(kvm->arch.hpt.virt + (index << 4));
arch/powerpc/kvm/book3s_hv_rm_mmu.c
1229
rev = real_vmalloc_addr(&kvm->arch.hpt.rev[index]);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
124
static unsigned long *revmap_for_hpte(struct kvm *kvm, unsigned long hpte_v,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
134
memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
147
static void remove_revmap_chain(struct kvm *kvm, long pte_index,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
160
rmap = revmap_for_hpte(kvm, hpte_v, ptel, &memslot, &gfn);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
166
next = real_vmalloc_addr(&kvm->arch.hpt.rev[rev->forw]);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
167
prev = real_vmalloc_addr(&kvm->arch.hpt.rev[rev->back]);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
184
long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
202
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
222
mmu_seq = kvm->mmu_invalidate_seq;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
228
memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
248
arch_spin_lock(&kvm->mmu_lock.rlock.raw_lock);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
249
ptep = find_kvm_host_pte(kvm, mmu_seq, hva, &hpage_shift);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
263
arch_spin_unlock(&kvm->mmu_lock.rlock.raw_lock);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
277
arch_spin_unlock(&kvm->mmu_lock.rlock.raw_lock);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
303
if (pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
307
hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4));
arch/powerpc/kvm/book3s_hv_rm_mmu.c
32
static int global_invalidates(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_rm_mmu.c
338
hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4));
arch/powerpc/kvm/book3s_hv_rm_mmu.c
355
rev = &kvm->arch.hpt.rev[pte_index];
arch/powerpc/kvm/book3s_hv_rm_mmu.c
360
note_hpte_modification(kvm, rev);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
369
if (mmu_invalidate_retry(kvm, mmu_seq)) {
arch/powerpc/kvm/book3s_hv_rm_mmu.c
376
kvmppc_add_revmap_chain(kvm, rev, rmap, pte_index,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
404
return kvmppc_do_h_enter(vcpu->kvm, flags, pte_index, pteh, ptel,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
44
if (kvm->arch.online_vcores == 1 && local_paca->kvm_hstate.kvm_vcpu)
arch/powerpc/kvm/book3s_hv_rm_mmu.c
454
static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
469
"r" (rbvalues[i]), "r" (kvm->arch.lpid));
arch/powerpc/kvm/book3s_hv_rm_mmu.c
472
fixup_tlbie_lpid(rbvalues[i - 1], kvm->arch.lpid);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
485
long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
494
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
496
if (pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
498
hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4));
arch/powerpc/kvm/book3s_hv_rm_mmu.c
514
rev = real_vmalloc_addr(&kvm->arch.hpt.rev[pte_index]);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
519
do_tlbies(kvm, &rb, 1, global_invalidates(kvm), true);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
529
remove_revmap_chain(kvm, pte_index, rev, v,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
533
note_hpte_modification(kvm, rev);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
537
atomic64_inc(&kvm->arch.mmio_update);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
550
return kvmppc_do_h_remove(vcpu->kvm, flags, pte_index, avpn,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
557
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
56
cpumask_setall(&kvm->arch.need_tlb_flush);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
568
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
570
global = global_invalidates(kvm);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
58
cpumask_clear_cpu(cpu, &kvm->arch.need_tlb_flush);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
585
pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt)) {
arch/powerpc/kvm/book3s_hv_rm_mmu.c
591
hp = (__be64 *) (kvm->arch.hpt.virt + (pte_index << 4));
arch/powerpc/kvm/book3s_hv_rm_mmu.c
628
rev = real_vmalloc_addr(&kvm->arch.hpt.rev[pte_index]);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
629
note_hpte_modification(kvm, rev);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
637
atomic64_inc(&kvm->arch.mmio_update);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
654
do_tlbies(kvm, tlbrb, n, global, true);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
662
remove_revmap_chain(kvm, pte_index, rev,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
677
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
68
void kvmppc_add_revmap_chain(struct kvm *kvm, struct revmap_entry *rev,
arch/powerpc/kvm/book3s_hv_rm_mmu.c
683
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
685
if (pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
688
hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4));
arch/powerpc/kvm/book3s_hv_rm_mmu.c
708
rev = real_vmalloc_addr(&kvm->arch.hpt.rev[pte_index]);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
712
note_hpte_modification(kvm, rev);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
730
do_tlbies(kvm, &rb, 1, global_invalidates(kvm), true);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
739
atomic64_inc(&kvm->arch.mmio_update);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
748
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
754
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
756
if (pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
76
head = &kvm->arch.hpt.rev[i];
arch/powerpc/kvm/book3s_hv_rm_mmu.c
762
rev = real_vmalloc_addr(&kvm->arch.hpt.rev[pte_index]);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
764
hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4));
arch/powerpc/kvm/book3s_hv_rm_mmu.c
789
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
79
tail = &kvm->arch.hpt.rev[head->back];
arch/powerpc/kvm/book3s_hv_rm_mmu.c
796
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
798
if (pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
801
rev = real_vmalloc_addr(&kvm->arch.hpt.rev[pte_index]);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
802
hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4));
arch/powerpc/kvm/book3s_hv_rm_mmu.c
813
note_hpte_modification(kvm, rev);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
818
kvmppc_clear_ref_hpte(kvm, hpte, pte_index);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
819
rmap = revmap_for_hpte(kvm, v, gr, NULL, NULL);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
838
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
844
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
846
if (pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt))
arch/powerpc/kvm/book3s_hv_rm_mmu.c
849
rev = real_vmalloc_addr(&kvm->arch.hpt.rev[pte_index]);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
850
hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4));
arch/powerpc/kvm/book3s_hv_rm_mmu.c
861
note_hpte_modification(kvm, rev);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
866
kvmppc_invalidate_hpte(kvm, hpte, pte_index);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
872
kvmppc_set_dirty_from_hpte(kvm, v, gr);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
887
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
895
memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
903
ptep = find_kvm_host_pte(kvm, mmu_seq, hva, &shift);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
929
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
935
mmu_seq = kvm->mmu_invalidate_seq;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
938
arch_spin_lock(&kvm->mmu_lock.rlock.raw_lock);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
950
arch_spin_unlock(&kvm->mmu_lock.rlock.raw_lock);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
959
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
963
mmu_seq = kvm->mmu_invalidate_seq;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
966
arch_spin_lock(&kvm->mmu_lock.rlock.raw_lock);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
981
arch_spin_unlock(&kvm->mmu_lock.rlock.raw_lock);
arch/powerpc/kvm/book3s_hv_rm_mmu.c
988
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_hv_rm_mmu.c
993
if (kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv_rm_xics.c
315
icp = kvmppc_xics_find_server(xics->kvm, state->server);
arch/powerpc/kvm/book3s_hv_rm_xics.c
491
struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_hv_rm_xics.c
530
struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_hv_rm_xics.c
543
icp = kvmppc_xics_find_server(vcpu->kvm, server);
arch/powerpc/kvm/book3s_hv_rm_xics.c
616
struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_hv_rm_xics.c
677
struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_hv_rm_xics.c
709
if (!hlist_empty(&vcpu->kvm->irq_ack_notifier_list)) {
arch/powerpc/kvm/book3s_hv_rm_xics.c
736
struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_hv_rm_xics.c
859
xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_hv_uvmem.c
1004
pvt->kvm, pvt->gpa, vmf->page))
arch/powerpc/kvm/book3s_hv_uvmem.c
1031
kvmppc_gfn_remove(pvt->gpa >> PAGE_SHIFT, pvt->kvm);
arch/powerpc/kvm/book3s_hv_uvmem.c
1033
kvmppc_gfn_secure_mem_pfn(pvt->gpa >> PAGE_SHIFT, pvt->kvm);
arch/powerpc/kvm/book3s_hv_uvmem.c
1046
kvmppc_h_svm_page_out(struct kvm *kvm, unsigned long gpa,
arch/powerpc/kvm/book3s_hv_uvmem.c
1055
if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
arch/powerpc/kvm/book3s_hv_uvmem.c
1065
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv_uvmem.c
1066
mmap_read_lock(kvm->mm);
arch/powerpc/kvm/book3s_hv_uvmem.c
1067
start = gfn_to_hva(kvm, gfn);
arch/powerpc/kvm/book3s_hv_uvmem.c
1072
vma = find_vma_intersection(kvm->mm, start, end);
arch/powerpc/kvm/book3s_hv_uvmem.c
1076
if (!kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa, NULL))
arch/powerpc/kvm/book3s_hv_uvmem.c
1079
mmap_read_unlock(kvm->mm);
arch/powerpc/kvm/book3s_hv_uvmem.c
1080
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv_uvmem.c
1084
int kvmppc_send_page_to_uv(struct kvm *kvm, unsigned long gfn)
arch/powerpc/kvm/book3s_hv_uvmem.c
1089
page = gfn_to_page(kvm, gfn);
arch/powerpc/kvm/book3s_hv_uvmem.c
1093
mutex_lock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
1094
if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL))
arch/powerpc/kvm/book3s_hv_uvmem.c
1097
ret = uv_page_in(kvm->arch.lpid, page_to_pfn(page) << PAGE_SHIFT,
arch/powerpc/kvm/book3s_hv_uvmem.c
1101
mutex_unlock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
1105
int kvmppc_uvmem_memslot_create(struct kvm *kvm, const struct kvm_memory_slot *new)
arch/powerpc/kvm/book3s_hv_uvmem.c
1107
int ret = __kvmppc_uvmem_memslot_create(kvm, new);
arch/powerpc/kvm/book3s_hv_uvmem.c
1110
ret = kvmppc_uv_migrate_mem_slot(kvm, new);
arch/powerpc/kvm/book3s_hv_uvmem.c
1115
void kvmppc_uvmem_memslot_delete(struct kvm *kvm, const struct kvm_memory_slot *old)
arch/powerpc/kvm/book3s_hv_uvmem.c
1117
__kvmppc_uvmem_memslot_delete(kvm, old);
arch/powerpc/kvm/book3s_hv_uvmem.c
233
struct kvm *kvm;
arch/powerpc/kvm/book3s_hv_uvmem.c
248
int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot)
arch/powerpc/kvm/book3s_hv_uvmem.c
263
mutex_lock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
264
list_add(&p->list, &kvm->arch.uvmem_pfns);
arch/powerpc/kvm/book3s_hv_uvmem.c
265
mutex_unlock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
273
void kvmppc_uvmem_slot_free(struct kvm *kvm, const struct kvm_memory_slot *slot)
arch/powerpc/kvm/book3s_hv_uvmem.c
277
mutex_lock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
278
list_for_each_entry_safe(p, next, &kvm->arch.uvmem_pfns, list) {
arch/powerpc/kvm/book3s_hv_uvmem.c
286
mutex_unlock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
289
static void kvmppc_mark_gfn(unsigned long gfn, struct kvm *kvm,
arch/powerpc/kvm/book3s_hv_uvmem.c
294
list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) {
arch/powerpc/kvm/book3s_hv_uvmem.c
309
unsigned long uvmem_pfn, struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_uvmem.c
311
kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_UVMEM_PFN, uvmem_pfn);
arch/powerpc/kvm/book3s_hv_uvmem.c
315
static void kvmppc_gfn_secure_mem_pfn(unsigned long gfn, struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_uvmem.c
317
kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_MEM_PFN, 0);
arch/powerpc/kvm/book3s_hv_uvmem.c
321
static void kvmppc_gfn_shared(unsigned long gfn, struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_uvmem.c
323
kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_SHARED, 0);
arch/powerpc/kvm/book3s_hv_uvmem.c
327
static void kvmppc_gfn_remove(unsigned long gfn, struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_uvmem.c
329
kvmppc_mark_gfn(gfn, kvm, 0, 0);
arch/powerpc/kvm/book3s_hv_uvmem.c
333
static bool kvmppc_gfn_is_uvmem_pfn(unsigned long gfn, struct kvm *kvm,
arch/powerpc/kvm/book3s_hv_uvmem.c
338
list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) {
arch/powerpc/kvm/book3s_hv_uvmem.c
362
struct kvm *kvm, unsigned long *gfn)
arch/powerpc/kvm/book3s_hv_uvmem.c
368
list_for_each_entry(iter, &kvm->arch.uvmem_pfns, list)
arch/powerpc/kvm/book3s_hv_uvmem.c
391
static int kvmppc_memslot_page_merge(struct kvm *kvm,
arch/powerpc/kvm/book3s_hv_uvmem.c
395
unsigned long end, start = gfn_to_hva(kvm, gfn);
arch/powerpc/kvm/book3s_hv_uvmem.c
406
mmap_write_lock(kvm->mm);
arch/powerpc/kvm/book3s_hv_uvmem.c
408
vma = find_vma_intersection(kvm->mm, start, end);
arch/powerpc/kvm/book3s_hv_uvmem.c
426
mmap_write_unlock(kvm->mm);
arch/powerpc/kvm/book3s_hv_uvmem.c
430
static void __kvmppc_uvmem_memslot_delete(struct kvm *kvm,
arch/powerpc/kvm/book3s_hv_uvmem.c
433
uv_unregister_mem_slot(kvm->arch.lpid, memslot->id);
arch/powerpc/kvm/book3s_hv_uvmem.c
434
kvmppc_uvmem_slot_free(kvm, memslot);
arch/powerpc/kvm/book3s_hv_uvmem.c
435
kvmppc_memslot_page_merge(kvm, memslot, true);
arch/powerpc/kvm/book3s_hv_uvmem.c
438
static int __kvmppc_uvmem_memslot_create(struct kvm *kvm,
arch/powerpc/kvm/book3s_hv_uvmem.c
443
if (kvmppc_memslot_page_merge(kvm, memslot, false))
arch/powerpc/kvm/book3s_hv_uvmem.c
446
if (kvmppc_uvmem_slot_init(kvm, memslot))
arch/powerpc/kvm/book3s_hv_uvmem.c
449
ret = uv_register_mem_slot(kvm->arch.lpid,
arch/powerpc/kvm/book3s_hv_uvmem.c
459
kvmppc_uvmem_slot_free(kvm, memslot);
arch/powerpc/kvm/book3s_hv_uvmem.c
461
kvmppc_memslot_page_merge(kvm, memslot, true);
arch/powerpc/kvm/book3s_hv_uvmem.c
465
unsigned long kvmppc_h_svm_init_start(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_uvmem.c
472
kvm->arch.secure_guest = KVMPPC_SECURE_INIT_START;
arch/powerpc/kvm/book3s_hv_uvmem.c
478
if (!kvm_is_radix(kvm))
arch/powerpc/kvm/book3s_hv_uvmem.c
482
if (!kvm->arch.svm_enabled)
arch/powerpc/kvm/book3s_hv_uvmem.c
485
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv_uvmem.c
488
slots = kvm_memslots(kvm);
arch/powerpc/kvm/book3s_hv_uvmem.c
490
ret = __kvmppc_uvmem_memslot_create(kvm, memslot);
arch/powerpc/kvm/book3s_hv_uvmem.c
496
slots = kvm_memslots(kvm);
arch/powerpc/kvm/book3s_hv_uvmem.c
500
__kvmppc_uvmem_memslot_delete(kvm, memslot);
arch/powerpc/kvm/book3s_hv_uvmem.c
504
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv_uvmem.c
516
struct kvm *kvm, unsigned long gpa, struct page *fault_page)
arch/powerpc/kvm/book3s_hv_uvmem.c
536
if (!kvmppc_gfn_is_uvmem_pfn(gpa >> page_shift, kvm, NULL))
arch/powerpc/kvm/book3s_hv_uvmem.c
568
ret = uv_page_out(kvm->arch.lpid, pfn << page_shift,
arch/powerpc/kvm/book3s_hv_uvmem.c
589
struct kvm *kvm, unsigned long gpa,
arch/powerpc/kvm/book3s_hv_uvmem.c
594
mutex_lock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
595
ret = __kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa,
arch/powerpc/kvm/book3s_hv_uvmem.c
597
mutex_unlock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
611
struct kvm *kvm, bool skip_page_out)
arch/powerpc/kvm/book3s_hv_uvmem.c
620
mmap_read_lock(kvm->mm);
arch/powerpc/kvm/book3s_hv_uvmem.c
629
vma = vma_lookup(kvm->mm, addr);
arch/powerpc/kvm/book3s_hv_uvmem.c
636
mutex_lock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
638
if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
arch/powerpc/kvm/book3s_hv_uvmem.c
645
PAGE_SHIFT, kvm, pvt->gpa, NULL))
arch/powerpc/kvm/book3s_hv_uvmem.c
650
kvmppc_gfn_remove(gfn, kvm);
arch/powerpc/kvm/book3s_hv_uvmem.c
653
mutex_unlock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
656
mmap_read_unlock(kvm->mm);
arch/powerpc/kvm/book3s_hv_uvmem.c
659
unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_uvmem.c
668
if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
arch/powerpc/kvm/book3s_hv_uvmem.c
671
if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
arch/powerpc/kvm/book3s_hv_uvmem.c
674
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv_uvmem.c
676
kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm))
arch/powerpc/kvm/book3s_hv_uvmem.c
677
kvmppc_uvmem_drop_pages(memslot, kvm, false);
arch/powerpc/kvm/book3s_hv_uvmem.c
679
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv_uvmem.c
681
kvm->arch.secure_guest = 0;
arch/powerpc/kvm/book3s_hv_uvmem.c
682
uv_svm_terminate(kvm->arch.lpid);
arch/powerpc/kvm/book3s_hv_uvmem.c
695
static struct page *kvmppc_uvmem_get_page(unsigned long gpa, struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_uvmem.c
719
kvmppc_gfn_secure_uvmem_pfn(gpa >> PAGE_SHIFT, uvmem_pfn, kvm);
arch/powerpc/kvm/book3s_hv_uvmem.c
722
pvt->kvm = kvm;
arch/powerpc/kvm/book3s_hv_uvmem.c
742
unsigned long end, unsigned long gpa, struct kvm *kvm,
arch/powerpc/kvm/book3s_hv_uvmem.c
770
dpage = kvmppc_uvmem_get_page(gpa, kvm);
arch/powerpc/kvm/book3s_hv_uvmem.c
780
ret = uv_page_in(kvm->arch.lpid, pfn << page_shift,
arch/powerpc/kvm/book3s_hv_uvmem.c
794
static int kvmppc_uv_migrate_mem_slot(struct kvm *kvm,
arch/powerpc/kvm/book3s_hv_uvmem.c
802
mmap_read_lock(kvm->mm);
arch/powerpc/kvm/book3s_hv_uvmem.c
803
mutex_lock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
804
while (kvmppc_next_nontransitioned_gfn(memslot, kvm, &gfn)) {
arch/powerpc/kvm/book3s_hv_uvmem.c
806
start = gfn_to_hva(kvm, gfn);
arch/powerpc/kvm/book3s_hv_uvmem.c
811
vma = find_vma_intersection(kvm->mm, start, end);
arch/powerpc/kvm/book3s_hv_uvmem.c
816
(gfn << PAGE_SHIFT), kvm, PAGE_SHIFT, false);
arch/powerpc/kvm/book3s_hv_uvmem.c
825
mutex_unlock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
826
mmap_read_unlock(kvm->mm);
arch/powerpc/kvm/book3s_hv_uvmem.c
830
unsigned long kvmppc_h_svm_init_done(struct kvm *kvm)
arch/powerpc/kvm/book3s_hv_uvmem.c
837
if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
arch/powerpc/kvm/book3s_hv_uvmem.c
841
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv_uvmem.c
842
slots = kvm_memslots(kvm);
arch/powerpc/kvm/book3s_hv_uvmem.c
844
ret = kvmppc_uv_migrate_mem_slot(kvm, memslot);
arch/powerpc/kvm/book3s_hv_uvmem.c
860
kvm->arch.secure_guest |= KVMPPC_SECURE_INIT_DONE;
arch/powerpc/kvm/book3s_hv_uvmem.c
861
pr_info("LPID %lld went secure\n", kvm->arch.lpid);
arch/powerpc/kvm/book3s_hv_uvmem.c
864
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv_uvmem.c
877
static unsigned long kvmppc_share_page(struct kvm *kvm, unsigned long gpa,
arch/powerpc/kvm/book3s_hv_uvmem.c
888
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv_uvmem.c
889
mutex_lock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
890
if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
arch/powerpc/kvm/book3s_hv_uvmem.c
902
mutex_unlock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
903
page = gfn_to_page(kvm, gfn);
arch/powerpc/kvm/book3s_hv_uvmem.c
907
mutex_lock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
908
if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
arch/powerpc/kvm/book3s_hv_uvmem.c
917
if (!uv_page_in(kvm->arch.lpid, page_to_pfn(page) << page_shift, gpa, 0,
arch/powerpc/kvm/book3s_hv_uvmem.c
919
kvmppc_gfn_shared(gfn, kvm);
arch/powerpc/kvm/book3s_hv_uvmem.c
923
mutex_unlock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
925
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_hv_uvmem.c
935
unsigned long kvmppc_h_svm_page_in(struct kvm *kvm, unsigned long gpa,
arch/powerpc/kvm/book3s_hv_uvmem.c
945
if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
arch/powerpc/kvm/book3s_hv_uvmem.c
955
return kvmppc_share_page(kvm, gpa, page_shift);
arch/powerpc/kvm/book3s_hv_uvmem.c
958
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_hv_uvmem.c
959
mmap_read_lock(kvm->mm);
arch/powerpc/kvm/book3s_hv_uvmem.c
961
start = gfn_to_hva(kvm, gfn);
arch/powerpc/kvm/book3s_hv_uvmem.c
965
mutex_lock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
967
if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL))
arch/powerpc/kvm/book3s_hv_uvmem.c
971
vma = find_vma_intersection(kvm->mm, start, end);
arch/powerpc/kvm/book3s_hv_uvmem.c
975
if (kvmppc_svm_page_in(vma, start, end, gpa, kvm, page_shift,
arch/powerpc/kvm/book3s_hv_uvmem.c
982
mutex_unlock(&kvm->arch.uvmem_lock);
arch/powerpc/kvm/book3s_hv_uvmem.c
984
mmap_read_unlock(kvm->mm);
arch/powerpc/kvm/book3s_hv_uvmem.c
985
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_pr.c
1180
int idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/book3s_pr.c
1182
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/powerpc/kvm/book3s_pr.c
1230
int idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/book3s_pr.c
1232
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/powerpc/kvm/book3s_pr.c
1862
static int kvm_vm_ioctl_get_dirty_log_pr(struct kvm *kvm,
arch/powerpc/kvm/book3s_pr.c
1872
mutex_lock(&kvm->slots_lock);
arch/powerpc/kvm/book3s_pr.c
1874
r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot);
arch/powerpc/kvm/book3s_pr.c
1883
kvm_for_each_vcpu(n, vcpu, kvm)
arch/powerpc/kvm/book3s_pr.c
1892
mutex_unlock(&kvm->slots_lock);
arch/powerpc/kvm/book3s_pr.c
1896
static void kvmppc_core_flush_memslot_pr(struct kvm *kvm,
arch/powerpc/kvm/book3s_pr.c
1902
static int kvmppc_core_prepare_memory_region_pr(struct kvm *kvm,
arch/powerpc/kvm/book3s_pr.c
1910
static void kvmppc_core_commit_memory_region_pr(struct kvm *kvm,
arch/powerpc/kvm/book3s_pr.c
1924
static int kvm_vm_ioctl_get_smmu_info_pr(struct kvm *kvm,
arch/powerpc/kvm/book3s_pr.c
1951
vcpu = kvm_get_vcpu(kvm, 0);
arch/powerpc/kvm/book3s_pr.c
1970
static int kvm_configure_mmu_pr(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg)
arch/powerpc/kvm/book3s_pr.c
1981
static int kvm_vm_ioctl_get_smmu_info_pr(struct kvm *kvm,
arch/powerpc/kvm/book3s_pr.c
1993
static int kvmppc_core_init_vm_pr(struct kvm *kvm)
arch/powerpc/kvm/book3s_pr.c
1995
mutex_init(&kvm->arch.hpt_mutex);
arch/powerpc/kvm/book3s_pr.c
1999
kvmppc_pr_init_default_hcalls(kvm);
arch/powerpc/kvm/book3s_pr.c
2011
static void kvmppc_core_destroy_vm_pr(struct kvm *kvm)
arch/powerpc/kvm/book3s_pr.c
2014
WARN_ON(!list_empty(&kvm->arch.spapr_tce_tables));
arch/powerpc/kvm/book3s_pr.c
435
static bool do_kvm_unmap_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/book3s_pr.c
440
kvm_for_each_vcpu(i, vcpu, kvm)
arch/powerpc/kvm/book3s_pr.c
447
static bool kvm_unmap_gfn_range_pr(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/book3s_pr.c
449
return do_kvm_unmap_gfn(kvm, range);
arch/powerpc/kvm/book3s_pr.c
452
static bool kvm_age_gfn_pr(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/book3s_pr.c
458
static bool kvm_test_age_gfn_pr(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/book3s_pr.c
677
return kvm_is_visible_gfn(vcpu->kvm, gpa >> PAGE_SHIFT);
arch/powerpc/kvm/book3s_pr_papr.c
121
mutex_unlock(&vcpu->kvm->arch.hpt_mutex);
arch/powerpc/kvm/book3s_pr_papr.c
151
mutex_lock(&vcpu->kvm->arch.hpt_mutex);
arch/powerpc/kvm/book3s_pr_papr.c
208
mutex_unlock(&vcpu->kvm->arch.hpt_mutex);
arch/powerpc/kvm/book3s_pr_papr.c
224
mutex_lock(&vcpu->kvm->arch.hpt_mutex);
arch/powerpc/kvm/book3s_pr_papr.c
256
mutex_unlock(&vcpu->kvm->arch.hpt_mutex);
arch/powerpc/kvm/book3s_pr_papr.c
375
!test_bit(cmd/4, vcpu->kvm->arch.enabled_hcalls))
arch/powerpc/kvm/book3s_pr_papr.c
414
if (list_empty(&vcpu->kvm->arch.rtas_tokens))
arch/powerpc/kvm/book3s_pr_papr.c
416
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/book3s_pr_papr.c
418
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/powerpc/kvm/book3s_pr_papr.c
486
void kvmppc_pr_init_default_hcalls(struct kvm *kvm)
arch/powerpc/kvm/book3s_pr_papr.c
49
mutex_lock(&vcpu->kvm->arch.hpt_mutex);
arch/powerpc/kvm/book3s_pr_papr.c
494
__set_bit(hcall / 4, kvm->arch.enabled_hcalls);
arch/powerpc/kvm/book3s_pr_papr.c
80
mutex_unlock(&vcpu->kvm->arch.hpt_mutex);
arch/powerpc/kvm/book3s_pr_papr.c
96
mutex_lock(&vcpu->kvm->arch.hpt_mutex);
arch/powerpc/kvm/book3s_rtas.c
106
rc = kvmppc_xive_int_on(vcpu->kvm, irq);
arch/powerpc/kvm/book3s_rtas.c
108
rc = kvmppc_xics_int_on(vcpu->kvm, irq);
arch/powerpc/kvm/book3s_rtas.c
142
static int rtas_token_undefine(struct kvm *kvm, char *name)
arch/powerpc/kvm/book3s_rtas.c
146
lockdep_assert_held(&kvm->arch.rtas_token_lock);
arch/powerpc/kvm/book3s_rtas.c
148
list_for_each_entry_safe(d, tmp, &kvm->arch.rtas_tokens, list) {
arch/powerpc/kvm/book3s_rtas.c
160
static int rtas_token_define(struct kvm *kvm, char *name, u64 token)
arch/powerpc/kvm/book3s_rtas.c
167
lockdep_assert_held(&kvm->arch.rtas_token_lock);
arch/powerpc/kvm/book3s_rtas.c
169
list_for_each_entry(d, &kvm->arch.rtas_tokens, list) {
arch/powerpc/kvm/book3s_rtas.c
193
list_add_tail(&d->list, &kvm->arch.rtas_tokens);
arch/powerpc/kvm/book3s_rtas.c
198
int kvm_vm_ioctl_rtas_define_token(struct kvm *kvm, void __user *argp)
arch/powerpc/kvm/book3s_rtas.c
206
mutex_lock(&kvm->arch.rtas_token_lock);
arch/powerpc/kvm/book3s_rtas.c
209
rc = rtas_token_define(kvm, args.name, args.token);
arch/powerpc/kvm/book3s_rtas.c
211
rc = rtas_token_undefine(kvm, args.name);
arch/powerpc/kvm/book3s_rtas.c
213
mutex_unlock(&kvm->arch.rtas_token_lock);
arch/powerpc/kvm/book3s_rtas.c
233
rc = kvm_read_guest(vcpu->kvm, args_phys, &args, sizeof(args));
arch/powerpc/kvm/book3s_rtas.c
258
mutex_lock(&vcpu->kvm->arch.rtas_token_lock);
arch/powerpc/kvm/book3s_rtas.c
261
list_for_each_entry(d, &vcpu->kvm->arch.rtas_tokens, list) {
arch/powerpc/kvm/book3s_rtas.c
269
mutex_unlock(&vcpu->kvm->arch.rtas_token_lock);
arch/powerpc/kvm/book3s_rtas.c
273
rc = kvm_write_guest(vcpu->kvm, args_phys, &args, sizeof(args));
arch/powerpc/kvm/book3s_rtas.c
299
void kvmppc_rtas_tokens_free(struct kvm *kvm)
arch/powerpc/kvm/book3s_rtas.c
303
list_for_each_entry_safe(d, tmp, &kvm->arch.rtas_tokens, list) {
arch/powerpc/kvm/book3s_rtas.c
34
rc = kvmppc_xive_set_xive(vcpu->kvm, irq, server, priority);
arch/powerpc/kvm/book3s_rtas.c
36
rc = kvmppc_xics_set_xive(vcpu->kvm, irq, server, priority);
arch/powerpc/kvm/book3s_rtas.c
57
rc = kvmppc_xive_get_xive(vcpu->kvm, irq, &server, &priority);
arch/powerpc/kvm/book3s_rtas.c
59
rc = kvmppc_xics_get_xive(vcpu->kvm, irq, &server, &priority);
arch/powerpc/kvm/book3s_rtas.c
84
rc = kvmppc_xive_int_off(vcpu->kvm, irq);
arch/powerpc/kvm/book3s_rtas.c
86
rc = kvmppc_xics_int_off(vcpu->kvm, irq);
arch/powerpc/kvm/book3s_xics.c
1019
xics->dentry = debugfs_create_file("xics", 0444, xics->kvm->debugfs_dentry,
arch/powerpc/kvm/book3s_xics.c
1025
static struct kvmppc_ics *kvmppc_xics_create_ics(struct kvm *kvm,
arch/powerpc/kvm/book3s_xics.c
1033
mutex_lock(&kvm->lock);
arch/powerpc/kvm/book3s_xics.c
1058
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/book3s_xics.c
1066
if (!vcpu->kvm->arch.xics)
arch/powerpc/kvm/book3s_xics.c
1069
if (kvmppc_xics_find_server(vcpu->kvm, server_num))
arch/powerpc/kvm/book3s_xics.c
1104
struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
1242
ics = kvmppc_xics_create_ics(xics->kvm, xics, irq);
arch/powerpc/kvm/book3s_xics.c
1253
kvmppc_xics_find_server(xics->kvm, server) == NULL)
arch/powerpc/kvm/book3s_xics.c
1284
int kvmppc_xics_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
arch/powerpc/kvm/book3s_xics.c
1287
struct kvmppc_xics *xics = kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
1335
struct kvm *kvm = xics->kvm;
arch/powerpc/kvm/book3s_xics.c
1354
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xics.c
1369
if (kvm)
arch/powerpc/kvm/book3s_xics.c
1370
kvm->arch.xics = NULL;
arch/powerpc/kvm/book3s_xics.c
1385
static struct kvmppc_xics *kvmppc_xics_get_device(struct kvm *kvm)
arch/powerpc/kvm/book3s_xics.c
1387
struct kvmppc_xics **kvm_xics_device = &kvm->arch.xics_device;
arch/powerpc/kvm/book3s_xics.c
1403
struct kvm *kvm = dev->kvm;
arch/powerpc/kvm/book3s_xics.c
1408
if (kvm->arch.xics)
arch/powerpc/kvm/book3s_xics.c
1411
xics = kvmppc_xics_get_device(kvm);
arch/powerpc/kvm/book3s_xics.c
1417
xics->kvm = kvm;
arch/powerpc/kvm/book3s_xics.c
1418
kvm->arch.xics = xics;
arch/powerpc/kvm/book3s_xics.c
1457
if (xics->kvm != vcpu->kvm)
arch/powerpc/kvm/book3s_xics.c
1478
void kvmppc_xics_set_mapped(struct kvm *kvm, unsigned long irq,
arch/powerpc/kvm/book3s_xics.c
1481
struct kvmppc_xics *xics = kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
1494
void kvmppc_xics_clr_mapped(struct kvm *kvm, unsigned long irq,
arch/powerpc/kvm/book3s_xics.c
1497
struct kvmppc_xics *xics = kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
166
int kvmppc_xics_set_xive(struct kvm *kvm, u32 irq, u32 server, u32 priority)
arch/powerpc/kvm/book3s_xics.c
168
struct kvmppc_xics *xics = kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
182
icp = kvmppc_xics_find_server(kvm, server);
arch/powerpc/kvm/book3s_xics.c
196
int kvmppc_xics_get_xive(struct kvm *kvm, u32 irq, u32 *server, u32 *priority)
arch/powerpc/kvm/book3s_xics.c
198
struct kvmppc_xics *xics = kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
222
int kvmppc_xics_int_on(struct kvm *kvm, u32 irq)
arch/powerpc/kvm/book3s_xics.c
224
struct kvmppc_xics *xics = kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
238
icp = kvmppc_xics_find_server(kvm, state->server);
arch/powerpc/kvm/book3s_xics.c
249
int kvmppc_xics_int_off(struct kvm *kvm, u32 irq)
arch/powerpc/kvm/book3s_xics.c
251
struct kvmppc_xics *xics = kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
416
icp = kvmppc_xics_find_server(xics->kvm, state->server);
arch/powerpc/kvm/book3s_xics.c
623
struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
635
icp = kvmppc_xics_find_server(vcpu->kvm, server);
arch/powerpc/kvm/book3s_xics.c
711
icp = kvmppc_xics_find_server(vcpu->kvm, server);
arch/powerpc/kvm/book3s_xics.c
724
struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
779
struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
812
kvm_notify_acked_irq(vcpu->kvm, 0, irq);
arch/powerpc/kvm/book3s_xics.c
819
struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
850
struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
866
kvm_notify_acked_irq(vcpu->kvm, 0, icp->rm_eoied_irq);
arch/powerpc/kvm/book3s_xics.c
877
struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
arch/powerpc/kvm/book3s_xics.c
898
if (xics->real_mode && is_kvmppc_hv_enabled(vcpu->kvm))
arch/powerpc/kvm/book3s_xics.c
943
struct kvm *kvm = xics->kvm;
arch/powerpc/kvm/book3s_xics.c
951
if (!kvm)
arch/powerpc/kvm/book3s_xics.c
960
xics_debugfs_irqmap(m, kvm->arch.pimap);
arch/powerpc/kvm/book3s_xics.c
964
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xics.h
104
struct kvm *kvm;
arch/powerpc/kvm/book3s_xics.h
115
static inline struct kvmppc_icp *kvmppc_xics_find_server(struct kvm *kvm,
arch/powerpc/kvm/book3s_xics.h
121
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xive.c
1004
static void xive_inc_q_pending(struct kvm *kvm, u32 server, u8 prio)
arch/powerpc/kvm/book3s_xive.c
1011
vcpu = kvmppc_xive_find_server(kvm, server);
arch/powerpc/kvm/book3s_xive.c
1044
int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio)
arch/powerpc/kvm/book3s_xive.c
1051
vcpu = kvmppc_xive_find_server(kvm, *server);
arch/powerpc/kvm/book3s_xive.c
1067
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xive.c
1182
static int xive_target_interrupt(struct kvm *kvm,
arch/powerpc/kvm/book3s_xive.c
1186
struct kvmppc_xive *xive = kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
1195
rc = kvmppc_xive_select_target(kvm, &server, prio);
arch/powerpc/kvm/book3s_xive.c
1210
xive_inc_q_pending(kvm,
arch/powerpc/kvm/book3s_xive.c
1267
int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,
arch/powerpc/kvm/book3s_xive.c
1270
struct kvmppc_xive *xive = kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
1286
rc = xive_check_provisioning(xive->kvm,
arch/powerpc/kvm/book3s_xive.c
1349
rc = xive_target_interrupt(kvm, state, server, new_act_prio);
arch/powerpc/kvm/book3s_xive.c
1368
int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server,
arch/powerpc/kvm/book3s_xive.c
1371
struct kvmppc_xive *xive = kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
1391
int kvmppc_xive_int_on(struct kvm *kvm, u32 irq)
arch/powerpc/kvm/book3s_xive.c
1393
struct kvmppc_xive *xive = kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
1430
int kvmppc_xive_int_off(struct kvm *kvm, u32 irq)
arch/powerpc/kvm/book3s_xive.c
1432
struct kvmppc_xive *xive = kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
1494
struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
1550
int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq,
arch/powerpc/kvm/book3s_xive.c
1553
struct kvmppc_xive *xive = kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
1610
xive->ops->reset_mapped(kvm, guest_irq);
arch/powerpc/kvm/book3s_xive.c
1648
int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
arch/powerpc/kvm/book3s_xive.c
1651
struct kvmppc_xive *xive = kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
1701
xive->ops->reset_mapped(kvm, guest_irq);
arch/powerpc/kvm/book3s_xive.c
1731
struct kvm *kvm = vcpu->kvm;
arch/powerpc/kvm/book3s_xive.c
1732
struct kvmppc_xive *xive = kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
1805
struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
1870
return kvmppc_pack_vcpu_id(xive->kvm, cpu) < xive->nr_servers;
arch/powerpc/kvm/book3s_xive.c
1891
if (kvmppc_xive_vp_in_use(xive->kvm, vp_id)) {
arch/powerpc/kvm/book3s_xive.c
1915
if (xive->kvm != vcpu->kvm)
arch/powerpc/kvm/book3s_xive.c
2141
kvm_for_each_vcpu(i, vcpu, xive->kvm) {
arch/powerpc/kvm/book3s_xive.c
2305
struct kvm *kvm = xive->kvm;
arch/powerpc/kvm/book3s_xive.c
2309
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xive.c
2410
rc = xive_check_provisioning(xive->kvm, act_prio);
arch/powerpc/kvm/book3s_xive.c
2415
rc = xive_target_interrupt(xive->kvm, state,
arch/powerpc/kvm/book3s_xive.c
2487
int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
arch/powerpc/kvm/book3s_xive.c
2490
struct kvmppc_xive *xive = kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
2642
struct kvm *kvm = xive->kvm;
arch/powerpc/kvm/book3s_xive.c
2662
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xive.c
2682
kvm->arch.xive = NULL;
arch/powerpc/kvm/book3s_xive.c
2714
struct kvmppc_xive *kvmppc_xive_get_device(struct kvm *kvm, u32 type)
arch/powerpc/kvm/book3s_xive.c
2717
&kvm->arch.xive_devices.native :
arch/powerpc/kvm/book3s_xive.c
2718
&kvm->arch.xive_devices.xics_on_xive;
arch/powerpc/kvm/book3s_xive.c
2737
struct kvm *kvm = dev->kvm;
arch/powerpc/kvm/book3s_xive.c
2742
if (kvm->arch.xive)
arch/powerpc/kvm/book3s_xive.c
2745
xive = kvmppc_xive_get_device(kvm, type);
arch/powerpc/kvm/book3s_xive.c
2751
xive->kvm = kvm;
arch/powerpc/kvm/book3s_xive.c
2774
kvm->arch.xive = xive;
arch/powerpc/kvm/book3s_xive.c
2882
struct kvm *kvm = xive->kvm;
arch/powerpc/kvm/book3s_xive.c
2896
if (!kvm)
arch/powerpc/kvm/book3s_xive.c
2901
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xive.c
2954
xive->dentry = debugfs_create_file("xive", S_IRUGO, xive->kvm->debugfs_dentry,
arch/powerpc/kvm/book3s_xive.c
348
vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
arch/powerpc/kvm/book3s_xive.c
459
struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
517
struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
630
vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
arch/powerpc/kvm/book3s_xive.c
888
vcpu->kvm->arch.lpid, xc->server_num);
arch/powerpc/kvm/book3s_xive.c
891
vcpu->kvm->arch.lpid, xc->server_num, prio);
arch/powerpc/kvm/book3s_xive.c
971
static int xive_check_provisioning(struct kvm *kvm, u8 prio)
arch/powerpc/kvm/book3s_xive.c
973
struct kvmppc_xive *xive = kvm->arch.xive;
arch/powerpc/kvm/book3s_xive.c
987
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xive.h
104
struct kvm *kvm;
arch/powerpc/kvm/book3s_xive.h
199
static inline struct kvm_vcpu *kvmppc_xive_find_server(struct kvm *kvm, u32 nr)
arch/powerpc/kvm/book3s_xive.h
204
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xive.h
237
return xive->vp_base + kvmppc_pack_vcpu_id(xive->kvm, server);
arch/powerpc/kvm/book3s_xive.h
240
static inline bool kvmppc_xive_vp_in_use(struct kvm *kvm, u32 vp_id)
arch/powerpc/kvm/book3s_xive.h
245
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xive.h
298
int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio);
arch/powerpc/kvm/book3s_xive.h
301
struct kvmppc_xive *kvmppc_xive_get_device(struct kvm *kvm, u32 type);
arch/powerpc/kvm/book3s_xive.h
97
int (*reset_mapped)(struct kvm *kvm, unsigned long guest_irq);
arch/powerpc/kvm/book3s_xive_native.c
1017
struct kvm *kvm = xive->kvm;
arch/powerpc/kvm/book3s_xive_native.c
1046
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xive_native.c
1066
kvm->arch.xive = NULL;
arch/powerpc/kvm/book3s_xive_native.c
1094
struct kvm *kvm = dev->kvm;
arch/powerpc/kvm/book3s_xive_native.c
1098
if (kvm->arch.xive)
arch/powerpc/kvm/book3s_xive_native.c
1101
xive = kvmppc_xive_get_device(kvm, type);
arch/powerpc/kvm/book3s_xive_native.c
1107
xive->kvm = kvm;
arch/powerpc/kvm/book3s_xive_native.c
1126
kvm->arch.xive = xive;
arch/powerpc/kvm/book3s_xive_native.c
1177
struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
arch/powerpc/kvm/book3s_xive_native.c
1214
struct kvm *kvm = xive->kvm;
arch/powerpc/kvm/book3s_xive_native.c
1218
if (!kvm)
arch/powerpc/kvm/book3s_xive_native.c
1223
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xive_native.c
1261
xive->dentry = debugfs_create_file("xive", 0444, xive->kvm->debugfs_dentry,
arch/powerpc/kvm/book3s_xive_native.c
137
if (xive->kvm != vcpu->kvm)
arch/powerpc/kvm/book3s_xive_native.c
201
static int kvmppc_xive_native_reset_mapped(struct kvm *kvm, unsigned long irq)
arch/powerpc/kvm/book3s_xive_native.c
203
struct kvmppc_xive *xive = kvm->arch.xive;
arch/powerpc/kvm/book3s_xive_native.c
422
struct kvm *kvm = xive->kvm;
arch/powerpc/kvm/book3s_xive_native.c
439
rc = kvmppc_xive_select_target(kvm, &server, priority);
arch/powerpc/kvm/book3s_xive_native.c
562
struct kvm *kvm = xive->kvm;
arch/powerpc/kvm/book3s_xive_native.c
588
vcpu = kvmppc_xive_find_server(kvm, server);
arch/powerpc/kvm/book3s_xive_native.c
646
srcu_idx = srcu_read_lock(&kvm->srcu);
arch/powerpc/kvm/book3s_xive_native.c
651
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_xive_native.c
656
page = gfn_to_page(kvm, gfn);
arch/powerpc/kvm/book3s_xive_native.c
658
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_xive_native.c
664
srcu_read_unlock(&kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_xive_native.c
710
struct kvm *kvm = xive->kvm;
arch/powerpc/kvm/book3s_xive_native.c
733
vcpu = kvmppc_xive_find_server(kvm, server);
arch/powerpc/kvm/book3s_xive_native.c
807
struct kvm *kvm = xive->kvm;
arch/powerpc/kvm/book3s_xive_native.c
815
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/book3s_xive_native.c
907
srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/book3s_xive_native.c
908
mark_page_dirty(vcpu->kvm, gpa_to_gfn(q->guest_qaddr));
arch/powerpc/kvm/book3s_xive_native.c
909
srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
arch/powerpc/kvm/book3s_xive_native.c
916
struct kvm *kvm = xive->kvm;
arch/powerpc/kvm/book3s_xive_native.c
933
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/powerpc/kvm/booke.c
1313
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/booke.c
1318
if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
arch/powerpc/kvm/booke.c
1337
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/powerpc/kvm/booke.c
1361
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/booke.c
1366
if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
arch/powerpc/kvm/booke.c
1379
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/powerpc/kvm/booke.c
1641
ret = vcpu->kvm->arch.kvm_ops->get_sregs(vcpu, sregs);
arch/powerpc/kvm/booke.c
1664
ret = vcpu->kvm->arch.kvm_ops->set_sregs(vcpu, sregs);
arch/powerpc/kvm/booke.c
1720
r = vcpu->kvm->arch.kvm_ops->get_one_reg(vcpu, id, val);
arch/powerpc/kvm/booke.c
1789
r = vcpu->kvm->arch.kvm_ops->set_one_reg(vcpu, id, val);
arch/powerpc/kvm/booke.c
1817
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
arch/powerpc/kvm/booke.c
1822
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
arch/powerpc/kvm/booke.c
1827
void kvmppc_core_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
arch/powerpc/kvm/booke.c
1831
int kvmppc_core_prepare_memory_region(struct kvm *kvm,
arch/powerpc/kvm/booke.c
1839
void kvmppc_core_commit_memory_region(struct kvm *kvm,
arch/powerpc/kvm/booke.c
1846
void kvmppc_core_flush_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot)
arch/powerpc/kvm/booke.c
2124
int kvmppc_core_init_vm(struct kvm *kvm)
arch/powerpc/kvm/booke.c
2126
return kvm->arch.kvm_ops->init_vm(kvm);
arch/powerpc/kvm/booke.c
2134
r = vcpu->kvm->arch.kvm_ops->vcpu_create(vcpu);
arch/powerpc/kvm/booke.c
2160
vcpu->kvm->arch.kvm_ops->vcpu_free(vcpu);
arch/powerpc/kvm/booke.c
2167
vcpu->kvm->arch.kvm_ops->vcpu_free(vcpu);
arch/powerpc/kvm/booke.c
2170
void kvmppc_core_destroy_vm(struct kvm *kvm)
arch/powerpc/kvm/booke.c
2172
kvm->arch.kvm_ops->destroy_vm(kvm);
arch/powerpc/kvm/booke.c
2177
vcpu->kvm->arch.kvm_ops->vcpu_load(vcpu, cpu);
arch/powerpc/kvm/booke.c
2182
vcpu->kvm->arch.kvm_ops->vcpu_put(vcpu);
arch/powerpc/kvm/booke.c
805
vcpu->arch.pgdir = vcpu->kvm->mm->pgd;
arch/powerpc/kvm/e500.c
475
static int kvmppc_core_init_vm_e500(struct kvm *kvm)
arch/powerpc/kvm/e500.c
480
static void kvmppc_core_destroy_vm_e500(struct kvm *kvm)
arch/powerpc/kvm/e500.h
266
if (!gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT))
arch/powerpc/kvm/e500.h
305
return get_thread_specific_lpid(vcpu->kvm->arch.lpid);
arch/powerpc/kvm/e500_emulate.c
74
kvm_for_each_vcpu(i, cvcpu, vcpu->kvm) {
arch/powerpc/kvm/e500_mmu.c
431
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/e500_mmu.c
447
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/powerpc/kvm/e500_mmu_host.c
134
__write_host_tlbe(stlbe, mas0, vcpu_e500->vcpu.kvm->arch.lpid);
arch/powerpc/kvm/e500_mmu_host.c
139
vcpu_e500->vcpu.kvm->arch.lpid);
arch/powerpc/kvm/e500_mmu_host.c
333
struct kvm *kvm = vcpu_e500->vcpu.kvm;
arch/powerpc/kvm/e500_mmu_host.c
341
mmu_seq = kvm->mmu_invalidate_seq;
arch/powerpc/kvm/e500_mmu_host.c
352
slot = gfn_to_memslot(vcpu_e500->vcpu.kvm, gfn);
arch/powerpc/kvm/e500_mmu_host.c
363
spin_lock(&kvm->mmu_lock);
arch/powerpc/kvm/e500_mmu_host.c
364
if (mmu_invalidate_retry(kvm, mmu_seq)) {
arch/powerpc/kvm/e500_mmu_host.c
464
kvm_release_faultin_page(kvm, page, !!ret, writable);
arch/powerpc/kvm/e500_mmu_host.c
465
spin_unlock(&kvm->mmu_lock);
arch/powerpc/kvm/e500_mmu_host.c
683
static bool kvm_e500_mmu_unmap_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/e500_mmu_host.c
692
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/e500_mmu_host.c
694
return kvm_e500_mmu_unmap_gfn(kvm, range);
arch/powerpc/kvm/e500_mmu_host.c
697
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/e500_mmu_host.c
703
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/powerpc/kvm/e500mc.c
345
static int kvmppc_core_init_vm_e500mc(struct kvm *kvm)
arch/powerpc/kvm/e500mc.c
361
kvm->arch.lpid = lpid;
arch/powerpc/kvm/e500mc.c
365
static void kvmppc_core_destroy_vm_e500mc(struct kvm *kvm)
arch/powerpc/kvm/e500mc.c
367
int lpid = kvm->arch.lpid;
arch/powerpc/kvm/emulate.c
115
emulated = vcpu->kvm->arch.kvm_ops->emulate_mtspr(vcpu, sprn,
arch/powerpc/kvm/emulate.c
176
emulated = vcpu->kvm->arch.kvm_ops->emulate_mfspr(vcpu, sprn,
arch/powerpc/kvm/emulate.c
290
emulated = vcpu->kvm->arch.kvm_ops->emulate_op(vcpu, inst,
arch/powerpc/kvm/emulate_loadstore.c
246
if (vcpu->kvm->arch.kvm_ops->giveup_ext)
arch/powerpc/kvm/emulate_loadstore.c
247
vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu,
arch/powerpc/kvm/emulate_loadstore.c
270
if (vcpu->kvm->arch.kvm_ops->giveup_ext)
arch/powerpc/kvm/emulate_loadstore.c
271
vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu,
arch/powerpc/kvm/emulate_loadstore.c
315
if (vcpu->kvm->arch.kvm_ops->giveup_ext)
arch/powerpc/kvm/emulate_loadstore.c
316
vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu,
arch/powerpc/kvm/mpic.c
1106
kvm_notify_acked_irq(opp->kvm, 0, notify_eoi);
arch/powerpc/kvm/mpic.c
1449
kvm_io_bus_register_dev(opp->kvm, KVM_MMIO_BUS,
arch/powerpc/kvm/mpic.c
1456
kvm_io_bus_unregister_dev(opp->kvm, KVM_MMIO_BUS, &opp->mmio);
arch/powerpc/kvm/mpic.c
1475
mutex_lock(&opp->kvm->slots_lock);
arch/powerpc/kvm/mpic.c
1489
mutex_unlock(&opp->kvm->slots_lock);
arch/powerpc/kvm/mpic.c
1567
mutex_lock(&opp->kvm->slots_lock);
arch/powerpc/kvm/mpic.c
1569
mutex_unlock(&opp->kvm->slots_lock);
arch/powerpc/kvm/mpic.c
1635
dev->kvm->arch.mpic = NULL;
arch/powerpc/kvm/mpic.c
1649
kvm_set_irq_routing(opp->kvm, routing, 0, 0);
arch/powerpc/kvm/mpic.c
1661
if (dev->kvm->arch.mpic)
arch/powerpc/kvm/mpic.c
1669
opp->kvm = dev->kvm;
arch/powerpc/kvm/mpic.c
1714
dev->kvm->arch.mpic = opp;
arch/powerpc/kvm/mpic.c
1740
if (opp->kvm != vcpu->kvm)
arch/powerpc/kvm/mpic.c
1791
struct kvm *kvm, int irq_source_id, int level,
arch/powerpc/kvm/mpic.c
1795
struct openpic *opp = kvm->arch.mpic;
arch/powerpc/kvm/mpic.c
1807
struct kvm *kvm, int irq_source_id, int level, bool line_status)
arch/powerpc/kvm/mpic.c
1809
struct openpic *opp = kvm->arch.mpic;
arch/powerpc/kvm/mpic.c
1818
openpic_msi_write(kvm->arch.mpic, MSIIR_OFFSET, e->msi.data);
arch/powerpc/kvm/mpic.c
1825
int kvm_set_routing_entry(struct kvm *kvm,
arch/powerpc/kvm/mpic.c
191
struct kvm *kvm;
arch/powerpc/kvm/powerpc.c
1176
if (vcpu->kvm->arch.kvm_ops->giveup_ext)
arch/powerpc/kvm/powerpc.c
1177
vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_FP);
arch/powerpc/kvm/powerpc.c
1192
if (vcpu->kvm->arch.kvm_ops->giveup_ext)
arch/powerpc/kvm/powerpc.c
1193
vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VSX);
arch/powerpc/kvm/powerpc.c
1209
if (vcpu->kvm->arch.kvm_ops->giveup_ext)
arch/powerpc/kvm/powerpc.c
1210
vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VEC);
arch/powerpc/kvm/powerpc.c
1265
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/powerpc.c
1270
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/powerpc/kvm/powerpc.c
1368
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/powerpc/kvm/powerpc.c
1373
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/powerpc/kvm/powerpc.c
1983
if (!is_kvmppc_hv_enabled(vcpu->kvm))
arch/powerpc/kvm/powerpc.c
1986
vcpu->kvm->arch.fwnmi_enabled = true;
arch/powerpc/kvm/powerpc.c
2000
bool kvm_arch_intc_initialized(struct kvm *kvm)
arch/powerpc/kvm/powerpc.c
2003
if (kvm->arch.mpic)
arch/powerpc/kvm/powerpc.c
2007
if (kvm->arch.xics || kvm->arch.xive)
arch/powerpc/kvm/powerpc.c
2134
bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
arch/powerpc/kvm/powerpc.c
2139
ret = ret || (kvm->arch.mpic != NULL);
arch/powerpc/kvm/powerpc.c
2142
ret = ret || (kvm->arch.xics != NULL);
arch/powerpc/kvm/powerpc.c
2143
ret = ret || (kvm->arch.xive != NULL);
arch/powerpc/kvm/powerpc.c
2149
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
arch/powerpc/kvm/powerpc.c
2152
if (!kvm_arch_irqchip_in_kernel(kvm))
arch/powerpc/kvm/powerpc.c
2155
irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
arch/powerpc/kvm/powerpc.c
2162
int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
arch/powerpc/kvm/powerpc.c
2179
if (!kvmppc_book3s_hcall_implemented(kvm, hcall))
arch/powerpc/kvm/powerpc.c
2182
set_bit(hcall / 4, kvm->arch.enabled_hcalls);
arch/powerpc/kvm/powerpc.c
2184
clear_bit(hcall / 4, kvm->arch.enabled_hcalls);
arch/powerpc/kvm/powerpc.c
2193
if (kvm->arch.kvm_ops->set_smt_mode)
arch/powerpc/kvm/powerpc.c
2194
r = kvm->arch.kvm_ops->set_smt_mode(kvm, mode, flags);
arch/powerpc/kvm/powerpc.c
2200
if (!is_kvmppc_hv_enabled(kvm) ||
arch/powerpc/kvm/powerpc.c
2201
!kvm->arch.kvm_ops->enable_nested)
arch/powerpc/kvm/powerpc.c
2203
r = kvm->arch.kvm_ops->enable_nested(kvm);
arch/powerpc/kvm/powerpc.c
2209
if (!is_kvmppc_hv_enabled(kvm) || !kvm->arch.kvm_ops->enable_svm)
arch/powerpc/kvm/powerpc.c
2211
r = kvm->arch.kvm_ops->enable_svm(kvm);
arch/powerpc/kvm/powerpc.c
2215
if (!is_kvmppc_hv_enabled(kvm) || !kvm->arch.kvm_ops->enable_dawr1)
arch/powerpc/kvm/powerpc.c
2217
r = kvm->arch.kvm_ops->enable_dawr1(kvm);
arch/powerpc/kvm/powerpc.c
2358
struct kvm *kvm __maybe_unused = filp->private_data;
arch/powerpc/kvm/powerpc.c
2385
r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64);
arch/powerpc/kvm/powerpc.c
2402
r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64);
arch/powerpc/kvm/powerpc.c
2409
struct kvm *kvm = filp->private_data;
arch/powerpc/kvm/powerpc.c
2412
r = kvm->arch.kvm_ops->get_smmu_info(kvm, &info);
arch/powerpc/kvm/powerpc.c
2418
struct kvm *kvm = filp->private_data;
arch/powerpc/kvm/powerpc.c
2420
r = kvm_vm_ioctl_rtas_define_token(kvm, argp);
arch/powerpc/kvm/powerpc.c
2424
struct kvm *kvm = filp->private_data;
arch/powerpc/kvm/powerpc.c
2428
if (!kvm->arch.kvm_ops->configure_mmu)
arch/powerpc/kvm/powerpc.c
2433
r = kvm->arch.kvm_ops->configure_mmu(kvm, &cfg);
arch/powerpc/kvm/powerpc.c
2437
struct kvm *kvm = filp->private_data;
arch/powerpc/kvm/powerpc.c
2441
if (!kvm->arch.kvm_ops->get_rmmu_info)
arch/powerpc/kvm/powerpc.c
2443
r = kvm->arch.kvm_ops->get_rmmu_info(kvm, &info);
arch/powerpc/kvm/powerpc.c
2457
struct kvm *kvm = filp->private_data;
arch/powerpc/kvm/powerpc.c
2460
if (!kvm->arch.kvm_ops->svm_off)
arch/powerpc/kvm/powerpc.c
2463
r = kvm->arch.kvm_ops->svm_off(kvm);
arch/powerpc/kvm/powerpc.c
2467
struct kvm *kvm = filp->private_data;
arch/powerpc/kvm/powerpc.c
2468
r = kvm->arch.kvm_ops->arch_vm_ioctl(filp, ioctl, arg);
arch/powerpc/kvm/powerpc.c
2517
if (vcpu->kvm->arch.kvm_ops->create_vcpu_debugfs)
arch/powerpc/kvm/powerpc.c
2518
vcpu->kvm->arch.kvm_ops->create_vcpu_debugfs(vcpu, debugfs_dentry);
arch/powerpc/kvm/powerpc.c
2521
void kvm_arch_create_vm_debugfs(struct kvm *kvm)
arch/powerpc/kvm/powerpc.c
2523
if (kvm->arch.kvm_ops->create_vm_debugfs)
arch/powerpc/kvm/powerpc.c
2524
kvm->arch.kvm_ops->create_vm_debugfs(kvm);
arch/powerpc/kvm/powerpc.c
266
if (!vcpu->arch.papr_enabled && is_kvmppc_hv_enabled(vcpu->kvm))
arch/powerpc/kvm/powerpc.c
357
if (vcpu->kvm->arch.kvm_ops && vcpu->kvm->arch.kvm_ops->store_to_eaddr)
arch/powerpc/kvm/powerpc.c
358
r = vcpu->kvm->arch.kvm_ops->store_to_eaddr(vcpu, eaddr, ptr,
arch/powerpc/kvm/powerpc.c
384
if (kvm_write_guest(vcpu->kvm, pte.raddr, ptr, size))
arch/powerpc/kvm/powerpc.c
400
if (vcpu->kvm->arch.kvm_ops && vcpu->kvm->arch.kvm_ops->load_from_eaddr)
arch/powerpc/kvm/powerpc.c
401
rc = vcpu->kvm->arch.kvm_ops->load_from_eaddr(vcpu, eaddr, ptr,
arch/powerpc/kvm/powerpc.c
431
rc = kvm_read_guest(vcpu->kvm, pte.raddr, ptr, size);
arch/powerpc/kvm/powerpc.c
440
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
arch/powerpc/kvm/powerpc.c
469
kvm->arch.kvm_ops = kvm_ops;
arch/powerpc/kvm/powerpc.c
470
r = kvmppc_core_init_vm(kvm);
arch/powerpc/kvm/powerpc.c
478
void kvm_arch_destroy_vm(struct kvm *kvm)
arch/powerpc/kvm/powerpc.c
486
if (is_kvmppc_hv_enabled(kvm))
arch/powerpc/kvm/powerpc.c
490
kvm_destroy_vcpus(kvm);
arch/powerpc/kvm/powerpc.c
492
mutex_lock(&kvm->lock);
arch/powerpc/kvm/powerpc.c
494
kvmppc_core_destroy_vm(kvm);
arch/powerpc/kvm/powerpc.c
496
mutex_unlock(&kvm->lock);
arch/powerpc/kvm/powerpc.c
499
module_put(kvm->arch.kvm_ops->owner);
arch/powerpc/kvm/powerpc.c
502
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
arch/powerpc/kvm/powerpc.c
508
if (kvm) {
arch/powerpc/kvm/powerpc.c
513
hv_enabled = is_kvmppc_hv_enabled(kvm);
arch/powerpc/kvm/powerpc.c
590
if (kvm) {
arch/powerpc/kvm/powerpc.c
591
if (kvm->arch.emul_smt_mode > 1)
arch/powerpc/kvm/powerpc.c
592
r = kvm->arch.emul_smt_mode;
arch/powerpc/kvm/powerpc.c
594
r = kvm->arch.smt_mode;
arch/powerpc/kvm/powerpc.c
714
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
arch/powerpc/kvm/powerpc.c
716
kvmppc_core_free_memslot(kvm, slot);
arch/powerpc/kvm/powerpc.c
719
int kvm_arch_prepare_memory_region(struct kvm *kvm,
arch/powerpc/kvm/powerpc.c
724
return kvmppc_core_prepare_memory_region(kvm, old, new, change);
arch/powerpc/kvm/powerpc.c
727
void kvm_arch_commit_memory_region(struct kvm *kvm,
arch/powerpc/kvm/powerpc.c
732
kvmppc_core_commit_memory_region(kvm, old, new, change);
arch/powerpc/kvm/powerpc.c
735
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
arch/powerpc/kvm/powerpc.c
738
kvmppc_core_flush_memslot(kvm, slot);
arch/powerpc/kvm/powerpc.c
741
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
arch/powerpc/kvm/powerpc.c
857
struct kvm *kvm = irqfd->kvm;
arch/powerpc/kvm/powerpc.c
859
if (kvm->arch.kvm_ops->irq_bypass_add_producer)
arch/powerpc/kvm/powerpc.c
860
return kvm->arch.kvm_ops->irq_bypass_add_producer(cons, prod);
arch/powerpc/kvm/powerpc.c
870
struct kvm *kvm = irqfd->kvm;
arch/powerpc/kvm/powerpc.c
872
if (kvm->arch.kvm_ops->irq_bypass_del_producer)
arch/powerpc/kvm/powerpc.c
873
kvm->arch.kvm_ops->irq_bypass_del_producer(cons, prod);
arch/powerpc/kvm/trace.h
8
#define TRACE_SYSTEM kvm
arch/riscv/include/asm/kvm_aia.h
100
int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type,
arch/riscv/include/asm/kvm_aia.h
102
int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type);
arch/riscv/include/asm/kvm_aia.h
109
int kvm_riscv_aia_aplic_set_attr(struct kvm *kvm, unsigned long type, u32 v);
arch/riscv/include/asm/kvm_aia.h
110
int kvm_riscv_aia_aplic_get_attr(struct kvm *kvm, unsigned long type, u32 *v);
arch/riscv/include/asm/kvm_aia.h
111
int kvm_riscv_aia_aplic_has_attr(struct kvm *kvm, unsigned long type);
arch/riscv/include/asm/kvm_aia.h
112
int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level);
arch/riscv/include/asm/kvm_aia.h
113
int kvm_riscv_aia_aplic_init(struct kvm *kvm);
arch/riscv/include/asm/kvm_aia.h
114
void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm);
arch/riscv/include/asm/kvm_aia.h
156
int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index,
arch/riscv/include/asm/kvm_aia.h
158
int kvm_riscv_aia_inject_msi(struct kvm *kvm, struct kvm_msi *msi);
arch/riscv/include/asm/kvm_aia.h
159
int kvm_riscv_aia_inject_irq(struct kvm *kvm, unsigned int irq, bool level);
arch/riscv/include/asm/kvm_aia.h
161
void kvm_riscv_aia_init_vm(struct kvm *kvm);
arch/riscv/include/asm/kvm_aia.h
162
void kvm_riscv_aia_destroy_vm(struct kvm *kvm);
arch/riscv/include/asm/kvm_gstage.h
104
static inline void kvm_riscv_gstage_init(struct kvm_gstage *gstage, struct kvm *kvm)
arch/riscv/include/asm/kvm_gstage.h
106
gstage->kvm = kvm;
arch/riscv/include/asm/kvm_gstage.h
108
gstage->vmid = READ_ONCE(kvm->arch.vmid.vmid);
arch/riscv/include/asm/kvm_gstage.h
109
gstage->pgd = kvm->arch.pgd;
arch/riscv/include/asm/kvm_gstage.h
110
gstage->pgd_levels = kvm->arch.pgd_levels;
arch/riscv/include/asm/kvm_gstage.h
13
struct kvm *kvm;
arch/riscv/include/asm/kvm_host.h
285
int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines);
arch/riscv/include/asm/kvm_mmu.h
11
int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa,
arch/riscv/include/asm/kvm_mmu.h
13
void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size);
arch/riscv/include/asm/kvm_mmu.h
17
int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm);
arch/riscv/include/asm/kvm_mmu.h
18
void kvm_riscv_mmu_free_pgd(struct kvm *kvm);
arch/riscv/include/asm/kvm_tlb.h
60
void kvm_riscv_fence_i(struct kvm *kvm,
arch/riscv/include/asm/kvm_tlb.h
62
void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
arch/riscv/include/asm/kvm_tlb.h
66
void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm,
arch/riscv/include/asm/kvm_tlb.h
69
void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
arch/riscv/include/asm/kvm_tlb.h
74
void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
arch/riscv/include/asm/kvm_tlb.h
77
void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
arch/riscv/include/asm/kvm_tlb.h
81
void kvm_riscv_hfence_vvma_all(struct kvm *kvm,
arch/riscv/include/asm/kvm_vcpu_timer.h
47
void kvm_riscv_guest_timer_init(struct kvm *kvm);
arch/riscv/include/asm/kvm_vmid.h
22
int kvm_riscv_gstage_vmid_init(struct kvm *kvm);
arch/riscv/kvm/aia.c
108
if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip)
arch/riscv/kvm/aia.c
158
if (kvm_riscv_aia_initialized(vcpu->kvm))
arch/riscv/kvm/aia.c
170
if (kvm_riscv_aia_initialized(vcpu->kvm))
arch/riscv/kvm/aia.c
262
if (!kvm_riscv_aia_initialized(vcpu->kvm))
arch/riscv/kvm/aia.c
426
kvm_riscv_aia_initialized(vcpu->kvm))
arch/riscv/kvm/aia_aplic.c
232
static void aplic_inject_msi(struct kvm *kvm, u32 irq, u32 target)
arch/riscv/kvm/aia_aplic.c
241
kvm_riscv_aia_inject_msi_by_id(kvm, hart_idx, guest_idx, eiid);
arch/riscv/kvm/aia_aplic.c
244
static void aplic_update_irq_range(struct kvm *kvm, u32 first, u32 last)
arch/riscv/kvm/aia_aplic.c
250
struct aplic *aplic = kvm->arch.aia.aplic_state;
arch/riscv/kvm/aia_aplic.c
273
aplic_inject_msi(kvm, irq, target);
arch/riscv/kvm/aia_aplic.c
277
int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level)
arch/riscv/kvm/aia_aplic.c
283
struct aplic *aplic = kvm->arch.aia.aplic_state;
arch/riscv/kvm/aia_aplic.c
332
aplic_inject_msi(kvm, source, target);
arch/riscv/kvm/aia_aplic.c
389
static int aplic_mmio_read_offset(struct kvm *kvm, gpa_t off, u32 *val32)
arch/riscv/kvm/aia_aplic.c
392
struct aplic *aplic = kvm->arch.aia.aplic_state;
arch/riscv/kvm/aia_aplic.c
449
return aplic_mmio_read_offset(vcpu->kvm,
arch/riscv/kvm/aia_aplic.c
450
addr - vcpu->kvm->arch.aia.aplic_addr,
arch/riscv/kvm/aia_aplic.c
454
static int aplic_mmio_write_offset(struct kvm *kvm, gpa_t off, u32 val32)
arch/riscv/kvm/aia_aplic.c
457
struct aplic *aplic = kvm->arch.aia.aplic_state;
arch/riscv/kvm/aia_aplic.c
500
kvm_riscv_aia_inject_msi_by_id(kvm,
arch/riscv/kvm/aia_aplic.c
510
aplic_update_irq_range(kvm, 1, aplic->nr_irqs - 1);
arch/riscv/kvm/aia_aplic.c
521
return aplic_mmio_write_offset(vcpu->kvm,
arch/riscv/kvm/aia_aplic.c
522
addr - vcpu->kvm->arch.aia.aplic_addr,
arch/riscv/kvm/aia_aplic.c
531
int kvm_riscv_aia_aplic_set_attr(struct kvm *kvm, unsigned long type, u32 v)
arch/riscv/kvm/aia_aplic.c
535
if (!kvm->arch.aia.aplic_state)
arch/riscv/kvm/aia_aplic.c
538
rc = aplic_mmio_write_offset(kvm, type, v);
arch/riscv/kvm/aia_aplic.c
545
int kvm_riscv_aia_aplic_get_attr(struct kvm *kvm, unsigned long type, u32 *v)
arch/riscv/kvm/aia_aplic.c
549
if (!kvm->arch.aia.aplic_state)
arch/riscv/kvm/aia_aplic.c
552
rc = aplic_mmio_read_offset(kvm, type, v);
arch/riscv/kvm/aia_aplic.c
559
int kvm_riscv_aia_aplic_has_attr(struct kvm *kvm, unsigned long type)
arch/riscv/kvm/aia_aplic.c
564
if (!kvm->arch.aia.aplic_state)
arch/riscv/kvm/aia_aplic.c
567
rc = aplic_mmio_read_offset(kvm, type, &val);
arch/riscv/kvm/aia_aplic.c
574
int kvm_riscv_aia_aplic_init(struct kvm *kvm)
arch/riscv/kvm/aia_aplic.c
580
if (!kvm->arch.aia.nr_sources)
arch/riscv/kvm/aia_aplic.c
584
aplic = kzalloc_flex(*aplic, irqs, kvm->arch.aia.nr_sources + 1);
arch/riscv/kvm/aia_aplic.c
587
kvm->arch.aia.aplic_state = aplic;
arch/riscv/kvm/aia_aplic.c
590
aplic->nr_irqs = kvm->arch.aia.nr_sources + 1;
arch/riscv/kvm/aia_aplic.c
597
mutex_lock(&kvm->slots_lock);
arch/riscv/kvm/aia_aplic.c
598
ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
arch/riscv/kvm/aia_aplic.c
599
kvm->arch.aia.aplic_addr,
arch/riscv/kvm/aia_aplic.c
602
mutex_unlock(&kvm->slots_lock);
arch/riscv/kvm/aia_aplic.c
607
ret = kvm_riscv_setup_default_irq_routing(kvm, aplic->nr_irqs);
arch/riscv/kvm/aia_aplic.c
614
mutex_lock(&kvm->slots_lock);
arch/riscv/kvm/aia_aplic.c
615
kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev);
arch/riscv/kvm/aia_aplic.c
616
mutex_unlock(&kvm->slots_lock);
arch/riscv/kvm/aia_aplic.c
618
kvm->arch.aia.aplic_state = NULL;
arch/riscv/kvm/aia_aplic.c
623
void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm)
arch/riscv/kvm/aia_aplic.c
625
struct aplic *aplic = kvm->arch.aia.aplic_state;
arch/riscv/kvm/aia_aplic.c
630
mutex_lock(&kvm->slots_lock);
arch/riscv/kvm/aia_aplic.c
631
kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev);
arch/riscv/kvm/aia_aplic.c
632
mutex_unlock(&kvm->slots_lock);
arch/riscv/kvm/aia_aplic.c
634
kvm->arch.aia.aplic_state = NULL;
arch/riscv/kvm/aia_device.c
145
static int aia_aplic_addr(struct kvm *kvm, u64 *addr, bool write)
arch/riscv/kvm/aia_device.c
147
struct kvm_aia *aia = &kvm->arch.aia;
arch/riscv/kvm/aia_device.c
151
if (kvm_riscv_aia_initialized(kvm))
arch/riscv/kvm/aia_device.c
164
static int aia_imsic_addr(struct kvm *kvm, u64 *addr,
arch/riscv/kvm/aia_device.c
170
vcpu = kvm_get_vcpu(kvm, vcpu_idx);
arch/riscv/kvm/aia_device.c
177
if (kvm_riscv_aia_initialized(kvm))
arch/riscv/kvm/aia_device.c
20
struct kvm *kvm = dev->kvm;
arch/riscv/kvm/aia_device.c
226
static int aia_init(struct kvm *kvm)
arch/riscv/kvm/aia_device.c
23
if (irqchip_in_kernel(kvm))
arch/riscv/kvm/aia_device.c
232
struct kvm_aia *aia = &kvm->arch.aia;
arch/riscv/kvm/aia_device.c
236
if (kvm_riscv_aia_initialized(kvm))
arch/riscv/kvm/aia_device.c
240
if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
arch/riscv/kvm/aia_device.c
252
ret = kvm_riscv_aia_aplic_init(kvm);
arch/riscv/kvm/aia_device.c
257
kvm_for_each_vcpu(idx, vcpu, kvm) {
arch/riscv/kvm/aia_device.c
285
kvm->arch.aia.initialized = true;
arch/riscv/kvm/aia_device.c
291
vcpu = kvm_get_vcpu(kvm, i);
arch/riscv/kvm/aia_device.c
296
kvm_riscv_aia_aplic_cleanup(kvm);
arch/riscv/kvm/aia_device.c
30
if (kvm_trylock_all_vcpus(kvm))
arch/riscv/kvm/aia_device.c
313
mutex_lock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
314
r = aia_config(dev->kvm, type, &nr, true);
arch/riscv/kvm/aia_device.c
315
mutex_unlock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
323
nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
arch/riscv/kvm/aia_device.c
324
mutex_lock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
326
r = aia_aplic_addr(dev->kvm, &addr, true);
arch/riscv/kvm/aia_device.c
328
r = aia_imsic_addr(dev->kvm, &addr,
arch/riscv/kvm/aia_device.c
33
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/riscv/kvm/aia_device.c
330
mutex_unlock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
337
mutex_lock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
338
r = aia_init(dev->kvm);
arch/riscv/kvm/aia_device.c
339
mutex_unlock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
348
mutex_lock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
349
r = kvm_riscv_aia_aplic_set_attr(dev->kvm, type, nr);
arch/riscv/kvm/aia_device.c
350
mutex_unlock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
357
mutex_lock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
358
r = kvm_riscv_aia_imsic_rw_attr(dev->kvm, type, true, &v);
arch/riscv/kvm/aia_device.c
359
mutex_unlock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
380
mutex_lock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
381
r = aia_config(dev->kvm, type, &nr, false);
arch/riscv/kvm/aia_device.c
382
mutex_unlock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
39
kvm->arch.aia.in_kernel = true;
arch/riscv/kvm/aia_device.c
394
nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
arch/riscv/kvm/aia_device.c
395
mutex_lock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
397
r = aia_aplic_addr(dev->kvm, &addr, false);
arch/riscv/kvm/aia_device.c
399
r = aia_imsic_addr(dev->kvm, &addr,
arch/riscv/kvm/aia_device.c
401
mutex_unlock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
413
mutex_lock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
414
r = kvm_riscv_aia_aplic_get_attr(dev->kvm, type, &nr);
arch/riscv/kvm/aia_device.c
415
mutex_unlock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
42
kvm_unlock_all_vcpus(kvm);
arch/riscv/kvm/aia_device.c
427
mutex_lock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
428
r = kvm_riscv_aia_imsic_rw_attr(dev->kvm, type, false, &v);
arch/riscv/kvm/aia_device.c
429
mutex_unlock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
460
nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
arch/riscv/kvm/aia_device.c
473
mutex_lock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
474
r = kvm_riscv_aia_aplic_has_attr(dev->kvm, attr->attr);
arch/riscv/kvm/aia_device.c
475
mutex_unlock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
478
mutex_lock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
479
r = kvm_riscv_aia_imsic_has_attr(dev->kvm, attr->attr);
arch/riscv/kvm/aia_device.c
480
mutex_unlock(&dev->kvm->lock);
arch/riscv/kvm/aia_device.c
499
if (!kvm_riscv_aia_initialized(vcpu->kvm))
arch/riscv/kvm/aia_device.c
51
static int aia_config(struct kvm *kvm, unsigned long type,
arch/riscv/kvm/aia_device.c
515
if (!kvm_riscv_aia_initialized(vcpu->kvm))
arch/riscv/kvm/aia_device.c
54
struct kvm_aia *aia = &kvm->arch.aia;
arch/riscv/kvm/aia_device.c
544
if (!kvm_riscv_aia_initialized(vcpu->kvm))
arch/riscv/kvm/aia_device.c
551
int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index,
arch/riscv/kvm/aia_device.c
558
if (!kvm_riscv_aia_initialized(kvm))
arch/riscv/kvm/aia_device.c
562
kvm_for_each_vcpu(idx, vcpu, kvm) {
arch/riscv/kvm/aia_device.c
57
if (write && kvm_riscv_aia_initialized(kvm))
arch/riscv/kvm/aia_device.c
572
int kvm_riscv_aia_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
arch/riscv/kvm/aia_device.c
578
struct kvm_aia *aia = &kvm->arch.aia;
arch/riscv/kvm/aia_device.c
582
if (!kvm_riscv_aia_initialized(kvm))
arch/riscv/kvm/aia_device.c
593
kvm_for_each_vcpu(idx, vcpu, kvm) {
arch/riscv/kvm/aia_device.c
606
int kvm_riscv_aia_inject_irq(struct kvm *kvm, unsigned int irq, bool level)
arch/riscv/kvm/aia_device.c
609
if (!kvm_riscv_aia_initialized(kvm))
arch/riscv/kvm/aia_device.c
613
return kvm_riscv_aia_aplic_inject(kvm, irq, level);
arch/riscv/kvm/aia_device.c
616
void kvm_riscv_aia_init_vm(struct kvm *kvm)
arch/riscv/kvm/aia_device.c
618
struct kvm_aia *aia = &kvm->arch.aia;
arch/riscv/kvm/aia_device.c
642
void kvm_riscv_aia_destroy_vm(struct kvm *kvm)
arch/riscv/kvm/aia_device.c
645
if (!kvm_riscv_aia_initialized(kvm))
arch/riscv/kvm/aia_device.c
649
kvm_riscv_aia_aplic_cleanup(kvm);
arch/riscv/kvm/aia_imsic.c
1001
int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type)
arch/riscv/kvm/aia_imsic.c
1007
if (!kvm_riscv_aia_initialized(kvm))
arch/riscv/kvm/aia_imsic.c
1011
vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
arch/riscv/kvm/aia_imsic.c
1089
kvm_riscv_aia_inject_msi(vcpu->kvm, &msi);
arch/riscv/kvm/aia_imsic.c
1104
struct kvm *kvm = vcpu->kvm;
arch/riscv/kvm/aia_imsic.c
1107
if (!kvm->arch.aia.nr_ids)
arch/riscv/kvm/aia_imsic.c
1117
imsic->nr_msis = kvm->arch.aia.nr_ids + 1;
arch/riscv/kvm/aia_imsic.c
1136
mutex_lock(&kvm->slots_lock);
arch/riscv/kvm/aia_imsic.c
1137
ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
arch/riscv/kvm/aia_imsic.c
1141
mutex_unlock(&kvm->slots_lock);
arch/riscv/kvm/aia_imsic.c
1158
struct kvm *kvm = vcpu->kvm;
arch/riscv/kvm/aia_imsic.c
1166
mutex_lock(&kvm->slots_lock);
arch/riscv/kvm/aia_imsic.c
1167
kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &imsic->iodev);
arch/riscv/kvm/aia_imsic.c
1168
mutex_unlock(&kvm->slots_lock);
arch/riscv/kvm/aia_imsic.c
770
kvm_riscv_mmu_iounmap(vcpu->kvm, vcpu->arch.aia_context.imsic_addr,
arch/riscv/kvm/aia_imsic.c
799
struct kvm *kvm = vcpu->kvm;
arch/riscv/kvm/aia_imsic.c
806
if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_EMUL)
arch/riscv/kvm/aia_imsic.c
828
if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_HWACCEL) {
arch/riscv/kvm/aia_imsic.c
858
ret = kvm_riscv_mmu_ioremap(kvm, vcpu->arch.aia_context.imsic_addr,
arch/riscv/kvm/aia_imsic.c
955
int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type,
arch/riscv/kvm/aia_imsic.c
964
if (!kvm_riscv_aia_initialized(kvm))
arch/riscv/kvm/aia_imsic.c
968
vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
arch/riscv/kvm/gstage.c
124
kvm_riscv_hfence_gvma_vmid_gpa(gstage->kvm, -1UL, 0, addr, BIT(order), order,
arch/riscv/kvm/gstage.c
444
cond_resched_rwlock_write(&gstage->kvm->mmu_lock);
arch/riscv/kvm/mmu.c
101
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
arch/riscv/kvm/mmu.c
112
kvm_riscv_gstage_init(&gstage, kvm);
arch/riscv/kvm/mmu.c
116
kvm_flush_remote_tlbs_range(kvm, start >> PAGE_SHIFT,
arch/riscv/kvm/mmu.c
120
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
arch/riscv/kvm/mmu.c
124
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free)
arch/riscv/kvm/mmu.c
128
void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
arch/riscv/kvm/mmu.c
132
void kvm_arch_flush_shadow_all(struct kvm *kvm)
arch/riscv/kvm/mmu.c
134
kvm_riscv_mmu_free_pgd(kvm);
arch/riscv/kvm/mmu.c
137
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
arch/riscv/kvm/mmu.c
145
kvm_riscv_gstage_init(&gstage, kvm);
arch/riscv/kvm/mmu.c
147
write_lock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
149
write_unlock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
151
kvm_flush_remote_tlbs_range(kvm, gpa >> PAGE_SHIFT,
arch/riscv/kvm/mmu.c
155
void kvm_arch_commit_memory_region(struct kvm *kvm,
arch/riscv/kvm/mmu.c
165
if (kvm_dirty_log_manual_protect_and_init_set(kvm))
arch/riscv/kvm/mmu.c
167
mmu_wp_memory_region(kvm, new->id);
arch/riscv/kvm/mmu.c
171
int kvm_arch_prepare_memory_region(struct kvm *kvm,
arch/riscv/kvm/mmu.c
189
kvm_riscv_gstage_gpa_size(kvm->arch.pgd_levels) >> PAGE_SHIFT)
arch/riscv/kvm/mmu.c
19
static void mmu_wp_memory_region(struct kvm *kvm, int slot)
arch/riscv/kvm/mmu.c
21
struct kvm_memslots *slots = kvm_memslots(kvm);
arch/riscv/kvm/mmu.c
245
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
arch/riscv/kvm/mmu.c
250
if (!kvm->arch.pgd)
arch/riscv/kvm/mmu.c
253
lockdep_assert_held_write(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
255
kvm_riscv_gstage_init(&gstage, kvm);
arch/riscv/kvm/mmu.c
260
kvm_flush_remote_tlbs_range(kvm, range->start,
arch/riscv/kvm/mmu.c
265
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/riscv/kvm/mmu.c
272
if (!kvm->arch.pgd)
arch/riscv/kvm/mmu.c
277
kvm_riscv_gstage_init(&gstage, kvm);
arch/riscv/kvm/mmu.c
28
kvm_riscv_gstage_init(&gstage, kvm);
arch/riscv/kvm/mmu.c
285
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/riscv/kvm/mmu.c
292
if (!kvm->arch.pgd)
arch/riscv/kvm/mmu.c
297
kvm_riscv_gstage_init(&gstage, kvm);
arch/riscv/kvm/mmu.c
30
write_lock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
32
write_unlock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
34
kvm_flush_remote_tlbs_memslot(kvm, memslot);
arch/riscv/kvm/mmu.c
361
static int get_hva_mapping_size(struct kvm *kvm,
arch/riscv/kvm/mmu.c
37
int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa,
arch/riscv/kvm/mmu.c
385
pgd = pgdp_get(pgd_offset(kvm->mm, hva));
arch/riscv/kvm/mmu.c
414
static unsigned long transparent_hugepage_adjust(struct kvm *kvm,
arch/riscv/kvm/mmu.c
429
sz = get_hva_mapping_size(kvm, hva);
arch/riscv/kvm/mmu.c
44
unsigned long pgd_levels = kvm->arch.pgd_levels;
arch/riscv/kvm/mmu.c
465
static bool kvm_riscv_mmu_dirty_log_write_fault_fast(struct kvm *kvm,
arch/riscv/kvm/mmu.c
479
kvm_riscv_gstage_init(&gstage, kvm);
arch/riscv/kvm/mmu.c
480
mmu_seq = kvm->mmu_invalidate_seq;
arch/riscv/kvm/mmu.c
482
read_lock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
484
if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) {
arch/riscv/kvm/mmu.c
503
mark_page_dirty_in_slot(kvm, memslot, gfn);
arch/riscv/kvm/mmu.c
52
kvm_riscv_gstage_init(&gstage, kvm);
arch/riscv/kvm/mmu.c
525
read_unlock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
547
struct kvm *kvm = vcpu->kvm;
arch/riscv/kvm/mmu.c
555
kvm_riscv_gstage_init(&gstage, kvm);
arch/riscv/kvm/mmu.c
561
kvm_riscv_mmu_dirty_log_write_fault_fast(kvm, memslot, gpa, out_map))
arch/riscv/kvm/mmu.c
565
ret = kvm_mmu_topup_memory_cache(pcache, kvm->arch.pgd_levels);
arch/riscv/kvm/mmu.c
606
mmu_seq = kvm->mmu_invalidate_seq;
arch/riscv/kvm/mmu.c
633
write_lock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
635
if (mmu_invalidate_retry(kvm, mmu_seq))
arch/riscv/kvm/mmu.c
644
vma_pagesize = transparent_hugepage_adjust(kvm, memslot, hva, &hfn, &gpa);
arch/riscv/kvm/mmu.c
647
mark_page_dirty_in_slot(kvm, memslot, gfn);
arch/riscv/kvm/mmu.c
659
kvm_release_faultin_page(kvm, page, ret && ret != -EEXIST, writable);
arch/riscv/kvm/mmu.c
660
write_unlock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
664
int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm)
arch/riscv/kvm/mmu.c
668
if (kvm->arch.pgd != NULL) {
arch/riscv/kvm/mmu.c
677
kvm->arch.pgd = page_to_virt(pgd_page);
arch/riscv/kvm/mmu.c
678
kvm->arch.pgd_phys = page_to_phys(pgd_page);
arch/riscv/kvm/mmu.c
679
kvm->arch.pgd_levels = kvm_riscv_gstage_max_pgd_levels;
arch/riscv/kvm/mmu.c
684
void kvm_riscv_mmu_free_pgd(struct kvm *kvm)
arch/riscv/kvm/mmu.c
690
write_lock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
691
if (kvm->arch.pgd) {
arch/riscv/kvm/mmu.c
692
kvm_riscv_gstage_init(&gstage, kvm);
arch/riscv/kvm/mmu.c
694
kvm_riscv_gstage_gpa_size(kvm->arch.pgd_levels), false);
arch/riscv/kvm/mmu.c
695
pgd = READ_ONCE(kvm->arch.pgd);
arch/riscv/kvm/mmu.c
696
kvm->arch.pgd = NULL;
arch/riscv/kvm/mmu.c
697
kvm->arch.pgd_phys = 0;
arch/riscv/kvm/mmu.c
698
kvm->arch.pgd_levels = 0;
arch/riscv/kvm/mmu.c
700
write_unlock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
703
kvm_flush_remote_tlbs(kvm);
arch/riscv/kvm/mmu.c
71
write_lock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
711
struct kvm_arch *ka = &vcpu->kvm->arch;
arch/riscv/kvm/mmu.c
73
write_unlock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
85
void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size)
arch/riscv/kvm/mmu.c
90
kvm_riscv_gstage_init(&gstage, kvm);
arch/riscv/kvm/mmu.c
92
write_lock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
94
write_unlock(&kvm->mmu_lock);
arch/riscv/kvm/mmu.c
97
kvm_flush_remote_tlbs_range(kvm, gpa >> PAGE_SHIFT,
arch/riscv/kvm/tlb.c
180
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
arch/riscv/kvm/tlb.c
199
struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
arch/riscv/kvm/tlb.c
210
struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
arch/riscv/kvm/tlb.c
328
static void make_xfence_request(struct kvm *kvm,
arch/riscv/kvm/tlb.c
339
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/riscv/kvm/tlb.c
362
kvm_make_vcpus_request_mask(kvm, actual_req, vcpu_mask);
arch/riscv/kvm/tlb.c
365
void kvm_riscv_fence_i(struct kvm *kvm,
arch/riscv/kvm/tlb.c
368
make_xfence_request(kvm, hbase, hmask, KVM_REQ_FENCE_I,
arch/riscv/kvm/tlb.c
372
void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
arch/riscv/kvm/tlb.c
385
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
arch/riscv/kvm/tlb.c
389
void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm,
arch/riscv/kvm/tlb.c
397
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
arch/riscv/kvm/tlb.c
401
void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
arch/riscv/kvm/tlb.c
415
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
arch/riscv/kvm/tlb.c
419
void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
arch/riscv/kvm/tlb.c
428
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
arch/riscv/kvm/tlb.c
432
void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
arch/riscv/kvm/tlb.c
445
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
arch/riscv/kvm/tlb.c
449
void kvm_riscv_hfence_vvma_all(struct kvm *kvm,
arch/riscv/kvm/tlb.c
457
make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
arch/riscv/kvm/tlb.c
461
int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages)
arch/riscv/kvm/tlb.c
463
kvm_riscv_hfence_gvma_vmid_gpa(kvm, -1UL, 0,
arch/riscv/kvm/tlb.c
465
PAGE_SHIFT, READ_ONCE(kvm->arch.vmid.vmid));
arch/riscv/kvm/trace.h
14
#define TRACE_SYSTEM kvm
arch/riscv/kvm/vcpu.c
127
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
arch/riscv/kvm/vcpu.c
547
if (vcpu->kvm->arch.mp_state_reset)
arch/riscv/kvm/vcpu.c
949
if (kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) ||
arch/riscv/kvm/vcpu_exit.c
27
memslot = gfn_to_memslot(vcpu->kvm, gfn);
arch/riscv/kvm/vcpu_sbi.c
157
kvm_for_each_vcpu(i, tmp, vcpu->kvm) {
arch/riscv/kvm/vcpu_sbi.c
162
kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP);
arch/riscv/kvm/vcpu_sbi_hsm.c
23
target_vcpu = kvm_get_vcpu_by_id(vcpu->kvm, target_vcpuid);
arch/riscv/kvm/vcpu_sbi_hsm.c
69
target_vcpu = kvm_get_vcpu_by_id(vcpu->kvm, target_vcpuid);
arch/riscv/kvm/vcpu_sbi_replace.c
103
kvm_riscv_fence_i(vcpu->kvm, hbase, hmask);
arch/riscv/kvm/vcpu_sbi_replace.c
107
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
arch/riscv/kvm/vcpu_sbi_replace.c
109
kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask, vmid);
arch/riscv/kvm/vcpu_sbi_replace.c
111
kvm_riscv_hfence_vvma_gva(vcpu->kvm, hbase, hmask,
arch/riscv/kvm/vcpu_sbi_replace.c
116
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
arch/riscv/kvm/vcpu_sbi_replace.c
118
kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, hbase, hmask,
arch/riscv/kvm/vcpu_sbi_replace.c
121
kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, hbase, hmask, cp->a2,
arch/riscv/kvm/vcpu_sbi_replace.c
62
kvm_for_each_vcpu(i, tmp, vcpu->kvm) {
arch/riscv/kvm/vcpu_sbi_system.c
52
kvm_for_each_vcpu(i, tmp, vcpu->kvm) {
arch/riscv/kvm/vcpu_sbi_v01.c
23
struct kvm *kvm = vcpu->kvm;
arch/riscv/kvm/vcpu_sbi_v01.c
52
hmask = (1UL << atomic_read(&kvm->online_vcpus)) - 1;
arch/riscv/kvm/vcpu_sbi_v01.c
57
rvcpu = kvm_get_vcpu_by_id(vcpu->kvm, i);
arch/riscv/kvm/vcpu_sbi_v01.c
76
hmask = (1UL << atomic_read(&kvm->online_vcpus)) - 1;
arch/riscv/kvm/vcpu_sbi_v01.c
81
kvm_riscv_fence_i(vcpu->kvm, 0, hmask);
arch/riscv/kvm/vcpu_sbi_v01.c
83
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
arch/riscv/kvm/vcpu_sbi_v01.c
85
kvm_riscv_hfence_vvma_all(vcpu->kvm, 0, hmask, vmid);
arch/riscv/kvm/vcpu_sbi_v01.c
87
kvm_riscv_hfence_vvma_gva(vcpu->kvm, 0, hmask, cp->a1,
arch/riscv/kvm/vcpu_sbi_v01.c
90
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
arch/riscv/kvm/vcpu_sbi_v01.c
92
kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, 0, hmask,
arch/riscv/kvm/vcpu_sbi_v01.c
95
kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, 0, hmask,
arch/riscv/kvm/vcpu_timer.c
116
struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
arch/riscv/kvm/vcpu_timer.c
133
struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
arch/riscv/kvm/vcpu_timer.c
145
struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
arch/riscv/kvm/vcpu_timer.c
165
struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
arch/riscv/kvm/vcpu_timer.c
205
struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
arch/riscv/kvm/vcpu_timer.c
292
struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
arch/riscv/kvm/vcpu_timer.c
377
void kvm_riscv_guest_timer_init(struct kvm *kvm)
arch/riscv/kvm/vcpu_timer.c
379
struct kvm_guest_timer *gt = &kvm->arch.timer;
arch/riscv/kvm/vcpu_timer.c
48
struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
arch/riscv/kvm/vcpu_timer.c
88
struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
arch/riscv/kvm/vm.c
108
rc = kvm_set_irq_routing(kvm, ents, lines, 0);
arch/riscv/kvm/vm.c
114
bool kvm_arch_can_set_irq_routing(struct kvm *kvm)
arch/riscv/kvm/vm.c
116
return irqchip_in_kernel(kvm);
arch/riscv/kvm/vm.c
119
int kvm_set_routing_entry(struct kvm *kvm,
arch/riscv/kvm/vm.c
151
struct kvm *kvm, int irq_source_id, int level,
arch/riscv/kvm/vm.c
159
return kvm_set_msi(e, kvm, irq_source_id, level, line_status);
arch/riscv/kvm/vm.c
162
return kvm_riscv_set_irq(e, kvm, irq_source_id,
arch/riscv/kvm/vm.c
169
bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
arch/riscv/kvm/vm.c
171
return irqchip_in_kernel(kvm);
arch/riscv/kvm/vm.c
174
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
arch/riscv/kvm/vm.c
202
if (!kvm)
arch/riscv/kvm/vm.c
205
r = kvm_riscv_gstage_gpa_bits(kvm->arch.pgd_levels);
arch/riscv/kvm/vm.c
215
int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
arch/riscv/kvm/vm.c
222
kvm->arch.mp_state_reset = true;
arch/riscv/kvm/vm.c
250
mutex_lock(&kvm->lock);
arch/riscv/kvm/vm.c
251
mutex_lock(&kvm->slots_lock);
arch/riscv/kvm/vm.c
253
if (kvm->created_vcpus || !kvm_are_all_memslots_empty(kvm))
arch/riscv/kvm/vm.c
256
kvm->arch.pgd_levels = new_levels;
arch/riscv/kvm/vm.c
258
mutex_unlock(&kvm->slots_lock);
arch/riscv/kvm/vm.c
259
mutex_unlock(&kvm->lock);
arch/riscv/kvm/vm.c
31
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
arch/riscv/kvm/vm.c
35
r = kvm_riscv_mmu_alloc_pgd(kvm);
arch/riscv/kvm/vm.c
39
r = kvm_riscv_gstage_vmid_init(kvm);
arch/riscv/kvm/vm.c
41
kvm_riscv_mmu_free_pgd(kvm);
arch/riscv/kvm/vm.c
45
kvm_riscv_aia_init_vm(kvm);
arch/riscv/kvm/vm.c
47
kvm_riscv_guest_timer_init(kvm);
arch/riscv/kvm/vm.c
52
void kvm_arch_destroy_vm(struct kvm *kvm)
arch/riscv/kvm/vm.c
54
kvm_destroy_vcpus(kvm);
arch/riscv/kvm/vm.c
56
kvm_riscv_aia_destroy_vm(kvm);
arch/riscv/kvm/vm.c
59
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irql,
arch/riscv/kvm/vm.c
62
if (!irqchip_in_kernel(kvm))
arch/riscv/kvm/vm.c
65
return kvm_riscv_aia_inject_irq(kvm, irql->irq, irql->level);
arch/riscv/kvm/vm.c
69
struct kvm *kvm, int irq_source_id,
arch/riscv/kvm/vm.c
83
return kvm_riscv_aia_inject_msi(kvm, &msi);
arch/riscv/kvm/vm.c
87
struct kvm *kvm, int irq_source_id,
arch/riscv/kvm/vm.c
90
return kvm_riscv_aia_inject_irq(kvm, e->irqchip.pin, level);
arch/riscv/kvm/vm.c
93
int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines)
arch/riscv/kvm/vmid.c
123
kvm_for_each_vcpu(i, v, vcpu->kvm)
arch/riscv/kvm/vmid.c
49
int kvm_riscv_gstage_vmid_init(struct kvm *kvm)
arch/riscv/kvm/vmid.c
52
kvm->arch.vmid.vmid_version = 0;
arch/riscv/kvm/vmid.c
53
kvm->arch.vmid.vmid = 0;
arch/riscv/kvm/vmid.c
76
struct kvm_vmid *vmid = &vcpu->kvm->arch.vmid;
arch/s390/include/asm/kvm_host.h
624
struct kvm *kvm; /* 0x0920 */
arch/s390/include/asm/kvm_host.h
736
void kvm_arch_crypto_clear_masks(struct kvm *kvm);
arch/s390/include/asm/kvm_host.h
737
void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
arch/s390/include/asm/kvm_host.h
753
bool kvm_s390_pv_is_protected(struct kvm *kvm);
arch/s390/include/asm/kvm_host.h
759
extern int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc);
arch/s390/include/asm/kvm_host.h
760
extern int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc);
arch/s390/include/asm/kvm_host.h
762
bool kvm_s390_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa);
arch/s390/include/asm/kvm_host.h
764
static inline void kvm_arch_free_memslot(struct kvm *kvm,
arch/s390/include/asm/kvm_host.h
766
static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {}
arch/s390/include/asm/kvm_host.h
767
static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {}
arch/s390/include/asm/kvm_host.h
768
static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
arch/s390/include/asm/kvm_host.h
774
void kvm_arch_free_vm(struct kvm *kvm);
arch/s390/include/asm/kvm_host.h
777
int (*kvm_register)(void *opaque, struct kvm *kvm);
arch/s390/include/asm/machine.h
96
DEFINE_MACHINE_HAS_FEATURE(kvm, MFEATURE_KVM)
arch/s390/kvm/diag.c
125
if (!kvm_is_gpa_in_memslot(vcpu->kvm, parm.token_addr))
arch/s390/kvm/diag.c
200
tcpu = kvm_get_vcpu_by_id(vcpu->kvm, tid);
arch/s390/kvm/diag.c
254
if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
arch/s390/kvm/diag.c
272
if (!vcpu->kvm->arch.css_support ||
arch/s390/kvm/diag.c
34
gmap_helper_discard(vcpu->kvm->mm, start, end);
arch/s390/kvm/diag.c
53
mmap_read_lock(vcpu->kvm->mm);
arch/s390/kvm/diag.c
74
mmap_read_unlock(vcpu->kvm->mm);
arch/s390/kvm/faultin.c
105
if (mmu_invalidate_retry_gfn_unsafe(kvm, inv_seq, f->gfn)) {
arch/s390/kvm/faultin.c
106
kvm_release_faultin_page(kvm, f->page, true, false);
arch/s390/kvm/faultin.c
110
scoped_guard(read_lock, &kvm->mmu_lock) {
arch/s390/kvm/faultin.c
111
if (!mmu_invalidate_retry_gfn(kvm, inv_seq, f->gfn)) {
arch/s390/kvm/faultin.c
113
rc = gmap_link(mc, kvm->arch.gmap, f, slot);
arch/s390/kvm/faultin.c
115
kvm_release_faultin_page(kvm, f->page, !!rc, f->write_attempt);
arch/s390/kvm/faultin.c
129
int kvm_s390_get_guest_page(struct kvm *kvm, struct guest_fault *f, gfn_t gfn, bool w)
arch/s390/kvm/faultin.c
131
struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
arch/s390/kvm/faultin.c
33
int kvm_s390_faultin_gfn(struct kvm_vcpu *vcpu, struct kvm *kvm, struct guest_fault *f)
arch/s390/kvm/faultin.c
46
kvm = vcpu->kvm;
arch/s390/kvm/faultin.c
50
lockdep_assert_held(&kvm->srcu);
arch/s390/kvm/faultin.c
52
scoped_guard(read_lock, &kvm->mmu_lock) {
arch/s390/kvm/faultin.c
53
if (gmap_try_fixup_minor(kvm->arch.gmap, f) == 0)
arch/s390/kvm/faultin.c
66
inv_seq = kvm->mmu_invalidate_seq;
arch/s390/kvm/faultin.c
73
slot = gfn_to_memslot(kvm, f->gfn);
arch/s390/kvm/faultin.h
16
int kvm_s390_faultin_gfn(struct kvm_vcpu *vcpu, struct kvm *kvm, struct guest_fault *f);
arch/s390/kvm/faultin.h
17
int kvm_s390_get_guest_page(struct kvm *kvm, struct guest_fault *f, gfn_t gfn, bool w);
arch/s390/kvm/faultin.h
19
static inline int kvm_s390_faultin_gfn_simple(struct kvm_vcpu *vcpu, struct kvm *kvm,
arch/s390/kvm/faultin.h
24
return kvm_s390_faultin_gfn(vcpu, kvm, &f);
arch/s390/kvm/faultin.h
27
static inline int kvm_s390_get_guest_page_and_read_gpa(struct kvm *kvm, struct guest_fault *f,
arch/s390/kvm/faultin.h
32
rc = kvm_s390_get_guest_page(kvm, f, gpa_to_gfn(gaddr), false);
arch/s390/kvm/faultin.h
41
static inline void kvm_s390_release_multiple(struct kvm *kvm, struct guest_fault *guest_faults,
arch/s390/kvm/faultin.h
47
kvm_release_faultin_page(kvm, guest_faults[i].page, ignore,
arch/s390/kvm/faultin.h
53
static inline bool kvm_s390_multiple_faults_need_retry(struct kvm *kvm, unsigned long seq,
arch/s390/kvm/faultin.h
62
if (unsafe && mmu_invalidate_retry_gfn_unsafe(kvm, seq, guest_faults[i].gfn))
arch/s390/kvm/faultin.h
64
if (!unsafe && mmu_invalidate_retry_gfn(kvm, seq, guest_faults[i].gfn))
arch/s390/kvm/faultin.h
70
static inline int kvm_s390_get_guest_pages(struct kvm *kvm, struct guest_fault *guest_faults,
arch/s390/kvm/faultin.h
76
rc = kvm_s390_get_guest_page(kvm, guest_faults + i, start + i, write_attempt);
arch/s390/kvm/faultin.h
83
#define kvm_s390_release_faultin_array(kvm, array, ignore) \
arch/s390/kvm/faultin.h
84
kvm_s390_release_multiple(kvm, array, ARRAY_SIZE(array), ignore)
arch/s390/kvm/faultin.h
86
#define kvm_s390_array_needs_retry_unsafe(kvm, seq, array) \
arch/s390/kvm/faultin.h
87
kvm_s390_multiple_faults_need_retry(kvm, seq, array, ARRAY_SIZE(array), true)
arch/s390/kvm/faultin.h
89
#define kvm_s390_array_needs_retry_safe(kvm, seq, array) \
arch/s390/kvm/faultin.h
90
kvm_s390_multiple_faults_need_retry(kvm, seq, array, ARRAY_SIZE(array), false)
arch/s390/kvm/gaccess.c
1003
ipte_unlock(vcpu->kvm);
arch/s390/kvm/gaccess.c
1019
rc = access_guest_page_gpa(vcpu->kvm, mode, gpa, data, fragment_len);
arch/s390/kvm/gaccess.c
1137
int cmpxchg_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, int len, union kvm_s390_quad *old,
arch/s390/kvm/gaccess.c
1155
lockdep_assert_held(&kvm->srcu);
arch/s390/kvm/gaccess.c
1160
rc = kvm_s390_faultin_gfn(NULL, kvm, &fault);
arch/s390/kvm/gaccess.c
1217
ipte_lock(vcpu->kvm);
arch/s390/kvm/gaccess.c
1220
ipte_unlock(vcpu->kvm);
arch/s390/kvm/gaccess.c
1233
int check_gpa_range(struct kvm *kvm, unsigned long gpa, unsigned long length,
arch/s390/kvm/gaccess.c
1241
rc = vm_check_access_key_gpa(kvm, access_key, mode, gpa);
arch/s390/kvm/gaccess.c
1286
struct kvm *kvm;
arch/s390/kvm/gaccess.c
1292
kvm = parent->kvm;
arch/s390/kvm/gaccess.c
1293
WARN_ON(!kvm);
arch/s390/kvm/gaccess.c
1300
return kvm_s390_get_guest_page(kvm, entries + LEVEL_MEM, gpa_to_gfn(saddr), false);
arch/s390/kvm/gaccess.c
1330
rc = kvm_s390_get_guest_page_and_read_gpa(kvm, entries + w->level,
arch/s390/kvm/gaccess.c
134
int ipte_lock_held(struct kvm *kvm)
arch/s390/kvm/gaccess.c
1347
rc = kvm_s390_get_guest_page_and_read_gpa(kvm, entries + w->level,
arch/s390/kvm/gaccess.c
1364
rc = kvm_s390_get_guest_page_and_read_gpa(kvm, entries + w->level,
arch/s390/kvm/gaccess.c
137
return kvm->arch.sca->ipte_control.kh != 0;
arch/s390/kvm/gaccess.c
1387
rc = kvm_s390_get_guest_page_and_read_gpa(kvm, entries + w->level,
arch/s390/kvm/gaccess.c
139
return kvm->arch.ipte_lock_count != 0;
arch/s390/kvm/gaccess.c
1406
rc = kvm_s390_get_guest_page_and_read_gpa(kvm, entries + w->level,
arch/s390/kvm/gaccess.c
1419
return kvm_s390_get_guest_page(kvm, entries + LEVEL_MEM, table.pte.pfra, wr);
arch/s390/kvm/gaccess.c
142
static void ipte_lock_simple(struct kvm *kvm)
arch/s390/kvm/gaccess.c
1429
lockdep_assert_held(&sg->kvm->mmu_lock);
arch/s390/kvm/gaccess.c
146
mutex_lock(&kvm->arch.ipte_mutex);
arch/s390/kvm/gaccess.c
147
kvm->arch.ipte_lock_count++;
arch/s390/kvm/gaccess.c
1473
lockdep_assert_held(&sg->kvm->mmu_lock);
arch/s390/kvm/gaccess.c
148
if (kvm->arch.ipte_lock_count > 1)
arch/s390/kvm/gaccess.c
151
ic = &kvm->arch.sca->ipte_control;
arch/s390/kvm/gaccess.c
1515
lockdep_assert_held(&sg->kvm->mmu_lock);
arch/s390/kvm/gaccess.c
1578
if (KVM_BUG_ON(l > TABLE_TYPE_REGION3, sg->kvm))
arch/s390/kvm/gaccess.c
1591
if (kvm_s390_array_needs_retry_safe(vcpu->kvm, seq, walk->raw_entries))
arch/s390/kvm/gaccess.c
1603
kvm_s390_release_faultin_array(vcpu->kvm, walk->raw_entries, false);
arch/s390/kvm/gaccess.c
1612
if (kvm_s390_array_needs_retry_unsafe(vcpu->kvm, seq, walk->raw_entries))
arch/s390/kvm/gaccess.c
162
mutex_unlock(&kvm->arch.ipte_mutex);
arch/s390/kvm/gaccess.c
1622
scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
arch/s390/kvm/gaccess.c
165
static void ipte_unlock_simple(struct kvm *kvm)
arch/s390/kvm/gaccess.c
1653
seq = vcpu->kvm->mmu_invalidate_seq;
arch/s390/kvm/gaccess.c
1667
kvm_s390_release_faultin_array(vcpu->kvm, walk.raw_entries, true);
arch/s390/kvm/gaccess.c
1676
if (KVM_BUG_ON(!test_bit(GMAP_FLAG_SHADOW, &sg->flags), vcpu->kvm))
arch/s390/kvm/gaccess.c
1683
ipte_lock(vcpu->kvm);
arch/s390/kvm/gaccess.c
1685
ipte_unlock(vcpu->kvm);
arch/s390/kvm/gaccess.c
169
mutex_lock(&kvm->arch.ipte_mutex);
arch/s390/kvm/gaccess.c
170
kvm->arch.ipte_lock_count--;
arch/s390/kvm/gaccess.c
171
if (kvm->arch.ipte_lock_count)
arch/s390/kvm/gaccess.c
173
ic = &kvm->arch.sca->ipte_control;
arch/s390/kvm/gaccess.c
179
wake_up(&kvm->arch.ipte_wq);
arch/s390/kvm/gaccess.c
181
mutex_unlock(&kvm->arch.ipte_mutex);
arch/s390/kvm/gaccess.c
184
static void ipte_lock_siif(struct kvm *kvm)
arch/s390/kvm/gaccess.c
189
ic = &kvm->arch.sca->ipte_control;
arch/s390/kvm/gaccess.c
202
static void ipte_unlock_siif(struct kvm *kvm)
arch/s390/kvm/gaccess.c
206
ic = &kvm->arch.sca->ipte_control;
arch/s390/kvm/gaccess.c
215
wake_up(&kvm->arch.ipte_wq);
arch/s390/kvm/gaccess.c
218
void ipte_lock(struct kvm *kvm)
arch/s390/kvm/gaccess.c
221
ipte_lock_siif(kvm);
arch/s390/kvm/gaccess.c
223
ipte_lock_simple(kvm);
arch/s390/kvm/gaccess.c
226
void ipte_unlock(struct kvm *kvm)
arch/s390/kvm/gaccess.c
229
ipte_unlock_siif(kvm);
arch/s390/kvm/gaccess.c
231
ipte_unlock_simple(kvm);
arch/s390/kvm/gaccess.c
444
static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val)
arch/s390/kvm/gaccess.c
446
return kvm_read_guest(kvm, gpa, val, sizeof(*val));
arch/s390/kvm/gaccess.c
484
edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
arch/s390/kvm/gaccess.c
485
edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
arch/s390/kvm/gaccess.c
486
iep = ctlreg0.iep && test_kvm_facility(vcpu->kvm, 130);
arch/s390/kvm/gaccess.c
522
if (!kvm_is_gpa_in_memslot(vcpu->kvm, ptr))
arch/s390/kvm/gaccess.c
524
if (deref_table(vcpu->kvm, ptr, &rfte.val))
arch/s390/kvm/gaccess.c
540
if (!kvm_is_gpa_in_memslot(vcpu->kvm, ptr))
arch/s390/kvm/gaccess.c
542
if (deref_table(vcpu->kvm, ptr, &rste.val))
arch/s390/kvm/gaccess.c
558
if (!kvm_is_gpa_in_memslot(vcpu->kvm, ptr))
arch/s390/kvm/gaccess.c
560
if (deref_table(vcpu->kvm, ptr, &rtte.val))
arch/s390/kvm/gaccess.c
586
if (!kvm_is_gpa_in_memslot(vcpu->kvm, ptr))
arch/s390/kvm/gaccess.c
588
if (deref_table(vcpu->kvm, ptr, &ste.val))
arch/s390/kvm/gaccess.c
606
if (!kvm_is_gpa_in_memslot(vcpu->kvm, ptr))
arch/s390/kvm/gaccess.c
608
if (deref_table(vcpu->kvm, ptr, &pte.val))
arch/s390/kvm/gaccess.c
628
if (!kvm_is_gpa_in_memslot(vcpu->kvm, raddr.addr))
arch/s390/kvm/gaccess.c
653
static int vm_check_access_key_gpa(struct kvm *kvm, u8 access_key,
arch/s390/kvm/gaccess.c
659
scoped_guard(read_lock, &kvm->mmu_lock)
arch/s390/kvm/gaccess.c
660
r = dat_get_storage_key(kvm->arch.gmap->asce, gpa_to_gfn(gpa), &storage_key);
arch/s390/kvm/gaccess.c
718
scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
arch/s390/kvm/gaccess.c
799
if (!kvm_is_gpa_in_memslot(vcpu->kvm, gpa)) {
arch/s390/kvm/gaccess.c
818
static int access_guest_page_gpa(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa,
arch/s390/kvm/gaccess.c
825
if (!gfn_to_memslot(kvm, gfn))
arch/s390/kvm/gaccess.c
828
rc = kvm_write_guest_page(kvm, gfn, data, offset, len);
arch/s390/kvm/gaccess.c
830
rc = kvm_read_guest_page(kvm, gfn, data, offset, len);
arch/s390/kvm/gaccess.c
882
static int access_guest_page_with_key_gpa(struct kvm *kvm, enum gacc_mode mode, gpa_t gpa,
arch/s390/kvm/gaccess.c
900
if (KVM_BUG_ON((len + context.offset) > PAGE_SIZE, kvm))
arch/s390/kvm/gaccess.c
903
rc = kvm_s390_faultin_gfn(NULL, kvm, &fault);
arch/s390/kvm/gaccess.c
909
int access_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, void *data,
arch/s390/kvm/gaccess.c
918
rc = access_guest_page_with_key_gpa(kvm, mode, gpa, data, fragment_len, access_key);
arch/s390/kvm/gaccess.c
961
ipte_lock(vcpu->kvm);
arch/s390/kvm/gaccess.c
978
rc = access_guest_page_gpa(vcpu->kvm, mode, gpas[idx], data, fragment_len);
arch/s390/kvm/gaccess.c
980
rc = access_guest_page_with_key_gpa(vcpu->kvm, mode, gpas[idx],
arch/s390/kvm/gaccess.c
984
rc = access_guest_page_with_key_gpa(vcpu->kvm, mode, gpas[idx],
arch/s390/kvm/gaccess.h
128
kvm_write_guest(__vcpu->kvm, __gpa, &__x, sizeof(__x)); \
arch/s390/kvm/gaccess.h
154
return kvm_write_guest(vcpu->kvm, gpa, data, len);
arch/s390/kvm/gaccess.h
180
return kvm_read_guest(vcpu->kvm, gpa, data, len);
arch/s390/kvm/gaccess.h
196
int check_gpa_range(struct kvm *kvm, unsigned long gpa, unsigned long length,
arch/s390/kvm/gaccess.h
199
int access_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, void *data,
arch/s390/kvm/gaccess.h
209
int cmpxchg_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, int len, union kvm_s390_quad *old,
arch/s390/kvm/gaccess.h
374
return kvm_write_guest(vcpu->kvm, gpa, data, len);
arch/s390/kvm/gaccess.h
397
return kvm_read_guest(vcpu->kvm, gpa, data, len);
arch/s390/kvm/gaccess.h
448
void ipte_lock(struct kvm *kvm);
arch/s390/kvm/gaccess.h
449
void ipte_unlock(struct kvm *kvm);
arch/s390/kvm/gaccess.h
450
int ipte_lock_held(struct kvm *kvm);
arch/s390/kvm/gmap.c
1017
scoped_guard(read_lock, &gmap->kvm->mmu_lock)
arch/s390/kvm/gmap.c
1035
KVM_BUG_ON(!is_shadow(sg), sg->kvm);
arch/s390/kvm/gmap.c
1073
if (KVM_BUG_ON(!is_shadow(sg) || level <= TABLE_TYPE_PAGE_TABLE, sg->kvm))
arch/s390/kvm/gmap.c
113
if (kvm_is_ucontrol(parent->kvm))
arch/s390/kvm/gmap.c
1137
scoped_guard(read_lock, &gmap->kvm->mmu_lock)
arch/s390/kvm/gmap.c
1140
&gmap->kvm->arch.cmma_dirty_pages);
arch/s390/kvm/gmap.c
1178
KVM_BUG_ON(!is_shadow(sg), sg->kvm);
arch/s390/kvm/gmap.c
1179
KVM_BUG_ON(!sg->parent, sg->kvm);
arch/s390/kvm/gmap.c
123
res = gmap_new(parent->kvm, limit);
arch/s390/kvm/gmap.c
1253
guard(write_lock)(&sg->kvm->mmu_lock);
arch/s390/kvm/gmap.c
1255
if (kvm_s390_array_needs_retry_safe(sg->kvm, context->seq, context->f))
arch/s390/kvm/gmap.c
1276
kvm_s390_release_faultin_array(sg->kvm, context->f, false);
arch/s390/kvm/gmap.c
1285
if (kvm_s390_array_needs_retry_unsafe(sg->kvm, context->seq, context->f))
arch/s390/kvm/gmap.c
1307
KVM_BUG_ON(!is_shadow(sg), sg->kvm);
arch/s390/kvm/gmap.c
1309
context.seq = sg->kvm->mmu_invalidate_seq;
arch/s390/kvm/gmap.c
1313
rc = kvm_s390_get_guest_pages(sg->kvm, context.f, asce.rsto, asce.tl + 1, false);
arch/s390/kvm/gmap.c
1319
kvm_s390_release_faultin_array(sg->kvm, context.f, true);
arch/s390/kvm/gmap.c
1358
new = gmap_new(parent->kvm, asce.r ? 1UL << (64 - PAGE_SHIFT) : asce_end(asce));
arch/s390/kvm/gmap.c
1377
if (write_trylock(&parent->kvm->mmu_lock)) {
arch/s390/kvm/gmap.c
1379
write_unlock(&parent->kvm->mmu_lock);
arch/s390/kvm/gmap.c
146
scoped_guard(write_lock, &gmap->kvm->mmu_lock)
arch/s390/kvm/gmap.c
182
if (KVM_BUG_ON(!child->parent, child->kvm))
arch/s390/kvm/gmap.c
198
KVM_BUG_ON(gmap->parent, gmap->kvm);
arch/s390/kvm/gmap.c
200
KVM_BUG_ON(!list_empty(&gmap->children), gmap->kvm);
arch/s390/kvm/gmap.c
202
KVM_BUG_ON(!list_empty(&gmap->scb_users), gmap->kvm);
arch/s390/kvm/gmap.c
204
KVM_BUG_ON(!gmap->asce.val, gmap->kvm);
arch/s390/kvm/gmap.c
206
KVM_BUG_ON(refcount_read(&gmap->refcount), gmap->kvm);
arch/s390/kvm/gmap.c
261
struct kvm *kvm = gmap->kvm;
arch/s390/kvm/gmap.c
268
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/s390/kvm/gmap.c
386
gmap_helper_try_set_pte_unused(priv->gmap->kvm->mm, vmaddr);
arch/s390/kvm/gmap.c
420
KVM_BUG_ON(!old.s.fc1.vsie_notif, priv->gmap->kvm);
arch/s390/kvm/gmap.c
423
KVM_BUG_ON(!ok, priv->gmap->kvm);
arch/s390/kvm/gmap.c
453
lockdep_assert_held_write(&gmap->kvm->mmu_lock);
arch/s390/kvm/gmap.c
477
mark_page_dirty(gmap->kvm, gfn);
arch/s390/kvm/gmap.c
52
struct gmap *gmap_new(struct kvm *kvm, gfn_t limit)
arch/s390/kvm/gmap.c
522
mark_page_dirty(gmap->kvm, gfn);
arch/s390/kvm/gmap.c
535
lockdep_assert_held(&gmap->kvm->mmu_lock);
arch/s390/kvm/gmap.c
617
lockdep_assert_held(&gmap->kvm->mmu_lock);
arch/s390/kvm/gmap.c
697
if (KVM_BUG_ON(rc == -EINVAL, gmap->kvm))
arch/s390/kvm/gmap.c
701
if (KVM_BUG_ON(get_level(f->crstep, f->ptep) > level, gmap->kvm))
arch/s390/kvm/gmap.c
743
lockdep_assert_held(&gmap->kvm->mmu_lock);
arch/s390/kvm/gmap.c
83
gmap->kvm = kvm;
arch/s390/kvm/gmap.c
832
scoped_guard(read_lock, &gmap->kvm->mmu_lock) {
arch/s390/kvm/gmap.c
838
scoped_guard(write_lock, &gmap->kvm->mmu_lock) {
arch/s390/kvm/gmap.c
869
scoped_guard(write_lock, &gmap->kvm->mmu_lock)
arch/s390/kvm/gmap.c
902
guard(read_lock)(&gmap->kvm->mmu_lock);
arch/s390/kvm/gmap.c
91
KVM_BUG_ON(is_ucontrol(parent) && parent->parent, parent->kvm);
arch/s390/kvm/gmap.c
92
KVM_BUG_ON(is_ucontrol(parent) && !owns_page_tables(parent), parent->kvm);
arch/s390/kvm/gmap.c
93
KVM_BUG_ON(!refcount_read(&child->refcount), parent->kvm);
arch/s390/kvm/gmap.c
941
scoped_guard(read_lock, &gmap->kvm->mmu_lock)
arch/s390/kvm/gmap.c
964
scoped_guard(write_lock, &gmap->kvm->mmu_lock)
arch/s390/kvm/gmap.c
975
mmap_write_lock(gmap->kvm->mm);
arch/s390/kvm/gmap.c
977
mmap_write_unlock(gmap->kvm->mm);
arch/s390/kvm/gmap.h
214
lockdep_assert_held(&gmap->kvm->mmu_lock);
arch/s390/kvm/gmap.h
274
if (KVM_BUG_ON(crstep->h.tt != oldcrste.h.tt || newcrste.h.tt != oldcrste.h.tt, gmap->kvm))
arch/s390/kvm/gmap.h
277
lockdep_assert_held(&gmap->kvm->mmu_lock);
arch/s390/kvm/gmap.h
64
struct kvm *kvm;
arch/s390/kvm/gmap.h
90
struct gmap *gmap_new(struct kvm *kvm, gfn_t limit);
arch/s390/kvm/intercept.c
418
if (!test_kvm_facility(vcpu->kvm, 74))
arch/s390/kvm/intercept.c
481
if (vcpu->kvm->arch.user_operexec)
arch/s390/kvm/intercept.c
484
if (vcpu->arch.sie_block->ipa == 0 && vcpu->kvm->arch.user_instr0)
arch/s390/kvm/intercept.c
519
struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
arch/s390/kvm/intercept.c
548
.guest_handle = kvm_s390_pv_get_handle(vcpu->kvm),
arch/s390/kvm/intercept.c
558
rc = kvm_s390_pv_make_secure(vcpu->kvm, uvcb.gaddr, &uvcb);
arch/s390/kvm/intercept.c
617
if (kvm_is_ucontrol(vcpu->kvm))
arch/s390/kvm/intercept.c
666
kvm_s390_pv_convert_to_secure(vcpu->kvm, kvm_s390_get_prefix(vcpu));
arch/s390/kvm/intercept.c
667
kvm_s390_pv_convert_to_secure(vcpu->kvm, kvm_s390_get_prefix(vcpu) + PAGE_SIZE);
arch/s390/kvm/intercept.c
87
if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
arch/s390/kvm/intercept.c
99
current->pid, vcpu->kvm);
arch/s390/kvm/interrupt.c
1000
struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
arch/s390/kvm/interrupt.c
1043
struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
arch/s390/kvm/interrupt.c
1119
struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
1126
fi = &vcpu->kvm->arch.float_int;
arch/s390/kvm/interrupt.c
1221
const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
arch/s390/kvm/interrupt.c
1253
struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
1288
vcpu->kvm->arch.float_int.last_sleep_cpu = vcpu->vcpu_idx;
arch/s390/kvm/interrupt.c
131
const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
arch/s390/kvm/interrupt.c
1511
if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL)
arch/s390/kvm/interrupt.c
1594
if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL)
arch/s390/kvm/interrupt.c
1663
static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
arch/s390/kvm/interrupt.c
1666
struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
1690
static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
arch/s390/kvm/interrupt.c
1698
inti = get_io_int(kvm, isc, schid);
arch/s390/kvm/interrupt.c
1703
static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
arch/s390/kvm/interrupt.c
1705
struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
1737
struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
arch/s390/kvm/interrupt.c
1740
struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
1744
inti = get_top_io_int(kvm, isc_mask, schid);
arch/s390/kvm/interrupt.c
1746
isc = get_top_gisa_isc(kvm, isc_mask, schid);
arch/s390/kvm/interrupt.c
1767
kvm_s390_reinject_io_int(kvm, inti);
arch/s390/kvm/interrupt.c
1775
static int __inject_service(struct kvm *kvm,
arch/s390/kvm/interrupt.c
1778
struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
1781
kvm->stat.inject_service_signal++;
arch/s390/kvm/interrupt.c
1807
static int __inject_virtio(struct kvm *kvm,
arch/s390/kvm/interrupt.c
1810
struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
1813
kvm->stat.inject_virtio++;
arch/s390/kvm/interrupt.c
1826
static int __inject_pfault_done(struct kvm *kvm,
arch/s390/kvm/interrupt.c
1829
struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
1832
kvm->stat.inject_pfault_done++;
arch/s390/kvm/interrupt.c
1847
static int __inject_float_mchk(struct kvm *kvm,
arch/s390/kvm/interrupt.c
1850
struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
1853
kvm->stat.inject_float_mchk++;
arch/s390/kvm/interrupt.c
1863
static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
arch/s390/kvm/interrupt.c
1865
struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
1871
kvm->stat.inject_io++;
arch/s390/kvm/interrupt.c
1881
VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
arch/s390/kvm/interrupt.c
1887
fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
1896
VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)");
arch/s390/kvm/interrupt.c
1898
VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x",
arch/s390/kvm/interrupt.c
1912
static void __floating_irq_kick(struct kvm *kvm, u64 type)
arch/s390/kvm/interrupt.c
1917
online_vcpus = atomic_read(&kvm->online_vcpus);
arch/s390/kvm/interrupt.c
1921
for (sigcpu = kvm->arch.float_int.last_sleep_cpu; ; sigcpu++) {
arch/s390/kvm/interrupt.c
1923
dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
arch/s390/kvm/interrupt.c
1938
kvm->arch.gisa_int.origin) ||
arch/s390/kvm/interrupt.c
1949
static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
arch/s390/kvm/interrupt.c
1956
rc = __inject_float_mchk(kvm, inti);
arch/s390/kvm/interrupt.c
1959
rc = __inject_virtio(kvm, inti);
arch/s390/kvm/interrupt.c
1962
rc = __inject_service(kvm, inti);
arch/s390/kvm/interrupt.c
1965
rc = __inject_pfault_done(kvm, inti);
arch/s390/kvm/interrupt.c
1968
rc = __inject_io(kvm, inti);
arch/s390/kvm/interrupt.c
1976
__floating_irq_kick(kvm, type);
arch/s390/kvm/interrupt.c
1980
int kvm_s390_inject_vm(struct kvm *kvm,
arch/s390/kvm/interrupt.c
1988
VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
arch/s390/kvm/interrupt.c
1994
VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm);
arch/s390/kvm/interrupt.c
2001
VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx",
arch/s390/kvm/interrupt.c
2018
rc = __inject_vm(kvm, inti);
arch/s390/kvm/interrupt.c
2023
int kvm_s390_reinject_io_int(struct kvm *kvm,
arch/s390/kvm/interrupt.c
2026
return __inject_vm(kvm, inti);
arch/s390/kvm/interrupt.c
2180
void kvm_s390_clear_float_irqs(struct kvm *kvm)
arch/s390/kvm/interrupt.c
2182
struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
2186
mutex_lock(&kvm->lock);
arch/s390/kvm/interrupt.c
2187
if (!kvm_s390_pv_is_protected(kvm))
arch/s390/kvm/interrupt.c
2189
mutex_unlock(&kvm->lock);
arch/s390/kvm/interrupt.c
2199
kvm_s390_gisa_clear(kvm);
arch/s390/kvm/interrupt.c
2202
static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
arch/s390/kvm/interrupt.c
2204
struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
2244
fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
2293
static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/interrupt.c
2295
struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
2302
if (!test_kvm_facility(kvm, 72))
arch/s390/kvm/interrupt.c
2322
r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
arch/s390/kvm/interrupt.c
2326
r = flic_ais_mode_get_all(dev->kvm, attr);
arch/s390/kvm/interrupt.c
2397
r = __inject_vm(dev->kvm, inti);
arch/s390/kvm/interrupt.c
2409
static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id)
arch/s390/kvm/interrupt.c
2414
return kvm->arch.adapters[id];
arch/s390/kvm/interrupt.c
2424
mutex_lock(&dev->kvm->lock);
arch/s390/kvm/interrupt.c
2437
if (dev->kvm->arch.adapters[adapter_info.id] != NULL) {
arch/s390/kvm/interrupt.c
2457
dev->kvm->arch.adapters[adapter->id] = adapter;
arch/s390/kvm/interrupt.c
2460
mutex_unlock(&dev->kvm->lock);
arch/s390/kvm/interrupt.c
2464
int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked)
arch/s390/kvm/interrupt.c
2467
struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
arch/s390/kvm/interrupt.c
2476
static struct page *pin_map_page(struct kvm *kvm, u64 uaddr,
arch/s390/kvm/interrupt.c
2479
struct mm_struct *mm = kvm->mm;
arch/s390/kvm/interrupt.c
2495
static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr)
arch/s390/kvm/interrupt.c
2497
struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
arch/s390/kvm/interrupt.c
2511
idx = srcu_read_lock(&kvm->srcu);
arch/s390/kvm/interrupt.c
2512
host_addr = gpa_to_hva(kvm, addr);
arch/s390/kvm/interrupt.c
2514
srcu_read_unlock(&kvm->srcu, idx);
arch/s390/kvm/interrupt.c
2518
srcu_read_unlock(&kvm->srcu, idx);
arch/s390/kvm/interrupt.c
2521
map->page = pin_map_page(kvm, host_addr, FOLL_LONGTERM);
arch/s390/kvm/interrupt.c
2528
map->page = pin_map_page(kvm, host_addr, 0);
arch/s390/kvm/interrupt.c
2557
static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr)
arch/s390/kvm/interrupt.c
2559
struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
arch/s390/kvm/interrupt.c
2593
idx = srcu_read_lock(&kvm->srcu);
arch/s390/kvm/interrupt.c
2594
mark_page_dirty(kvm, map_addr_to_mark >> PAGE_SHIFT);
arch/s390/kvm/interrupt.c
2596
srcu_read_unlock(&kvm->srcu, idx);
arch/s390/kvm/interrupt.c
2604
void kvm_s390_unmap_all_adapters(struct kvm *kvm)
arch/s390/kvm/interrupt.c
2611
struct s390_io_adapter *adapter = kvm->arch.adapters[i];
arch/s390/kvm/interrupt.c
2625
idx = srcu_read_lock(&kvm->srcu);
arch/s390/kvm/interrupt.c
2626
mark_page_dirty(kvm, map->guest_addr >> PAGE_SHIFT);
arch/s390/kvm/interrupt.c
2628
srcu_read_unlock(&kvm->srcu, idx);
arch/s390/kvm/interrupt.c
2636
void kvm_s390_destroy_adapters(struct kvm *kvm)
arch/s390/kvm/interrupt.c
2640
kvm_s390_unmap_all_adapters(kvm);
arch/s390/kvm/interrupt.c
2643
kfree(kvm->arch.adapters[i]);
arch/s390/kvm/interrupt.c
2644
kvm->arch.adapters[i] = NULL;
arch/s390/kvm/interrupt.c
2658
adapter = get_io_adapter(dev->kvm, req.id);
arch/s390/kvm/interrupt.c
2663
ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask);
arch/s390/kvm/interrupt.c
2670
mutex_lock(&dev->kvm->lock);
arch/s390/kvm/interrupt.c
2671
if (kvm_s390_pv_is_protected(dev->kvm)) {
arch/s390/kvm/interrupt.c
2672
mutex_unlock(&dev->kvm->lock);
arch/s390/kvm/interrupt.c
2676
dev->kvm->stat.io_390_adapter_map++;
arch/s390/kvm/interrupt.c
2677
ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr);
arch/s390/kvm/interrupt.c
2679
dev->kvm->stat.io_390_adapter_unmap++;
arch/s390/kvm/interrupt.c
2680
ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr);
arch/s390/kvm/interrupt.c
2682
mutex_unlock(&dev->kvm->lock);
arch/s390/kvm/interrupt.c
269
unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs |
arch/s390/kvm/interrupt.c
2691
static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/interrupt.c
2705
kfree(kvm_s390_get_io_int(kvm, isc_mask, schid));
arch/s390/kvm/interrupt.c
2714
static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/interrupt.c
2716
struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
272
pending &= ~vcpu->kvm->arch.float_int.masked_irqs;
arch/s390/kvm/interrupt.c
2721
if (!test_kvm_facility(kvm, 72))
arch/s390/kvm/interrupt.c
2754
static int kvm_s390_inject_airq(struct kvm *kvm,
arch/s390/kvm/interrupt.c
2757
struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
2772
if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) {
arch/s390/kvm/interrupt.c
2773
ret = kvm_s390_inject_vm(kvm, &s390int, inti);
arch/s390/kvm/interrupt.c
278
struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
2787
ret = kvm_s390_inject_vm(kvm, &s390int, inti);
arch/s390/kvm/interrupt.c
2801
static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/interrupt.c
2804
struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
arch/s390/kvm/interrupt.c
2806
kvm->stat.io_flic_inject_airq++;
arch/s390/kvm/interrupt.c
2811
return kvm_s390_inject_airq(kvm, adapter);
arch/s390/kvm/interrupt.c
2814
static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/interrupt.c
2816
struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
2820
if (!test_kvm_facility(kvm, 72))
arch/s390/kvm/interrupt.c
2845
kvm_s390_clear_float_irqs(dev->kvm);
arch/s390/kvm/interrupt.c
2848
if (kvm_is_ucontrol(dev->kvm))
arch/s390/kvm/interrupt.c
2850
set_bit(GMAP_FLAG_PFAULT_ENABLED, &dev->kvm->arch.gmap->flags);
arch/s390/kvm/interrupt.c
2853
if (kvm_is_ucontrol(dev->kvm))
arch/s390/kvm/interrupt.c
2855
clear_bit(GMAP_FLAG_PFAULT_ENABLED, &dev->kvm->arch.gmap->flags);
arch/s390/kvm/interrupt.c
2861
synchronize_srcu(&dev->kvm->srcu);
arch/s390/kvm/interrupt.c
2862
kvm_for_each_vcpu(i, vcpu, dev->kvm)
arch/s390/kvm/interrupt.c
2872
r = clear_io_irq(dev->kvm, attr);
arch/s390/kvm/interrupt.c
2875
r = modify_ais_mode(dev->kvm, attr);
arch/s390/kvm/interrupt.c
2878
r = flic_inject_airq(dev->kvm, attr);
arch/s390/kvm/interrupt.c
2881
r = flic_ais_mode_set_all(dev->kvm, attr);
arch/s390/kvm/interrupt.c
2914
if (dev->kvm->arch.flic)
arch/s390/kvm/interrupt.c
2916
dev->kvm->arch.flic = dev;
arch/s390/kvm/interrupt.c
2922
dev->kvm->arch.flic = NULL;
arch/s390/kvm/interrupt.c
2966
static int adapter_indicators_set(struct kvm *kvm,
arch/s390/kvm/interrupt.c
2983
ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0);
arch/s390/kvm/interrupt.c
2986
idx = srcu_read_lock(&kvm->srcu);
arch/s390/kvm/interrupt.c
2991
mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT);
arch/s390/kvm/interrupt.c
2993
srcu_read_unlock(&kvm->srcu, idx);
arch/s390/kvm/interrupt.c
3006
summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0);
arch/s390/kvm/interrupt.c
3009
idx = srcu_read_lock(&kvm->srcu);
arch/s390/kvm/interrupt.c
3014
mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT);
arch/s390/kvm/interrupt.c
3016
srcu_read_unlock(&kvm->srcu, idx);
arch/s390/kvm/interrupt.c
3029
static int adapter_indicators_set_fast(struct kvm *kvm,
arch/s390/kvm/interrupt.c
3073
struct kvm *kvm, int irq_source_id, int level,
arch/s390/kvm/interrupt.c
3079
kvm->stat.io_set_adapter_int++;
arch/s390/kvm/interrupt.c
3084
adapter = get_io_adapter(kvm, e->adapter.adapter_id);
arch/s390/kvm/interrupt.c
3087
ret = adapter_indicators_set(kvm, adapter, &e->adapter);
arch/s390/kvm/interrupt.c
3089
ret = kvm_s390_inject_airq(kvm, adapter);
arch/s390/kvm/interrupt.c
3131
rc = __inject_vm(vcpu->kvm, &inti);
arch/s390/kvm/interrupt.c
3140
int kvm_set_routing_entry(struct kvm *kvm,
arch/s390/kvm/interrupt.c
3150
if (kvm_is_ucontrol(kvm))
arch/s390/kvm/interrupt.c
3162
idx = srcu_read_lock(&kvm->srcu);
arch/s390/kvm/interrupt.c
3163
uaddr_s = gpa_to_hva(kvm, ue->u.adapter.summary_addr);
arch/s390/kvm/interrupt.c
3164
uaddr_i = gpa_to_hva(kvm, ue->u.adapter.ind_addr);
arch/s390/kvm/interrupt.c
3165
srcu_read_unlock(&kvm->srcu, idx);
arch/s390/kvm/interrupt.c
3182
int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
arch/s390/kvm/interrupt.c
3326
static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask)
arch/s390/kvm/interrupt.c
3328
int vcpu_idx, online_vcpus = atomic_read(&kvm->online_vcpus);
arch/s390/kvm/interrupt.c
3329
struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
3333
for_each_set_bit(vcpu_idx, kvm->arch.idle_mask, online_vcpus) {
arch/s390/kvm/interrupt.c
3334
vcpu = kvm_get_vcpu(kvm, vcpu_idx);
arch/s390/kvm/interrupt.c
3352
struct kvm *kvm =
arch/s390/kvm/interrupt.c
3353
container_of(gi->origin, struct sie_page2, gisa)->kvm;
arch/s390/kvm/interrupt.c
3358
__airqs_kick_single_vcpu(kvm, pending_mask);
arch/s390/kvm/interrupt.c
3375
struct kvm *kvm;
arch/s390/kvm/interrupt.c
3405
kvm = container_of(gisa, struct sie_page2, gisa)->kvm;
arch/s390/kvm/interrupt.c
3406
gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
3415
void kvm_s390_gisa_clear(struct kvm *kvm)
arch/s390/kvm/interrupt.c
3417
struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
3422
VM_EVENT(kvm, 3, "gisa 0x%p cleared", gi->origin);
arch/s390/kvm/interrupt.c
3425
void kvm_s390_gisa_init(struct kvm *kvm)
arch/s390/kvm/interrupt.c
3427
struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
3431
gi->origin = &kvm->arch.sie_page2->gisa;
arch/s390/kvm/interrupt.c
3438
VM_EVENT(kvm, 3, "gisa 0x%p initialized", gi->origin);
arch/s390/kvm/interrupt.c
3441
void kvm_s390_gisa_enable(struct kvm *kvm)
arch/s390/kvm/interrupt.c
3443
struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
3450
kvm_s390_gisa_init(kvm);
arch/s390/kvm/interrupt.c
3451
gisa_desc = kvm_s390_get_gisa_desc(kvm);
arch/s390/kvm/interrupt.c
3454
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/s390/kvm/interrupt.c
3464
void kvm_s390_gisa_destroy(struct kvm *kvm)
arch/s390/kvm/interrupt.c
3466
struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
3479
VM_EVENT(kvm, 3, "gisa 0x%p destroyed", gisa);
arch/s390/kvm/interrupt.c
348
(vcpu->kvm->arch.float_int.mchk.cr14 |
arch/s390/kvm/interrupt.c
3482
void kvm_s390_gisa_disable(struct kvm *kvm)
arch/s390/kvm/interrupt.c
3484
struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
3490
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/s390/kvm/interrupt.c
3497
kvm_s390_gisa_destroy(kvm);
arch/s390/kvm/interrupt.c
3517
int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc)
arch/s390/kvm/interrupt.c
3519
struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
3556
int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc)
arch/s390/kvm/interrupt.c
3558
struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
3587
struct kvm *kvm;
arch/s390/kvm/interrupt.c
3595
kvm = kvm_s390_pci_si_to_kvm(aift, si);
arch/s390/kvm/interrupt.c
3596
if (!kvm)
arch/s390/kvm/interrupt.c
3598
gi = &kvm->arch.gisa_int;
arch/s390/kvm/interrupt.c
3606
kvm->stat.aen_forward++;
arch/s390/kvm/interrupt.c
364
set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
arch/s390/kvm/interrupt.c
370
clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
arch/s390/kvm/interrupt.c
3743
struct kvm *kvm, int irq_source_id, int level,
arch/s390/kvm/interrupt.c
3748
struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
arch/s390/kvm/interrupt.c
3755
kvm->stat.io_390_inatomic++;
arch/s390/kvm/interrupt.c
3763
adapter = get_io_adapter(kvm, e->adapter.adapter_id);
arch/s390/kvm/interrupt.c
3769
ret = adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit);
arch/s390/kvm/interrupt.c
3773
kvm->stat.io_390_inatomic_no_inject++;
arch/s390/kvm/interrupt.c
3780
adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit);
arch/s390/kvm/interrupt.c
3784
if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) {
arch/s390/kvm/interrupt.c
3785
ret = kvm_s390_inject_vm(kvm, &s390int, inti);
arch/s390/kvm/interrupt.c
3790
adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit);
arch/s390/kvm/interrupt.c
3801
kvm->stat.io_390_inatomic_no_inject++;
arch/s390/kvm/interrupt.c
3805
ret = kvm_s390_inject_vm(kvm, &s390int, inti);
arch/s390/kvm/interrupt.c
3813
adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit);
arch/s390/kvm/interrupt.c
47
struct esca_block *sca = vcpu->kvm->arch.sca;
arch/s390/kvm/interrupt.c
543
if (test_kvm_facility(vcpu->kvm, 133)) {
arch/s390/kvm/interrupt.c
563
if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) {
arch/s390/kvm/interrupt.c
570
if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133)
arch/s390/kvm/interrupt.c
622
struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
arch/s390/kvm/interrupt.c
63
struct esca_block *sca = vcpu->kvm->arch.sca;
arch/s390/kvm/interrupt.c
86
struct esca_block *sca = vcpu->kvm->arch.sca;
arch/s390/kvm/interrupt.c
943
struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
arch/s390/kvm/interrupt.c
975
struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
arch/s390/kvm/kvm-s390.c
1003
VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
arch/s390/kvm/kvm-s390.c
1004
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1005
if (kvm->created_vcpus)
arch/s390/kvm/kvm-s390.c
1008
kvm->arch.use_cmma = 1;
arch/s390/kvm/kvm-s390.c
1010
kvm->arch.use_pfmfi = 0;
arch/s390/kvm/kvm-s390.c
1013
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1022
if (!kvm->arch.use_cmma)
arch/s390/kvm/kvm-s390.c
1025
guard(mutex)(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1026
VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
arch/s390/kvm/kvm-s390.c
1028
scoped_guard(read_lock, &kvm->mmu_lock)
arch/s390/kvm/kvm-s390.c
1029
start_gfn = dat_reset_cmma(kvm->arch.gmap->asce, start_gfn);
arch/s390/kvm/kvm-s390.c
1041
if (kvm_is_ucontrol(kvm))
arch/s390/kvm/kvm-s390.c
1047
guard(mutex)(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1050
if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
arch/s390/kvm/kvm-s390.c
1051
new_limit > kvm->arch.mem_limit)
arch/s390/kvm/kvm-s390.c
1057
if (kvm->created_vcpus)
arch/s390/kvm/kvm-s390.c
1061
scoped_guard(mutex, &kvm->slots_lock) {
arch/s390/kvm/kvm-s390.c
1062
slots = kvm_memslots(kvm);
arch/s390/kvm/kvm-s390.c
1072
ret = gmap_set_limit(kvm->arch.gmap, gpa_to_gfn(new_limit));
arch/s390/kvm/kvm-s390.c
1076
VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
arch/s390/kvm/kvm-s390.c
1077
VM_EVENT(kvm, 3, "New guest asce: 0x%p", (void *)kvm->arch.gmap->asce.val);
arch/s390/kvm/kvm-s390.c
1089
void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
1094
kvm_s390_vcpu_block_all(kvm);
arch/s390/kvm/kvm-s390.c
1096
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/s390/kvm/kvm-s390.c
1102
kvm_s390_vcpu_unblock_all(kvm);
arch/s390/kvm/kvm-s390.c
1105
static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1107
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1110
if (!test_kvm_facility(kvm, 76)) {
arch/s390/kvm/kvm-s390.c
1111
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1115
kvm->arch.crypto.crycb->aes_wrapping_key_mask,
arch/s390/kvm/kvm-s390.c
1116
sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
arch/s390/kvm/kvm-s390.c
1117
kvm->arch.crypto.aes_kw = 1;
arch/s390/kvm/kvm-s390.c
1118
VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
arch/s390/kvm/kvm-s390.c
1121
if (!test_kvm_facility(kvm, 76)) {
arch/s390/kvm/kvm-s390.c
1122
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1126
kvm->arch.crypto.crycb->dea_wrapping_key_mask,
arch/s390/kvm/kvm-s390.c
1127
sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
arch/s390/kvm/kvm-s390.c
1128
kvm->arch.crypto.dea_kw = 1;
arch/s390/kvm/kvm-s390.c
1129
VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
arch/s390/kvm/kvm-s390.c
1132
if (!test_kvm_facility(kvm, 76)) {
arch/s390/kvm/kvm-s390.c
1133
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1136
kvm->arch.crypto.aes_kw = 0;
arch/s390/kvm/kvm-s390.c
1137
memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
arch/s390/kvm/kvm-s390.c
1138
sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
arch/s390/kvm/kvm-s390.c
1139
VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
arch/s390/kvm/kvm-s390.c
1142
if (!test_kvm_facility(kvm, 76)) {
arch/s390/kvm/kvm-s390.c
1143
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1146
kvm->arch.crypto.dea_kw = 0;
arch/s390/kvm/kvm-s390.c
1147
memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
arch/s390/kvm/kvm-s390.c
1148
sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
arch/s390/kvm/kvm-s390.c
1149
VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
arch/s390/kvm/kvm-s390.c
1153
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1156
kvm->arch.crypto.apie = 1;
arch/s390/kvm/kvm-s390.c
1160
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1163
kvm->arch.crypto.apie = 0;
arch/s390/kvm/kvm-s390.c
1166
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1170
kvm_s390_vcpu_crypto_reset_all(kvm);
arch/s390/kvm/kvm-s390.c
1171
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1178
if (!vcpu->kvm->arch.use_zpci_interp)
arch/s390/kvm/kvm-s390.c
1185
void kvm_s390_vcpu_pci_enable_interp(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
1190
lockdep_assert_held(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1199
kvm->arch.use_zpci_interp = 1;
arch/s390/kvm/kvm-s390.c
1201
kvm_s390_vcpu_block_all(kvm);
arch/s390/kvm/kvm-s390.c
1203
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/s390/kvm/kvm-s390.c
1208
kvm_s390_vcpu_unblock_all(kvm);
arch/s390/kvm/kvm-s390.c
1211
static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
arch/s390/kvm/kvm-s390.c
1216
kvm_for_each_vcpu(cx, vcpu, kvm)
arch/s390/kvm/kvm-s390.c
1225
static int kvm_s390_vm_start_migration(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
1232
if (kvm->arch.migration_mode)
arch/s390/kvm/kvm-s390.c
1234
slots = kvm_memslots(kvm);
arch/s390/kvm/kvm-s390.c
1238
if (!kvm->arch.use_cmma) {
arch/s390/kvm/kvm-s390.c
1239
kvm->arch.migration_mode = 1;
arch/s390/kvm/kvm-s390.c
1254
WRITE_ONCE(kvm->arch.migration_mode, 1);
arch/s390/kvm/kvm-s390.c
1255
kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
arch/s390/kvm/kvm-s390.c
1263
gmap_set_cmma_all_dirty(kvm->arch.gmap);
arch/s390/kvm/kvm-s390.c
1271
static int kvm_s390_vm_stop_migration(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
1274
if (!kvm->arch.migration_mode)
arch/s390/kvm/kvm-s390.c
1282
WRITE_ONCE(kvm->arch.migration_mode, 0);
arch/s390/kvm/kvm-s390.c
1283
if (!kvm->arch.use_cmma)
arch/s390/kvm/kvm-s390.c
1286
kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
arch/s390/kvm/kvm-s390.c
1288
gmap_set_cmma_all_clean(kvm->arch.gmap);
arch/s390/kvm/kvm-s390.c
1289
atomic64_set(&kvm->arch.cmma_dirty_pages, 0);
arch/s390/kvm/kvm-s390.c
1298
static int kvm_s390_vm_set_migration(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
1303
mutex_lock(&kvm->slots_lock);
arch/s390/kvm/kvm-s390.c
1306
res = kvm_s390_vm_start_migration(kvm);
arch/s390/kvm/kvm-s390.c
1309
res = kvm_s390_vm_stop_migration(kvm);
arch/s390/kvm/kvm-s390.c
1314
mutex_unlock(&kvm->slots_lock);
arch/s390/kvm/kvm-s390.c
1319
static int kvm_s390_vm_get_migration(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
1322
u64 mig = kvm->arch.migration_mode;
arch/s390/kvm/kvm-s390.c
1332
static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod);
arch/s390/kvm/kvm-s390.c
1334
static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1341
if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
arch/s390/kvm/kvm-s390.c
1343
__kvm_s390_set_tod_clock(kvm, &gtod);
arch/s390/kvm/kvm-s390.c
1345
VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
arch/s390/kvm/kvm-s390.c
1351
static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1361
VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
arch/s390/kvm/kvm-s390.c
1366
static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1374
__kvm_s390_set_tod_clock(kvm, &gtod);
arch/s390/kvm/kvm-s390.c
1375
VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
arch/s390/kvm/kvm-s390.c
1379
static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1386
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1391
if (kvm_s390_pv_is_protected(kvm)) {
arch/s390/kvm/kvm-s390.c
1398
ret = kvm_s390_set_tod_ext(kvm, attr);
arch/s390/kvm/kvm-s390.c
1401
ret = kvm_s390_set_tod_high(kvm, attr);
arch/s390/kvm/kvm-s390.c
1404
ret = kvm_s390_set_tod_low(kvm, attr);
arch/s390/kvm/kvm-s390.c
1412
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1416
static void kvm_s390_get_tod_clock(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
1425
gtod->tod = clk.tod + kvm->arch.epoch;
arch/s390/kvm/kvm-s390.c
1427
if (test_kvm_facility(kvm, 139)) {
arch/s390/kvm/kvm-s390.c
1428
gtod->epoch_idx = clk.ei + kvm->arch.epdx;
arch/s390/kvm/kvm-s390.c
1436
static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1441
kvm_s390_get_tod_clock(kvm, &gtod);
arch/s390/kvm/kvm-s390.c
1445
VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
arch/s390/kvm/kvm-s390.c
1450
static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1457
VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
arch/s390/kvm/kvm-s390.c
1462
static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1466
gtod = kvm_s390_get_tod_clock_fast(kvm);
arch/s390/kvm/kvm-s390.c
1469
VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
arch/s390/kvm/kvm-s390.c
1474
static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1483
ret = kvm_s390_get_tod_ext(kvm, attr);
arch/s390/kvm/kvm-s390.c
1486
ret = kvm_s390_get_tod_high(kvm, attr);
arch/s390/kvm/kvm-s390.c
1489
ret = kvm_s390_get_tod_low(kvm, attr);
arch/s390/kvm/kvm-s390.c
1498
static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1504
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1505
if (kvm->created_vcpus) {
arch/s390/kvm/kvm-s390.c
1516
kvm->arch.model.cpuid = proc->cpuid;
arch/s390/kvm/kvm-s390.c
1521
kvm->arch.model.ibc = unblocked_ibc;
arch/s390/kvm/kvm-s390.c
1523
kvm->arch.model.ibc = lowest_ibc;
arch/s390/kvm/kvm-s390.c
1525
kvm->arch.model.ibc = proc->ibc;
arch/s390/kvm/kvm-s390.c
1527
memcpy(kvm->arch.model.fac_list, proc->fac_list,
arch/s390/kvm/kvm-s390.c
1529
VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
arch/s390/kvm/kvm-s390.c
1530
kvm->arch.model.ibc,
arch/s390/kvm/kvm-s390.c
1531
kvm->arch.model.cpuid);
arch/s390/kvm/kvm-s390.c
1532
VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
arch/s390/kvm/kvm-s390.c
1533
kvm->arch.model.fac_list[0],
arch/s390/kvm/kvm-s390.c
1534
kvm->arch.model.fac_list[1],
arch/s390/kvm/kvm-s390.c
1535
kvm->arch.model.fac_list[2]);
arch/s390/kvm/kvm-s390.c
1540
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1544
static int kvm_s390_set_processor_feat(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
1556
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1557
if (kvm->created_vcpus) {
arch/s390/kvm/kvm-s390.c
1558
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1561
bitmap_from_arr64(kvm->arch.cpu_feat, data.feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
arch/s390/kvm/kvm-s390.c
1562
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1563
VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
arch/s390/kvm/kvm-s390.c
1570
static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
1573
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1574
if (kvm->created_vcpus) {
arch/s390/kvm/kvm-s390.c
1575
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1579
if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
arch/s390/kvm/kvm-s390.c
1581
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1584
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1586
VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1587
((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
arch/s390/kvm/kvm-s390.c
1588
((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
arch/s390/kvm/kvm-s390.c
1589
((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
arch/s390/kvm/kvm-s390.c
1590
((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
arch/s390/kvm/kvm-s390.c
1591
VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1592
((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
arch/s390/kvm/kvm-s390.c
1593
((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
arch/s390/kvm/kvm-s390.c
1594
VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1595
((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
arch/s390/kvm/kvm-s390.c
1596
((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
arch/s390/kvm/kvm-s390.c
1597
VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1598
((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
arch/s390/kvm/kvm-s390.c
1599
((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
arch/s390/kvm/kvm-s390.c
1600
VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1601
((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
arch/s390/kvm/kvm-s390.c
1602
((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
arch/s390/kvm/kvm-s390.c
1603
VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1604
((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
arch/s390/kvm/kvm-s390.c
1605
((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
arch/s390/kvm/kvm-s390.c
1606
VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1607
((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
arch/s390/kvm/kvm-s390.c
1608
((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
arch/s390/kvm/kvm-s390.c
1609
VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1610
((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
arch/s390/kvm/kvm-s390.c
1611
((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
arch/s390/kvm/kvm-s390.c
1612
VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1613
((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
arch/s390/kvm/kvm-s390.c
1614
((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
arch/s390/kvm/kvm-s390.c
1615
VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1616
((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
arch/s390/kvm/kvm-s390.c
1617
((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
arch/s390/kvm/kvm-s390.c
1618
VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1619
((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
arch/s390/kvm/kvm-s390.c
1620
((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
arch/s390/kvm/kvm-s390.c
1621
VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1622
((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
arch/s390/kvm/kvm-s390.c
1623
((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
arch/s390/kvm/kvm-s390.c
1624
VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1625
((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
arch/s390/kvm/kvm-s390.c
1626
((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
arch/s390/kvm/kvm-s390.c
1627
VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1628
((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
arch/s390/kvm/kvm-s390.c
1629
((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
arch/s390/kvm/kvm-s390.c
1630
VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1631
((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
arch/s390/kvm/kvm-s390.c
1632
((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
arch/s390/kvm/kvm-s390.c
1633
VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1634
((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
arch/s390/kvm/kvm-s390.c
1635
((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
arch/s390/kvm/kvm-s390.c
1636
((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
arch/s390/kvm/kvm-s390.c
1637
((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
arch/s390/kvm/kvm-s390.c
1638
VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1639
((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
arch/s390/kvm/kvm-s390.c
1640
((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
arch/s390/kvm/kvm-s390.c
1641
((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
arch/s390/kvm/kvm-s390.c
1642
((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
arch/s390/kvm/kvm-s390.c
1643
VM_EVENT(kvm, 3, "GET: guest PFCR subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1659
static int kvm_s390_set_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1670
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1671
if (kvm->created_vcpus) {
arch/s390/kvm/kvm-s390.c
1672
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1675
kvm->arch.model.uv_feat_guest.feat = data;
arch/s390/kvm/kvm-s390.c
1676
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
1678
VM_EVENT(kvm, 3, "SET: guest UV-feat: 0x%16.16lx", data);
arch/s390/kvm/kvm-s390.c
1683
static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1689
ret = kvm_s390_set_processor(kvm, attr);
arch/s390/kvm/kvm-s390.c
1692
ret = kvm_s390_set_processor_feat(kvm, attr);
arch/s390/kvm/kvm-s390.c
1695
ret = kvm_s390_set_processor_subfunc(kvm, attr);
arch/s390/kvm/kvm-s390.c
1698
ret = kvm_s390_set_uv_feat(kvm, attr);
arch/s390/kvm/kvm-s390.c
1704
static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1714
proc->cpuid = kvm->arch.model.cpuid;
arch/s390/kvm/kvm-s390.c
1715
proc->ibc = kvm->arch.model.ibc;
arch/s390/kvm/kvm-s390.c
1716
memcpy(&proc->fac_list, kvm->arch.model.fac_list,
arch/s390/kvm/kvm-s390.c
1718
VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
arch/s390/kvm/kvm-s390.c
1719
kvm->arch.model.ibc,
arch/s390/kvm/kvm-s390.c
1720
kvm->arch.model.cpuid);
arch/s390/kvm/kvm-s390.c
1721
VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
arch/s390/kvm/kvm-s390.c
1722
kvm->arch.model.fac_list[0],
arch/s390/kvm/kvm-s390.c
1723
kvm->arch.model.fac_list[1],
arch/s390/kvm/kvm-s390.c
1724
kvm->arch.model.fac_list[2]);
arch/s390/kvm/kvm-s390.c
1732
static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1744
memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
arch/s390/kvm/kvm-s390.c
1748
VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
arch/s390/kvm/kvm-s390.c
1749
kvm->arch.model.ibc,
arch/s390/kvm/kvm-s390.c
1750
kvm->arch.model.cpuid);
arch/s390/kvm/kvm-s390.c
1751
VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
arch/s390/kvm/kvm-s390.c
1755
VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
arch/s390/kvm/kvm-s390.c
1766
static int kvm_s390_get_processor_feat(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
1771
bitmap_to_arr64(data.feat, kvm->arch.cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
arch/s390/kvm/kvm-s390.c
1774
VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
arch/s390/kvm/kvm-s390.c
1781
static int kvm_s390_get_machine_feat(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
1789
VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
arch/s390/kvm/kvm-s390.c
1796
static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
1799
if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
arch/s390/kvm/kvm-s390.c
1803
VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1804
((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
arch/s390/kvm/kvm-s390.c
1805
((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
arch/s390/kvm/kvm-s390.c
1806
((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
arch/s390/kvm/kvm-s390.c
1807
((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
arch/s390/kvm/kvm-s390.c
1808
VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1809
((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
arch/s390/kvm/kvm-s390.c
1810
((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
arch/s390/kvm/kvm-s390.c
1811
VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1812
((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
arch/s390/kvm/kvm-s390.c
1813
((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
arch/s390/kvm/kvm-s390.c
1814
VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1815
((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
arch/s390/kvm/kvm-s390.c
1816
((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
arch/s390/kvm/kvm-s390.c
1817
VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1818
((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
arch/s390/kvm/kvm-s390.c
1819
((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
arch/s390/kvm/kvm-s390.c
1820
VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1821
((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
arch/s390/kvm/kvm-s390.c
1822
((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
arch/s390/kvm/kvm-s390.c
1823
VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1824
((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
arch/s390/kvm/kvm-s390.c
1825
((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
arch/s390/kvm/kvm-s390.c
1826
VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1827
((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
arch/s390/kvm/kvm-s390.c
1828
((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
arch/s390/kvm/kvm-s390.c
1829
VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1830
((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
arch/s390/kvm/kvm-s390.c
1831
((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
arch/s390/kvm/kvm-s390.c
1832
VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1833
((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
arch/s390/kvm/kvm-s390.c
1834
((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
arch/s390/kvm/kvm-s390.c
1835
VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1836
((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
arch/s390/kvm/kvm-s390.c
1837
((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
arch/s390/kvm/kvm-s390.c
1838
VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1839
((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
arch/s390/kvm/kvm-s390.c
1840
((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
arch/s390/kvm/kvm-s390.c
1841
VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1842
((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
arch/s390/kvm/kvm-s390.c
1843
((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
arch/s390/kvm/kvm-s390.c
1844
VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1845
((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
arch/s390/kvm/kvm-s390.c
1846
((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
arch/s390/kvm/kvm-s390.c
1847
VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1848
((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
arch/s390/kvm/kvm-s390.c
1849
((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
arch/s390/kvm/kvm-s390.c
1850
VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1851
((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
arch/s390/kvm/kvm-s390.c
1852
((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
arch/s390/kvm/kvm-s390.c
1853
((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
arch/s390/kvm/kvm-s390.c
1854
((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
arch/s390/kvm/kvm-s390.c
1855
VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1856
((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
arch/s390/kvm/kvm-s390.c
1857
((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
arch/s390/kvm/kvm-s390.c
1858
((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
arch/s390/kvm/kvm-s390.c
1859
((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
arch/s390/kvm/kvm-s390.c
1860
VM_EVENT(kvm, 3, "GET: guest PFCR subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1867
static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
1874
VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1879
VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1882
VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1885
VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1888
VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1891
VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1894
VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1897
VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1900
VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1903
VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1906
VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1909
VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1912
VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1915
VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1918
VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1921
VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1926
VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1931
VM_EVENT(kvm, 3, "GET: host PFCR subfunc 0x%16.16lx.%16.16lx",
arch/s390/kvm/kvm-s390.c
1938
static int kvm_s390_get_processor_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1941
unsigned long feat = kvm->arch.model.uv_feat_guest.feat;
arch/s390/kvm/kvm-s390.c
1945
VM_EVENT(kvm, 3, "GET: guest UV-feat: 0x%16.16lx", feat);
arch/s390/kvm/kvm-s390.c
1950
static int kvm_s390_get_machine_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1960
VM_EVENT(kvm, 3, "GET: guest UV-feat: 0x%16.16lx", feat);
arch/s390/kvm/kvm-s390.c
1965
static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
1971
ret = kvm_s390_get_processor(kvm, attr);
arch/s390/kvm/kvm-s390.c
1974
ret = kvm_s390_get_machine(kvm, attr);
arch/s390/kvm/kvm-s390.c
1977
ret = kvm_s390_get_processor_feat(kvm, attr);
arch/s390/kvm/kvm-s390.c
1980
ret = kvm_s390_get_machine_feat(kvm, attr);
arch/s390/kvm/kvm-s390.c
1983
ret = kvm_s390_get_processor_subfunc(kvm, attr);
arch/s390/kvm/kvm-s390.c
1986
ret = kvm_s390_get_machine_subfunc(kvm, attr);
arch/s390/kvm/kvm-s390.c
1989
ret = kvm_s390_get_processor_uv_feat(kvm, attr);
arch/s390/kvm/kvm-s390.c
1992
ret = kvm_s390_get_machine_uv_feat(kvm, attr);
arch/s390/kvm/kvm-s390.c
2007
static void kvm_s390_update_topology_change_report(struct kvm *kvm, bool val)
arch/s390/kvm/kvm-s390.c
2012
sca = kvm->arch.sca;
arch/s390/kvm/kvm-s390.c
2020
static int kvm_s390_set_topo_change_indication(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
2023
if (!test_kvm_facility(kvm, 11))
arch/s390/kvm/kvm-s390.c
2026
kvm_s390_update_topology_change_report(kvm, !!attr->attr);
arch/s390/kvm/kvm-s390.c
2030
static int kvm_s390_get_topo_change_indication(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
2035
if (!test_kvm_facility(kvm, 11))
arch/s390/kvm/kvm-s390.c
2038
topo = kvm->arch.sca->utility.mtcr;
arch/s390/kvm/kvm-s390.c
2043
static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
2049
ret = kvm_s390_set_mem_control(kvm, attr);
arch/s390/kvm/kvm-s390.c
2052
ret = kvm_s390_set_tod(kvm, attr);
arch/s390/kvm/kvm-s390.c
2055
ret = kvm_s390_set_cpu_model(kvm, attr);
arch/s390/kvm/kvm-s390.c
2058
ret = kvm_s390_vm_set_crypto(kvm, attr);
arch/s390/kvm/kvm-s390.c
2061
ret = kvm_s390_vm_set_migration(kvm, attr);
arch/s390/kvm/kvm-s390.c
2064
ret = kvm_s390_set_topo_change_indication(kvm, attr);
arch/s390/kvm/kvm-s390.c
2074
static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
2080
ret = kvm_s390_get_mem_control(kvm, attr);
arch/s390/kvm/kvm-s390.c
2083
ret = kvm_s390_get_tod(kvm, attr);
arch/s390/kvm/kvm-s390.c
2086
ret = kvm_s390_get_cpu_model(kvm, attr);
arch/s390/kvm/kvm-s390.c
2089
ret = kvm_s390_vm_get_migration(kvm, attr);
arch/s390/kvm/kvm-s390.c
2092
ret = kvm_s390_get_topo_change_indication(kvm, attr);
arch/s390/kvm/kvm-s390.c
2102
static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
2170
ret = test_kvm_facility(kvm, 11) ? 0 : -ENXIO;
arch/s390/kvm/kvm-s390.c
2180
static int kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
arch/s390/kvm/kvm-s390.c
2189
if (!uses_skeys(kvm->arch.gmap))
arch/s390/kvm/kvm-s390.c
2200
scoped_guard(read_lock, &kvm->mmu_lock) {
arch/s390/kvm/kvm-s390.c
2202
r = dat_get_storage_key(kvm->arch.gmap->asce,
arch/s390/kvm/kvm-s390.c
2220
static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
arch/s390/kvm/kvm-s390.c
2245
r = gmap_enable_skeys(kvm->arch.gmap);
arch/s390/kvm/kvm-s390.c
2267
scoped_guard(read_lock, &kvm->mmu_lock) {
arch/s390/kvm/kvm-s390.c
2269
r = dat_set_storage_key(mc, kvm->arch.gmap->asce,
arch/s390/kvm/kvm-s390.c
2290
static int kvm_s390_get_cmma_bits(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
2296
if (!kvm->arch.use_cmma)
arch/s390/kvm/kvm-s390.c
2303
if (!peek && !kvm->arch.migration_mode)
arch/s390/kvm/kvm-s390.c
2307
if (!args->count || !uses_cmm(kvm->arch.gmap)) {
arch/s390/kvm/kvm-s390.c
2312
if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
arch/s390/kvm/kvm-s390.c
2321
scoped_guard(read_lock, &kvm->mmu_lock) {
arch/s390/kvm/kvm-s390.c
2323
ret = dat_peek_cmma(args->start_gfn, kvm->arch.gmap->asce, &args->count,
arch/s390/kvm/kvm-s390.c
2326
ret = dat_get_cmma(kvm->arch.gmap->asce, &args->start_gfn, &args->count,
arch/s390/kvm/kvm-s390.c
2327
values, &kvm->arch.cmma_dirty_pages);
arch/s390/kvm/kvm-s390.c
2330
if (kvm->arch.migration_mode)
arch/s390/kvm/kvm-s390.c
2331
args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
arch/s390/kvm/kvm-s390.c
2347
static int kvm_s390_set_cmma_bits(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
2354
if (!kvm->arch.use_cmma)
arch/s390/kvm/kvm-s390.c
2381
scoped_guard(read_lock, &kvm->mmu_lock) {
arch/s390/kvm/kvm-s390.c
2382
r = dat_set_cmma_bits(mc, kvm->arch.gmap->asce, args->start_gfn,
arch/s390/kvm/kvm-s390.c
2387
set_bit(GMAP_FLAG_USES_CMM, &kvm->arch.gmap->flags);
arch/s390/kvm/kvm-s390.c
2405
int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
arch/s390/kvm/kvm-s390.c
2420
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/s390/kvm/kvm-s390.c
2431
kvm_s390_gisa_enable(kvm);
arch/s390/kvm/kvm-s390.c
2446
static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
arch/s390/kvm/kvm-s390.c
2456
kvm_s390_gisa_disable(kvm);
arch/s390/kvm/kvm-s390.c
2458
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/s390/kvm/kvm-s390.c
2466
kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
arch/s390/kvm/kvm-s390.c
2517
static int kvm_s390_pv_dmp(struct kvm *kvm, struct kvm_pv_cmd *cmd,
arch/s390/kvm/kvm-s390.c
2525
if (kvm->arch.pv.dumping)
arch/s390/kvm/kvm-s390.c
2532
kvm_s390_vcpu_block_all(kvm);
arch/s390/kvm/kvm-s390.c
2534
r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
arch/s390/kvm/kvm-s390.c
2536
KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP INIT: rc %x rrc %x",
arch/s390/kvm/kvm-s390.c
2539
kvm->arch.pv.dumping = true;
arch/s390/kvm/kvm-s390.c
2541
kvm_s390_vcpu_unblock_all(kvm);
arch/s390/kvm/kvm-s390.c
2547
if (!kvm->arch.pv.dumping)
arch/s390/kvm/kvm-s390.c
2555
r = kvm_s390_pv_dump_stor_state(kvm, result_buff, &dmp.gaddr, dmp.buff_len,
arch/s390/kvm/kvm-s390.c
2560
if (!kvm->arch.pv.dumping)
arch/s390/kvm/kvm-s390.c
2567
r = kvm_s390_pv_dump_complete(kvm, result_buff,
arch/s390/kvm/kvm-s390.c
2579
static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
arch/s390/kvm/kvm-s390.c
2587
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
2592
if (kvm_s390_pv_is_protected(kvm))
arch/s390/kvm/kvm-s390.c
2595
kvm_s390_unmap_all_adapters(kvm);
arch/s390/kvm/kvm-s390.c
2596
mmap_write_lock(kvm->mm);
arch/s390/kvm/kvm-s390.c
2601
mm_flags_clear(MMF_DISABLE_THP_EXCEPT_ADVISED, kvm->mm);
arch/s390/kvm/kvm-s390.c
2602
mm_flags_set(MMF_DISABLE_THP_COMPLETELY, kvm->mm);
arch/s390/kvm/kvm-s390.c
2603
set_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &kvm->arch.gmap->flags);
arch/s390/kvm/kvm-s390.c
2605
mmap_write_unlock(kvm->mm);
arch/s390/kvm/kvm-s390.c
2609
r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc);
arch/s390/kvm/kvm-s390.c
2613
r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc);
arch/s390/kvm/kvm-s390.c
2615
kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
arch/s390/kvm/kvm-s390.c
2618
set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
arch/s390/kvm/kvm-s390.c
2623
if (!kvm_s390_pv_is_protected(kvm) || !async_destroy)
arch/s390/kvm/kvm-s390.c
2626
r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
arch/s390/kvm/kvm-s390.c
2634
r = kvm_s390_pv_set_aside(kvm, &cmd->rc, &cmd->rrc);
arch/s390/kvm/kvm-s390.c
2637
clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
arch/s390/kvm/kvm-s390.c
2644
r = kvm_s390_pv_deinit_aside_vm(kvm, &cmd->rc, &cmd->rrc);
arch/s390/kvm/kvm-s390.c
2648
if (!kvm_s390_pv_is_protected(kvm))
arch/s390/kvm/kvm-s390.c
2651
r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
arch/s390/kvm/kvm-s390.c
2659
r = kvm_s390_pv_deinit_cleanup_all(kvm, &cmd->rc, &cmd->rrc);
arch/s390/kvm/kvm-s390.c
2662
clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
arch/s390/kvm/kvm-s390.c
2670
if (!kvm_s390_pv_is_protected(kvm))
arch/s390/kvm/kvm-s390.c
2690
r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length,
arch/s390/kvm/kvm-s390.c
2700
if (!kvm_s390_pv_is_protected(kvm) || !mm_is_protected(kvm->mm))
arch/s390/kvm/kvm-s390.c
2707
r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak,
arch/s390/kvm/kvm-s390.c
2713
if (!kvm_s390_pv_is_protected(kvm))
arch/s390/kvm/kvm-s390.c
2716
r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
arch/s390/kvm/kvm-s390.c
2718
KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
arch/s390/kvm/kvm-s390.c
2724
if (!kvm_s390_pv_is_protected(kvm))
arch/s390/kvm/kvm-s390.c
2727
r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
arch/s390/kvm/kvm-s390.c
2729
KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
arch/s390/kvm/kvm-s390.c
2735
if (!kvm_s390_pv_is_protected(kvm))
arch/s390/kvm/kvm-s390.c
2738
r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
arch/s390/kvm/kvm-s390.c
2740
KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
arch/s390/kvm/kvm-s390.c
2788
if (!kvm_s390_pv_is_protected(kvm))
arch/s390/kvm/kvm-s390.c
2795
r = kvm_s390_pv_dmp(kvm, cmd, dmp);
arch/s390/kvm/kvm-s390.c
2810
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
2830
static int kvm_s390_vm_mem_op_abs(struct kvm *kvm, struct kvm_s390_mem_op *mop)
arch/s390/kvm/kvm-s390.c
2850
scoped_guard(srcu, &kvm->srcu) {
arch/s390/kvm/kvm-s390.c
2852
return check_gpa_range(kvm, mop->gaddr, mop->size, acc_mode, mop->key);
arch/s390/kvm/kvm-s390.c
2856
r = access_guest_abs_with_key(kvm, mop->gaddr, tmpbuf,
arch/s390/kvm/kvm-s390.c
2866
static int kvm_s390_vm_mem_op_cmpxchg(struct kvm *kvm, struct kvm_s390_mem_op *mop)
arch/s390/kvm/kvm-s390.c
2890
scoped_guard(srcu, &kvm->srcu) {
arch/s390/kvm/kvm-s390.c
2891
r = cmpxchg_guest_abs_with_key(kvm, mop->gaddr, mop->size, &old, new,
arch/s390/kvm/kvm-s390.c
2900
static int kvm_s390_vm_mem_op(struct kvm *kvm, struct kvm_s390_mem_op *mop)
arch/s390/kvm/kvm-s390.c
2911
if (kvm_s390_pv_get_handle(kvm))
arch/s390/kvm/kvm-s390.c
2917
return kvm_s390_vm_mem_op_abs(kvm, mop);
arch/s390/kvm/kvm-s390.c
2919
return kvm_s390_vm_mem_op_cmpxchg(kvm, mop);
arch/s390/kvm/kvm-s390.c
2927
struct kvm *kvm = filp->private_data;
arch/s390/kvm/kvm-s390.c
2943
r = kvm_s390_inject_vm(kvm, &s390int, inti);
arch/s390/kvm/kvm-s390.c
2950
if (kvm->arch.use_irqchip)
arch/s390/kvm/kvm-s390.c
2958
r = kvm_s390_vm_set_attr(kvm, &attr);
arch/s390/kvm/kvm-s390.c
2965
r = kvm_s390_vm_get_attr(kvm, &attr);
arch/s390/kvm/kvm-s390.c
2972
r = kvm_s390_vm_has_attr(kvm, &attr);
arch/s390/kvm/kvm-s390.c
2982
r = kvm_s390_get_skeys(kvm, &args);
arch/s390/kvm/kvm-s390.c
2992
r = kvm_s390_set_skeys(kvm, &args);
arch/s390/kvm/kvm-s390.c
3001
mutex_lock(&kvm->slots_lock);
arch/s390/kvm/kvm-s390.c
3002
r = kvm_s390_get_cmma_bits(kvm, &args);
arch/s390/kvm/kvm-s390.c
3003
mutex_unlock(&kvm->slots_lock);
arch/s390/kvm/kvm-s390.c
3017
mutex_lock(&kvm->slots_lock);
arch/s390/kvm/kvm-s390.c
3018
r = kvm_s390_set_cmma_bits(kvm, &args);
arch/s390/kvm/kvm-s390.c
3019
mutex_unlock(&kvm->slots_lock);
arch/s390/kvm/kvm-s390.c
3026
kvm_s390_set_user_cpu_state_ctrl(kvm);
arch/s390/kvm/kvm-s390.c
3041
r = kvm_s390_handle_pv(kvm, &args);
arch/s390/kvm/kvm-s390.c
3052
r = kvm_s390_vm_mem_op(kvm, &mem_op);
arch/s390/kvm/kvm-s390.c
307
struct kvm *kvm;
arch/s390/kvm/kvm-s390.c
3072
r = kvm_s390_keyop(mc, kvm, kop.operation, kop.guest_addr, skey);
arch/s390/kvm/kvm-s390.c
3093
r = kvm_s390_pci_zpci_op(kvm, &args);
arch/s390/kvm/kvm-s390.c
312
list_for_each_entry(kvm, &vm_list, vm_list) {
arch/s390/kvm/kvm-s390.c
3123
static void kvm_s390_set_crycb_format(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
3125
kvm->arch.crypto.crycbd = virt_to_phys(kvm->arch.crypto.crycb);
arch/s390/kvm/kvm-s390.c
3128
kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
arch/s390/kvm/kvm-s390.c
313
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/s390/kvm/kvm-s390.c
3131
if (!test_kvm_facility(kvm, 76))
arch/s390/kvm/kvm-s390.c
3135
kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
arch/s390/kvm/kvm-s390.c
3137
kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
arch/s390/kvm/kvm-s390.c
3155
void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
arch/s390/kvm/kvm-s390.c
3158
struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
arch/s390/kvm/kvm-s390.c
316
kvm->arch.epoch = vcpu->arch.sie_block->epoch;
arch/s390/kvm/kvm-s390.c
3160
kvm_s390_vcpu_block_all(kvm);
arch/s390/kvm/kvm-s390.c
3162
switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
arch/s390/kvm/kvm-s390.c
3165
VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
arch/s390/kvm/kvm-s390.c
3168
VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
arch/s390/kvm/kvm-s390.c
317
kvm->arch.epdx = vcpu->arch.sie_block->epdx;
arch/s390/kvm/kvm-s390.c
3171
VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
arch/s390/kvm/kvm-s390.c
3179
VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
arch/s390/kvm/kvm-s390.c
3188
kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
arch/s390/kvm/kvm-s390.c
3189
kvm_s390_vcpu_unblock_all(kvm);
arch/s390/kvm/kvm-s390.c
3205
void kvm_arch_crypto_clear_masks(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
3207
kvm_s390_vcpu_block_all(kvm);
arch/s390/kvm/kvm-s390.c
3209
memset(&kvm->arch.crypto.crycb->apcb0, 0,
arch/s390/kvm/kvm-s390.c
3210
sizeof(kvm->arch.crypto.crycb->apcb0));
arch/s390/kvm/kvm-s390.c
3211
memset(&kvm->arch.crypto.crycb->apcb1, 0,
arch/s390/kvm/kvm-s390.c
3212
sizeof(kvm->arch.crypto.crycb->apcb1));
arch/s390/kvm/kvm-s390.c
3214
VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
arch/s390/kvm/kvm-s390.c
3216
kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
arch/s390/kvm/kvm-s390.c
3217
kvm_s390_vcpu_unblock_all(kvm);
arch/s390/kvm/kvm-s390.c
3230
static void kvm_s390_crypto_init(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
3232
kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
arch/s390/kvm/kvm-s390.c
3233
kvm_s390_set_crycb_format(kvm);
arch/s390/kvm/kvm-s390.c
3234
init_rwsem(&kvm->arch.crypto.pqap_hook_rwsem);
arch/s390/kvm/kvm-s390.c
3236
if (!test_kvm_facility(kvm, 76))
arch/s390/kvm/kvm-s390.c
3240
kvm->arch.crypto.aes_kw = 1;
arch/s390/kvm/kvm-s390.c
3241
kvm->arch.crypto.dea_kw = 1;
arch/s390/kvm/kvm-s390.c
3242
get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
arch/s390/kvm/kvm-s390.c
3243
sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
arch/s390/kvm/kvm-s390.c
3244
get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
arch/s390/kvm/kvm-s390.c
3245
sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
arch/s390/kvm/kvm-s390.c
3248
static void sca_dispose(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
3250
free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca));
arch/s390/kvm/kvm-s390.c
3251
kvm->arch.sca = NULL;
arch/s390/kvm/kvm-s390.c
3254
void kvm_arch_free_vm(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
3257
kvm_s390_pci_clear_list(kvm);
arch/s390/kvm/kvm-s390.c
3259
__kvm_arch_free_vm(kvm);
arch/s390/kvm/kvm-s390.c
3262
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
arch/s390/kvm/kvm-s390.c
3268
mutex_init(&kvm->arch.pv.import_lock);
arch/s390/kvm/kvm-s390.c
3286
kvm->arch.sca = alloc_pages_exact(sizeof(*kvm->arch.sca), alloc_flags);
arch/s390/kvm/kvm-s390.c
3288
if (!kvm->arch.sca)
arch/s390/kvm/kvm-s390.c
3293
kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
arch/s390/kvm/kvm-s390.c
3294
if (!kvm->arch.dbf)
arch/s390/kvm/kvm-s390.c
3298
kvm->arch.sie_page2 =
arch/s390/kvm/kvm-s390.c
3300
if (!kvm->arch.sie_page2)
arch/s390/kvm/kvm-s390.c
3303
kvm->arch.sie_page2->kvm = kvm;
arch/s390/kvm/kvm-s390.c
3304
kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
arch/s390/kvm/kvm-s390.c
3307
kvm->arch.model.fac_mask[i] = stfle_fac_list[i] &
arch/s390/kvm/kvm-s390.c
3309
kvm->arch.model.fac_list[i] = stfle_fac_list[i] &
arch/s390/kvm/kvm-s390.c
3313
kvm->arch.model.fac_mask[i] |= stfle_fac_list[i] &
arch/s390/kvm/kvm-s390.c
3316
kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
arch/s390/kvm/kvm-s390.c
3319
set_kvm_facility(kvm->arch.model.fac_mask, 138);
arch/s390/kvm/kvm-s390.c
3320
set_kvm_facility(kvm->arch.model.fac_list, 138);
arch/s390/kvm/kvm-s390.c
3322
set_kvm_facility(kvm->arch.model.fac_mask, 74);
arch/s390/kvm/kvm-s390.c
3323
set_kvm_facility(kvm->arch.model.fac_list, 74);
arch/s390/kvm/kvm-s390.c
3325
set_kvm_facility(kvm->arch.model.fac_mask, 147);
arch/s390/kvm/kvm-s390.c
3326
set_kvm_facility(kvm->arch.model.fac_list, 147);
arch/s390/kvm/kvm-s390.c
3330
set_kvm_facility(kvm->arch.model.fac_mask, 65);
arch/s390/kvm/kvm-s390.c
3332
kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
arch/s390/kvm/kvm-s390.c
3333
kvm->arch.model.ibc = sclp.ibc & 0x0fff;
arch/s390/kvm/kvm-s390.c
3335
kvm->arch.model.uv_feat_guest.feat = 0;
arch/s390/kvm/kvm-s390.c
3337
kvm_s390_crypto_init(kvm);
arch/s390/kvm/kvm-s390.c
3340
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
3341
kvm_s390_pci_init_list(kvm);
arch/s390/kvm/kvm-s390.c
3342
kvm_s390_vcpu_pci_enable_interp(kvm);
arch/s390/kvm/kvm-s390.c
3343
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
3346
spin_lock_init(&kvm->arch.float_int.ais_lock);
arch/s390/kvm/kvm-s390.c
3347
spin_lock_init(&kvm->arch.float_int.lock);
arch/s390/kvm/kvm-s390.c
3349
INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
arch/s390/kvm/kvm-s390.c
3350
init_waitqueue_head(&kvm->arch.ipte_wq);
arch/s390/kvm/kvm-s390.c
3351
mutex_init(&kvm->arch.ipte_mutex);
arch/s390/kvm/kvm-s390.c
3353
debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
arch/s390/kvm/kvm-s390.c
3354
VM_EVENT(kvm, 3, "vm created with type %lu", type);
arch/s390/kvm/kvm-s390.c
3356
kvm->arch.mem_limit = type & KVM_VM_S390_UCONTROL ? KVM_S390_NO_MEM_LIMIT : sclp.hamax + 1;
arch/s390/kvm/kvm-s390.c
3357
kvm->arch.gmap = gmap_new(kvm, gpa_to_gfn(kvm->arch.mem_limit));
arch/s390/kvm/kvm-s390.c
3358
if (!kvm->arch.gmap)
arch/s390/kvm/kvm-s390.c
3360
clear_bit(GMAP_FLAG_PFAULT_ENABLED, &kvm->arch.gmap->flags);
arch/s390/kvm/kvm-s390.c
3372
mutex_lock(&kvm->slots_lock);
arch/s390/kvm/kvm-s390.c
3373
KVM_BUG_ON(kvm_set_internal_memslot(kvm, &fake_memslot), kvm);
arch/s390/kvm/kvm-s390.c
3374
mutex_unlock(&kvm->slots_lock);
arch/s390/kvm/kvm-s390.c
3375
set_bit(GMAP_FLAG_IS_UCONTROL, &kvm->arch.gmap->flags);
arch/s390/kvm/kvm-s390.c
3377
struct crst_table *table = dereference_asce(kvm->arch.gmap->asce);
arch/s390/kvm/kvm-s390.c
3382
kvm->arch.use_pfmfi = sclp.has_pfmfi;
arch/s390/kvm/kvm-s390.c
3383
kvm->arch.use_skf = sclp.has_skey;
arch/s390/kvm/kvm-s390.c
3384
spin_lock_init(&kvm->arch.start_stop_lock);
arch/s390/kvm/kvm-s390.c
3385
kvm_s390_vsie_init(kvm);
arch/s390/kvm/kvm-s390.c
3387
kvm_s390_gisa_init(kvm);
arch/s390/kvm/kvm-s390.c
3388
INIT_LIST_HEAD(&kvm->arch.pv.need_cleanup);
arch/s390/kvm/kvm-s390.c
3389
kvm->arch.pv.set_aside = NULL;
arch/s390/kvm/kvm-s390.c
3390
KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
arch/s390/kvm/kvm-s390.c
3394
free_page((unsigned long)kvm->arch.sie_page2);
arch/s390/kvm/kvm-s390.c
3395
debug_unregister(kvm->arch.dbf);
arch/s390/kvm/kvm-s390.c
3396
sca_dispose(kvm);
arch/s390/kvm/kvm-s390.c
3409
if (!kvm_is_ucontrol(vcpu->kvm))
arch/s390/kvm/kvm-s390.c
3411
kvm_s390_update_topology_change_report(vcpu->kvm, 1);
arch/s390/kvm/kvm-s390.c
3413
if (kvm_is_ucontrol(vcpu->kvm)) {
arch/s390/kvm/kvm-s390.c
3414
scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock)
arch/s390/kvm/kvm-s390.c
3419
if (vcpu->kvm->arch.use_cmma)
arch/s390/kvm/kvm-s390.c
3428
void kvm_arch_destroy_vm(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
3432
kvm_destroy_vcpus(kvm);
arch/s390/kvm/kvm-s390.c
3433
sca_dispose(kvm);
arch/s390/kvm/kvm-s390.c
3434
kvm_s390_gisa_destroy(kvm);
arch/s390/kvm/kvm-s390.c
3440
kvm_s390_pv_deinit_cleanup_all(kvm, &rc, &rrc);
arch/s390/kvm/kvm-s390.c
3447
if (kvm->arch.pv.mmu_notifier.ops)
arch/s390/kvm/kvm-s390.c
3448
mmu_notifier_unregister(&kvm->arch.pv.mmu_notifier, kvm->mm);
arch/s390/kvm/kvm-s390.c
3450
debug_unregister(kvm->arch.dbf);
arch/s390/kvm/kvm-s390.c
3451
free_page((unsigned long)kvm->arch.sie_page2);
arch/s390/kvm/kvm-s390.c
3452
kvm_s390_destroy_adapters(kvm);
arch/s390/kvm/kvm-s390.c
3453
kvm_s390_clear_float_irqs(kvm);
arch/s390/kvm/kvm-s390.c
3454
kvm_s390_vsie_destroy(kvm);
arch/s390/kvm/kvm-s390.c
3455
kvm->arch.gmap = gmap_put(kvm->arch.gmap);
arch/s390/kvm/kvm-s390.c
3456
KVM_EVENT(3, "vm 0x%p destroyed", kvm);
arch/s390/kvm/kvm-s390.c
3462
struct esca_block *sca = vcpu->kvm->arch.sca;
arch/s390/kvm/kvm-s390.c
3473
struct esca_block *sca = vcpu->kvm->arch.sca;
arch/s390/kvm/kvm-s390.c
3488
static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
arch/s390/kvm/kvm-s390.c
3603
mutex_lock(&vcpu->kvm->lock);
arch/s390/kvm/kvm-s390.c
3605
vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
arch/s390/kvm/kvm-s390.c
3606
vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
arch/s390/kvm/kvm-s390.c
3608
mutex_unlock(&vcpu->kvm->lock);
arch/s390/kvm/kvm-s390.c
3609
if (!kvm_is_ucontrol(vcpu->kvm)) {
arch/s390/kvm/kvm-s390.c
3610
vcpu->arch.gmap = vcpu->kvm->arch.gmap;
arch/s390/kvm/kvm-s390.c
3613
if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0 ||
arch/s390/kvm/kvm-s390.c
3614
vcpu->kvm->arch.user_operexec)
arch/s390/kvm/kvm-s390.c
3618
static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
arch/s390/kvm/kvm-s390.c
3620
if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
arch/s390/kvm/kvm-s390.c
3626
static bool kvm_has_pckmo_ecc(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
3629
return kvm_has_pckmo_subfunc(kvm, 32) ||
arch/s390/kvm/kvm-s390.c
3630
kvm_has_pckmo_subfunc(kvm, 33) ||
arch/s390/kvm/kvm-s390.c
3631
kvm_has_pckmo_subfunc(kvm, 34) ||
arch/s390/kvm/kvm-s390.c
3632
kvm_has_pckmo_subfunc(kvm, 40) ||
arch/s390/kvm/kvm-s390.c
3633
kvm_has_pckmo_subfunc(kvm, 41);
arch/s390/kvm/kvm-s390.c
3637
static bool kvm_has_pckmo_hmac(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
3640
return kvm_has_pckmo_subfunc(kvm, 118) ||
arch/s390/kvm/kvm-s390.c
3641
kvm_has_pckmo_subfunc(kvm, 122);
arch/s390/kvm/kvm-s390.c
3650
if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
arch/s390/kvm/kvm-s390.c
3653
vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
arch/s390/kvm/kvm-s390.c
3658
if (vcpu->kvm->arch.crypto.apie)
arch/s390/kvm/kvm-s390.c
3662
if (vcpu->kvm->arch.crypto.aes_kw) {
arch/s390/kvm/kvm-s390.c
3665
if (kvm_has_pckmo_ecc(vcpu->kvm))
arch/s390/kvm/kvm-s390.c
3667
if (kvm_has_pckmo_hmac(vcpu->kvm))
arch/s390/kvm/kvm-s390.c
3671
if (vcpu->kvm->arch.crypto.dea_kw)
arch/s390/kvm/kvm-s390.c
3694
struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
arch/s390/kvm/kvm-s390.c
3697
if (test_kvm_facility(vcpu->kvm, 7))
arch/s390/kvm/kvm-s390.c
3710
if (test_kvm_facility(vcpu->kvm, 78))
arch/s390/kvm/kvm-s390.c
3712
else if (test_kvm_facility(vcpu->kvm, 8))
arch/s390/kvm/kvm-s390.c
3720
if (test_kvm_facility(vcpu->kvm, 9))
arch/s390/kvm/kvm-s390.c
3722
if (test_kvm_facility(vcpu->kvm, 11))
arch/s390/kvm/kvm-s390.c
3724
if (test_kvm_facility(vcpu->kvm, 73))
arch/s390/kvm/kvm-s390.c
3726
if (!kvm_is_ucontrol(vcpu->kvm))
arch/s390/kvm/kvm-s390.c
3729
if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
arch/s390/kvm/kvm-s390.c
3731
if (test_kvm_facility(vcpu->kvm, 130))
arch/s390/kvm/kvm-s390.c
3742
if (test_kvm_facility(vcpu->kvm, 129)) {
arch/s390/kvm/kvm-s390.c
3746
if (test_kvm_facility(vcpu->kvm, 139))
arch/s390/kvm/kvm-s390.c
3748
if (test_kvm_facility(vcpu->kvm, 156))
arch/s390/kvm/kvm-s390.c
3763
if (vcpu->kvm->arch.use_cmma) {
arch/s390/kvm/kvm-s390.c
3777
mutex_lock(&vcpu->kvm->lock);
arch/s390/kvm/kvm-s390.c
3778
if (kvm_s390_pv_is_protected(vcpu->kvm)) {
arch/s390/kvm/kvm-s390.c
3783
mutex_unlock(&vcpu->kvm->lock);
arch/s390/kvm/kvm-s390.c
3788
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
arch/s390/kvm/kvm-s390.c
3790
if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
arch/s390/kvm/kvm-s390.c
3820
vcpu->arch.sie_block->gd = kvm_s390_get_gisa_desc(vcpu->kvm);
arch/s390/kvm/kvm-s390.c
3834
if (test_kvm_facility(vcpu->kvm, 64))
arch/s390/kvm/kvm-s390.c
3836
if (test_kvm_facility(vcpu->kvm, 82))
arch/s390/kvm/kvm-s390.c
3838
if (test_kvm_facility(vcpu->kvm, 133))
arch/s390/kvm/kvm-s390.c
3840
if (test_kvm_facility(vcpu->kvm, 156))
arch/s390/kvm/kvm-s390.c
3850
if (kvm_is_ucontrol(vcpu->kvm)) {
arch/s390/kvm/kvm-s390.c
3852
vcpu->arch.gmap = gmap_new_child(vcpu->kvm->arch.gmap, -1UL);
arch/s390/kvm/kvm-s390.c
3857
VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%p, sie block at 0x%p",
arch/s390/kvm/kvm-s390.c
3865
kvm_s390_update_topology_change_report(vcpu->kvm, 1);
arch/s390/kvm/kvm-s390.c
3869
if (kvm_is_ucontrol(vcpu->kvm)) {
arch/s390/kvm/kvm-s390.c
3880
clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
arch/s390/kvm/kvm-s390.c
4063
if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
arch/s390/kvm/kvm-s390.c
4286
kvm_s390_set_user_cpu_state_ctrl(vcpu->kvm);
arch/s390/kvm/kvm-s390.c
4321
if (kvm_is_ucontrol(vcpu->kvm)) {
arch/s390/kvm/kvm-s390.c
4350
scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
arch/s390/kvm/kvm-s390.c
4351
rc = dat_set_prefix_notif_bit(vcpu->kvm->arch.gmap->asce, gfn);
arch/s390/kvm/kvm-s390.c
4420
if (vcpu->kvm->arch.use_cmma && uses_cmm(vcpu->arch.gmap))
arch/s390/kvm/kvm-s390.c
4431
static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
arch/s390/kvm/kvm-s390.c
4441
kvm->arch.epoch = gtod->tod - clk.tod;
arch/s390/kvm/kvm-s390.c
4442
kvm->arch.epdx = 0;
arch/s390/kvm/kvm-s390.c
4443
if (test_kvm_facility(kvm, 139)) {
arch/s390/kvm/kvm-s390.c
4444
kvm->arch.epdx = gtod->epoch_idx - clk.ei;
arch/s390/kvm/kvm-s390.c
4445
if (kvm->arch.epoch > gtod->tod)
arch/s390/kvm/kvm-s390.c
4446
kvm->arch.epdx -= 1;
arch/s390/kvm/kvm-s390.c
4449
kvm_s390_vcpu_block_all(kvm);
arch/s390/kvm/kvm-s390.c
4450
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/s390/kvm/kvm-s390.c
4451
vcpu->arch.sie_block->epoch = kvm->arch.epoch;
arch/s390/kvm/kvm-s390.c
4452
vcpu->arch.sie_block->epdx = kvm->arch.epdx;
arch/s390/kvm/kvm-s390.c
4455
kvm_s390_vcpu_unblock_all(kvm);
arch/s390/kvm/kvm-s390.c
4459
int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
arch/s390/kvm/kvm-s390.c
4461
if (!mutex_trylock(&kvm->lock))
arch/s390/kvm/kvm-s390.c
4463
__kvm_s390_set_tod_clock(kvm, gtod);
arch/s390/kvm/kvm-s390.c
4464
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
4487
ret = kvm_s390_inject_vm(vcpu->kvm, &inti, inti_mem);
arch/s390/kvm/kvm-s390.c
4544
hva = gfn_to_hva(vcpu->kvm, current->thread.gmap_teid.addr);
arch/s390/kvm/kvm-s390.c
4565
if (!kvm_is_ucontrol(vcpu->kvm)) {
arch/s390/kvm/kvm-s390.c
4580
clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
arch/s390/kvm/kvm-s390.c
4629
KVM_BUG(current->thread.gmap_teid.as != PSW_BITS_AS_PRIMARY, vcpu->kvm,
arch/s390/kvm/kvm-s390.c
4651
KVM_BUG_ON(rc, vcpu->kvm);
arch/s390/kvm/kvm-s390.c
4678
if (kvm_s390_pv_destroy_page(vcpu->kvm, gaddr)) {
arch/s390/kvm/kvm-s390.c
4700
rc = kvm_s390_pv_convert_to_secure(vcpu->kvm, gaddr);
arch/s390/kvm/kvm-s390.c
4717
KVM_BUG(1, vcpu->kvm, "Unexpected program interrupt 0x%x, TEID 0x%016lx",
arch/s390/kvm/kvm-s390.c
4894
test_kvm_facility(vcpu->kvm, 64) &&
arch/s390/kvm/kvm-s390.c
4905
test_kvm_facility(vcpu->kvm, 133) &&
arch/s390/kvm/kvm-s390.c
4914
test_kvm_facility(vcpu->kvm, 82)) {
arch/s390/kvm/kvm-s390.c
5032
if (vcpu->kvm->arch.pv.dumping)
arch/s390/kvm/kvm-s390.c
5056
if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
arch/s390/kvm/kvm-s390.c
5177
static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
5182
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/s390/kvm/kvm-s390.c
5204
spin_lock(&vcpu->kvm->arch.start_stop_lock);
arch/s390/kvm/kvm-s390.c
5205
online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
arch/s390/kvm/kvm-s390.c
5211
spin_unlock(&vcpu->kvm->arch.start_stop_lock);
arch/s390/kvm/kvm-s390.c
5217
if (!is_vcpu_stopped(kvm_get_vcpu(vcpu->kvm, i)))
arch/s390/kvm/kvm-s390.c
5230
__disable_ibs_on_all_vcpus(vcpu->kvm);
arch/s390/kvm/kvm-s390.c
5246
spin_unlock(&vcpu->kvm->arch.start_stop_lock);
arch/s390/kvm/kvm-s390.c
5260
spin_lock(&vcpu->kvm->arch.start_stop_lock);
arch/s390/kvm/kvm-s390.c
5261
online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
arch/s390/kvm/kvm-s390.c
5267
spin_unlock(&vcpu->kvm->arch.start_stop_lock);
arch/s390/kvm/kvm-s390.c
5284
struct kvm_vcpu *tmp = kvm_get_vcpu(vcpu->kvm, i);
arch/s390/kvm/kvm-s390.c
5300
spin_unlock(&vcpu->kvm->arch.start_stop_lock);
arch/s390/kvm/kvm-s390.c
5314
if (!vcpu->kvm->arch.css_support) {
arch/s390/kvm/kvm-s390.c
5315
vcpu->kvm->arch.css_support = 1;
arch/s390/kvm/kvm-s390.c
5316
VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
arch/s390/kvm/kvm-s390.c
5317
trace_kvm_s390_enable_css(vcpu->kvm);
arch/s390/kvm/kvm-s390.c
5410
srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/s390/kvm/kvm-s390.c
5426
srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
arch/s390/kvm/kvm-s390.c
5483
if (!vcpu->kvm->arch.pv.dumping)
arch/s390/kvm/kvm-s390.c
5530
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/s390/kvm/kvm-s390.c
5532
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/s390/kvm/kvm-s390.c
5598
if (!kvm_is_ucontrol(vcpu->kvm))
arch/s390/kvm/kvm-s390.c
5616
if (!kvm_is_ucontrol(vcpu->kvm))
arch/s390/kvm/kvm-s390.c
562
static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op,
arch/s390/kvm/kvm-s390.c
5630
scoped_guard(srcu, &vcpu->kvm->srcu) {
arch/s390/kvm/kvm-s390.c
5641
KVM_BUG_ON(r, vcpu->kvm);
arch/s390/kvm/kvm-s390.c
565
union asce asce = kvm->arch.gmap->asce;
arch/s390/kvm/kvm-s390.c
569
guard(read_lock)(&kvm->mmu_lock);
arch/s390/kvm/kvm-s390.c
5736
&& (kvm_is_ucontrol(vcpu->kvm))) {
arch/s390/kvm/kvm-s390.c
5745
bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
5751
int kvm_arch_prepare_memory_region(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
5756
if (kvm_is_ucontrol(kvm) && new && new->id < KVM_USER_MEM_SLOTS)
arch/s390/kvm/kvm-s390.c
5760
if (kvm_s390_pv_get_handle(kvm))
arch/s390/kvm/kvm-s390.c
5772
if ((new->base_gfn + new->npages) * PAGE_SIZE > kvm->arch.mem_limit)
arch/s390/kvm/kvm-s390.c
5774
if (!asce_contains_gfn(kvm->arch.gmap->asce, new->base_gfn + new->npages - 1))
arch/s390/kvm/kvm-s390.c
5778
if (!kvm->arch.migration_mode)
arch/s390/kvm/kvm-s390.c
5791
WARN(kvm_s390_vm_stop_migration(kvm),
arch/s390/kvm/kvm-s390.c
5797
void kvm_arch_commit_memory_region(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
5814
scoped_guard(write_lock, &kvm->mmu_lock) {
arch/s390/kvm/kvm-s390.c
5817
rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages);
arch/s390/kvm/kvm-s390.c
5820
rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages);
arch/s390/kvm/kvm-s390.c
5825
rc = dat_create_slot(mc, kvm->arch.gmap->asce, new->base_gfn, new->npages);
arch/s390/kvm/kvm-s390.c
5866
if (kvm_is_ucontrol(vcpu->kvm))
arch/s390/kvm/kvm-s390.c
5892
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/s390/kvm/kvm-s390.c
5894
scoped_guard(read_lock, &kvm->mmu_lock)
arch/s390/kvm/kvm-s390.c
5895
return dat_test_age_gfn(kvm->arch.gmap->asce, range->start, range->end);
arch/s390/kvm/kvm-s390.c
5906
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/s390/kvm/kvm-s390.c
5908
scoped_guard(read_lock, &kvm->mmu_lock)
arch/s390/kvm/kvm-s390.c
5909
return gmap_age_gfn(kvm->arch.gmap, range->start, range->end);
arch/s390/kvm/kvm-s390.c
5924
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
arch/s390/kvm/kvm-s390.c
5926
return gmap_unmap_gfn_range(kvm->arch.gmap, range->slot, range->start, range->end);
arch/s390/kvm/kvm-s390.c
602
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
arch/s390/kvm/kvm-s390.c
646
if (hpage && !(kvm && kvm_is_ucontrol(kvm)))
arch/s390/kvm/kvm-s390.c
651
if (hpage_2g && !(kvm && kvm_is_ucontrol(kvm)))
arch/s390/kvm/kvm-s390.c
732
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
arch/s390/kvm/kvm-s390.c
736
scoped_guard(read_lock, &kvm->mmu_lock)
arch/s390/kvm/kvm-s390.c
737
gmap_sync_dirty_log(kvm->arch.gmap, memslot->base_gfn, last_gfn);
arch/s390/kvm/kvm-s390.c
746
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
arch/s390/kvm/kvm-s390.c
754
if (kvm_is_ucontrol(kvm))
arch/s390/kvm/kvm-s390.c
757
mutex_lock(&kvm->slots_lock);
arch/s390/kvm/kvm-s390.c
763
r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot);
arch/s390/kvm/kvm-s390.c
774
mutex_unlock(&kvm->slots_lock);
arch/s390/kvm/kvm-s390.c
778
static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
arch/s390/kvm/kvm-s390.c
783
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/s390/kvm/kvm-s390.c
788
int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
arch/s390/kvm/kvm-s390.c
797
VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
arch/s390/kvm/kvm-s390.c
798
kvm->arch.use_irqchip = 1;
arch/s390/kvm/kvm-s390.c
802
VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
arch/s390/kvm/kvm-s390.c
803
kvm->arch.user_sigp = 1;
arch/s390/kvm/kvm-s390.c
807
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
808
if (kvm->created_vcpus) {
arch/s390/kvm/kvm-s390.c
811
set_kvm_facility(kvm->arch.model.fac_mask, 129);
arch/s390/kvm/kvm-s390.c
812
set_kvm_facility(kvm->arch.model.fac_list, 129);
arch/s390/kvm/kvm-s390.c
814
set_kvm_facility(kvm->arch.model.fac_mask, 134);
arch/s390/kvm/kvm-s390.c
815
set_kvm_facility(kvm->arch.model.fac_list, 134);
arch/s390/kvm/kvm-s390.c
818
set_kvm_facility(kvm->arch.model.fac_mask, 135);
arch/s390/kvm/kvm-s390.c
819
set_kvm_facility(kvm->arch.model.fac_list, 135);
arch/s390/kvm/kvm-s390.c
822
set_kvm_facility(kvm->arch.model.fac_mask, 148);
arch/s390/kvm/kvm-s390.c
823
set_kvm_facility(kvm->arch.model.fac_list, 148);
arch/s390/kvm/kvm-s390.c
826
set_kvm_facility(kvm->arch.model.fac_mask, 152);
arch/s390/kvm/kvm-s390.c
827
set_kvm_facility(kvm->arch.model.fac_list, 152);
arch/s390/kvm/kvm-s390.c
830
set_kvm_facility(kvm->arch.model.fac_mask, 192);
arch/s390/kvm/kvm-s390.c
831
set_kvm_facility(kvm->arch.model.fac_list, 192);
arch/s390/kvm/kvm-s390.c
834
set_kvm_facility(kvm->arch.model.fac_mask, 198);
arch/s390/kvm/kvm-s390.c
835
set_kvm_facility(kvm->arch.model.fac_list, 198);
arch/s390/kvm/kvm-s390.c
838
set_kvm_facility(kvm->arch.model.fac_mask, 199);
arch/s390/kvm/kvm-s390.c
839
set_kvm_facility(kvm->arch.model.fac_list, 199);
arch/s390/kvm/kvm-s390.c
844
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
845
VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
arch/s390/kvm/kvm-s390.c
850
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
851
if (kvm->created_vcpus) {
arch/s390/kvm/kvm-s390.c
854
set_kvm_facility(kvm->arch.model.fac_mask, 64);
arch/s390/kvm/kvm-s390.c
855
set_kvm_facility(kvm->arch.model.fac_list, 64);
arch/s390/kvm/kvm-s390.c
858
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
859
VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
arch/s390/kvm/kvm-s390.c
863
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
864
if (kvm->created_vcpus) {
arch/s390/kvm/kvm-s390.c
867
set_kvm_facility(kvm->arch.model.fac_mask, 72);
arch/s390/kvm/kvm-s390.c
868
set_kvm_facility(kvm->arch.model.fac_list, 72);
arch/s390/kvm/kvm-s390.c
871
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
872
VM_EVENT(kvm, 3, "ENABLE: AIS %s",
arch/s390/kvm/kvm-s390.c
877
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
878
if (kvm->created_vcpus) {
arch/s390/kvm/kvm-s390.c
881
set_kvm_facility(kvm->arch.model.fac_mask, 133);
arch/s390/kvm/kvm-s390.c
882
set_kvm_facility(kvm->arch.model.fac_list, 133);
arch/s390/kvm/kvm-s390.c
885
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
886
VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
arch/s390/kvm/kvm-s390.c
890
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
891
if (kvm->created_vcpus)
arch/s390/kvm/kvm-s390.c
893
else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
arch/s390/kvm/kvm-s390.c
897
set_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags);
arch/s390/kvm/kvm-s390.c
903
kvm->arch.use_skf = 0;
arch/s390/kvm/kvm-s390.c
904
kvm->arch.use_pfmfi = 0;
arch/s390/kvm/kvm-s390.c
906
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
907
VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
arch/s390/kvm/kvm-s390.c
911
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
912
if (kvm->created_vcpus) {
arch/s390/kvm/kvm-s390.c
914
} else if (!hpage_2g || kvm->arch.use_cmma || kvm_is_ucontrol(kvm)) {
arch/s390/kvm/kvm-s390.c
918
set_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &kvm->arch.gmap->flags);
arch/s390/kvm/kvm-s390.c
924
kvm->arch.use_skf = 0;
arch/s390/kvm/kvm-s390.c
925
kvm->arch.use_pfmfi = 0;
arch/s390/kvm/kvm-s390.c
927
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
928
VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE_2G %s",
arch/s390/kvm/kvm-s390.c
932
VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
arch/s390/kvm/kvm-s390.c
933
kvm->arch.user_stsi = 1;
arch/s390/kvm/kvm-s390.c
937
VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
arch/s390/kvm/kvm-s390.c
938
kvm->arch.user_instr0 = 1;
arch/s390/kvm/kvm-s390.c
939
icpt_operexc_on_all_vcpus(kvm);
arch/s390/kvm/kvm-s390.c
944
mutex_lock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
945
if (kvm->created_vcpus) {
arch/s390/kvm/kvm-s390.c
948
set_kvm_facility(kvm->arch.model.fac_mask, 11);
arch/s390/kvm/kvm-s390.c
949
set_kvm_facility(kvm->arch.model.fac_list, 11);
arch/s390/kvm/kvm-s390.c
952
mutex_unlock(&kvm->lock);
arch/s390/kvm/kvm-s390.c
953
VM_EVENT(kvm, 3, "ENABLE: CAP_S390_CPU_TOPOLOGY %s",
arch/s390/kvm/kvm-s390.c
957
VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_OPEREXEC");
arch/s390/kvm/kvm-s390.c
958
kvm->arch.user_operexec = 1;
arch/s390/kvm/kvm-s390.c
959
icpt_operexc_on_all_vcpus(kvm);
arch/s390/kvm/kvm-s390.c
963
VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_VSIE_ESAMODE");
arch/s390/kvm/kvm-s390.c
964
kvm->arch.allow_vsie_esamode = 1;
arch/s390/kvm/kvm-s390.c
974
static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.c
981
VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
arch/s390/kvm/kvm-s390.c
982
kvm->arch.mem_limit);
arch/s390/kvm/kvm-s390.c
983
if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
arch/s390/kvm/kvm-s390.c
993
static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
arch/s390/kvm/kvm-s390.h
113
return test_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
arch/s390/kvm/kvm-s390.h
116
static inline int kvm_is_ucontrol(struct kvm *kvm)
arch/s390/kvm/kvm-s390.h
119
return test_bit(GMAP_FLAG_IS_UCONTROL, &kvm->arch.gmap->flags);
arch/s390/kvm/kvm-s390.h
229
static inline int test_kvm_facility(struct kvm *kvm, unsigned long nr)
arch/s390/kvm/kvm-s390.h
231
return __test_facility(nr, kvm->arch.model.fac_mask) &&
arch/s390/kvm/kvm-s390.h
232
__test_facility(nr, kvm->arch.model.fac_list);
arch/s390/kvm/kvm-s390.h
246
static inline int test_kvm_cpu_feat(struct kvm *kvm, unsigned long nr)
arch/s390/kvm/kvm-s390.h
249
return test_bit_inv(nr, kvm->arch.cpu_feat);
arch/s390/kvm/kvm-s390.h
253
static inline int kvm_s390_user_cpu_state_ctrl(struct kvm *kvm)
arch/s390/kvm/kvm-s390.h
255
return kvm->arch.user_cpu_state_ctrl != 0;
arch/s390/kvm/kvm-s390.h
258
static inline void kvm_s390_set_user_cpu_state_ctrl(struct kvm *kvm)
arch/s390/kvm/kvm-s390.h
260
if (kvm->arch.user_cpu_state_ctrl)
arch/s390/kvm/kvm-s390.h
263
VM_EVENT(kvm, 3, "%s", "ENABLE: Userspace CPU state control");
arch/s390/kvm/kvm-s390.h
264
kvm->arch.user_cpu_state_ctrl = 1;
arch/s390/kvm/kvm-s390.h
281
static inline u32 kvm_s390_get_gisa_desc(struct kvm *kvm)
arch/s390/kvm/kvm-s390.h
285
if (!kvm->arch.gisa_int.origin)
arch/s390/kvm/kvm-s390.h
288
gd = virt_to_phys(kvm->arch.gisa_int.origin);
arch/s390/kvm/kvm-s390.h
295
static inline hva_t gpa_to_hva(struct kvm *kvm, gpa_t gpa)
arch/s390/kvm/kvm-s390.h
297
hva_t hva = gfn_to_hva(kvm, gpa_to_gfn(gpa));
arch/s390/kvm/kvm-s390.h
307
int kvm_s390_pv_set_aside(struct kvm *kvm, u16 *rc, u16 *rrc);
arch/s390/kvm/kvm-s390.h
308
int kvm_s390_pv_deinit_aside_vm(struct kvm *kvm, u16 *rc, u16 *rrc);
arch/s390/kvm/kvm-s390.h
309
int kvm_s390_pv_deinit_cleanup_all(struct kvm *kvm, u16 *rc, u16 *rrc);
arch/s390/kvm/kvm-s390.h
310
int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc);
arch/s390/kvm/kvm-s390.h
311
int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc);
arch/s390/kvm/kvm-s390.h
312
int kvm_s390_pv_set_sec_parms(struct kvm *kvm, void *hdr, u64 length, u16 *rc,
arch/s390/kvm/kvm-s390.h
314
int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size,
arch/s390/kvm/kvm-s390.h
318
int kvm_s390_pv_dump_stor_state(struct kvm *kvm, void __user *buff_user,
arch/s390/kvm/kvm-s390.h
320
int kvm_s390_pv_dump_complete(struct kvm *kvm, void __user *buff_user,
arch/s390/kvm/kvm-s390.h
322
int kvm_s390_pv_destroy_page(struct kvm *kvm, unsigned long gaddr);
arch/s390/kvm/kvm-s390.h
323
int kvm_s390_pv_convert_to_secure(struct kvm *kvm, unsigned long gaddr);
arch/s390/kvm/kvm-s390.h
324
int kvm_s390_pv_make_secure(struct kvm *kvm, unsigned long gaddr, void *uvcb);
arch/s390/kvm/kvm-s390.h
326
static inline u64 kvm_s390_pv_get_handle(struct kvm *kvm)
arch/s390/kvm/kvm-s390.h
328
return kvm->arch.pv.handle;
arch/s390/kvm/kvm-s390.h
377
void kvm_s390_clear_float_irqs(struct kvm *kvm);
arch/s390/kvm/kvm-s390.h
378
int __must_check kvm_s390_inject_vm(struct kvm *kvm,
arch/s390/kvm/kvm-s390.h
402
struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
arch/s390/kvm/kvm-s390.h
404
int kvm_s390_reinject_io_int(struct kvm *kvm,
arch/s390/kvm/kvm-s390.h
406
int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked);
arch/s390/kvm/kvm-s390.h
448
void kvm_s390_vsie_init(struct kvm *kvm);
arch/s390/kvm/kvm-s390.h
449
void kvm_s390_vsie_destroy(struct kvm *kvm);
arch/s390/kvm/kvm-s390.h
456
int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod);
arch/s390/kvm/kvm-s390.h
470
int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc);
arch/s390/kvm/kvm-s390.h
479
static inline void kvm_s390_vcpu_block_all(struct kvm *kvm)
arch/s390/kvm/kvm-s390.h
484
WARN_ON(!mutex_is_locked(&kvm->lock));
arch/s390/kvm/kvm-s390.h
485
kvm_for_each_vcpu(i, vcpu, kvm)
arch/s390/kvm/kvm-s390.h
489
static inline void kvm_s390_vcpu_unblock_all(struct kvm *kvm)
arch/s390/kvm/kvm-s390.h
494
kvm_for_each_vcpu(i, vcpu, kvm)
arch/s390/kvm/kvm-s390.h
498
static inline u64 kvm_s390_get_tod_clock_fast(struct kvm *kvm)
arch/s390/kvm/kvm-s390.h
503
rc = get_tod_clock_fast() + kvm->arch.epoch;
arch/s390/kvm/kvm-s390.h
547
void kvm_s390_destroy_adapters(struct kvm *kvm);
arch/s390/kvm/kvm-s390.h
557
void kvm_s390_gisa_init(struct kvm *kvm);
arch/s390/kvm/kvm-s390.h
558
void kvm_s390_gisa_clear(struct kvm *kvm);
arch/s390/kvm/kvm-s390.h
559
void kvm_s390_gisa_destroy(struct kvm *kvm);
arch/s390/kvm/kvm-s390.h
560
void kvm_s390_gisa_disable(struct kvm *kvm);
arch/s390/kvm/kvm-s390.h
561
void kvm_s390_gisa_enable(struct kvm *kvm);
arch/s390/kvm/kvm-s390.h
564
void kvm_s390_unmap_all_adapters(struct kvm *kvm);
arch/s390/kvm/kvm-s390.h
608
void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm);
arch/s390/kvm/kvm-s390.h
618
void kvm_s390_vcpu_pci_enable_interp(struct kvm *kvm);
arch/s390/kvm/kvm-s390.h
85
debug_sprintf_event(d_vcpu->kvm->arch.dbf, d_loglevel, \
arch/s390/kvm/pci.c
250
struct kvm *kvm;
arch/s390/kvm/pci.c
265
kvm = zdev->kzdev->kvm;
arch/s390/kvm/pci.c
269
gisc = kvm_s390_gisc_register(kvm, fib->fmt0.isc);
arch/s390/kvm/pci.c
282
idx = srcu_read_lock(&kvm->srcu);
arch/s390/kvm/pci.c
283
hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aibv));
arch/s390/kvm/pci.c
285
srcu_read_unlock(&kvm->srcu, idx);
arch/s390/kvm/pci.c
303
idx = srcu_read_lock(&kvm->srcu);
arch/s390/kvm/pci.c
304
hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aisb));
arch/s390/kvm/pci.c
307
srcu_read_unlock(&kvm->srcu, idx);
arch/s390/kvm/pci.c
344
gaite->gisa = (u32)virt_to_phys(&kvm->arch.sie_page2->gisa);
arch/s390/kvm/pci.c
401
kvm_s390_gisc_unregister(kvm, fib->fmt0.isc);
arch/s390/kvm/pci.c
451
kvm_s390_gisc_unregister(kzdev->kvm, isc);
arch/s390/kvm/pci.c
501
static int kvm_s390_pci_register_kvm(void *opaque, struct kvm *kvm)
arch/s390/kvm/pci.c
511
if (zdev->kzdev || zdev->gisa != 0 || !kvm) {
arch/s390/kvm/pci.c
516
kvm_get_kvm(kvm);
arch/s390/kvm/pci.c
518
mutex_lock(&kvm->lock);
arch/s390/kvm/pci.c
535
if (!kvm->arch.use_zpci_interp)
arch/s390/kvm/pci.c
536
kvm_s390_vcpu_pci_enable_interp(kvm);
arch/s390/kvm/pci.c
548
zdev->gisa = (u32)virt_to_phys(&kvm->arch.sie_page2->gisa);
arch/s390/kvm/pci.c
555
zdev->kzdev->kvm = kvm;
arch/s390/kvm/pci.c
557
spin_lock(&kvm->arch.kzdev_list_lock);
arch/s390/kvm/pci.c
558
list_add_tail(&zdev->kzdev->entry, &kvm->arch.kzdev_list);
arch/s390/kvm/pci.c
559
spin_unlock(&kvm->arch.kzdev_list_lock);
arch/s390/kvm/pci.c
561
mutex_unlock(&kvm->lock);
arch/s390/kvm/pci.c
570
mutex_unlock(&kvm->lock);
arch/s390/kvm/pci.c
572
kvm_put_kvm(kvm);
arch/s390/kvm/pci.c
579
struct kvm *kvm;
arch/s390/kvm/pci.c
591
kvm = zdev->kzdev->kvm;
arch/s390/kvm/pci.c
592
mutex_lock(&kvm->lock);
arch/s390/kvm/pci.c
616
spin_lock(&kvm->arch.kzdev_list_lock);
arch/s390/kvm/pci.c
618
spin_unlock(&kvm->arch.kzdev_list_lock);
arch/s390/kvm/pci.c
621
mutex_unlock(&kvm->lock);
arch/s390/kvm/pci.c
624
kvm_put_kvm(kvm);
arch/s390/kvm/pci.c
627
void kvm_s390_pci_init_list(struct kvm *kvm)
arch/s390/kvm/pci.c
629
spin_lock_init(&kvm->arch.kzdev_list_lock);
arch/s390/kvm/pci.c
630
INIT_LIST_HEAD(&kvm->arch.kzdev_list);
arch/s390/kvm/pci.c
633
void kvm_s390_pci_clear_list(struct kvm *kvm)
arch/s390/kvm/pci.c
639
spin_lock(&kvm->arch.kzdev_list_lock);
arch/s390/kvm/pci.c
640
WARN_ON_ONCE(!list_empty(&kvm->arch.kzdev_list));
arch/s390/kvm/pci.c
641
spin_unlock(&kvm->arch.kzdev_list_lock);
arch/s390/kvm/pci.c
644
static struct zpci_dev *get_zdev_from_kvm_by_fh(struct kvm *kvm, u32 fh)
arch/s390/kvm/pci.c
649
spin_lock(&kvm->arch.kzdev_list_lock);
arch/s390/kvm/pci.c
650
list_for_each_entry(kzdev, &kvm->arch.kzdev_list, entry) {
arch/s390/kvm/pci.c
656
spin_unlock(&kvm->arch.kzdev_list_lock);
arch/s390/kvm/pci.c
684
int kvm_s390_pci_zpci_op(struct kvm *kvm, struct kvm_s390_zpci_op *args)
arch/s390/kvm/pci.c
690
zdev = get_zdev_from_kvm_by_fh(kvm, args->fh);
arch/s390/kvm/pci.c
695
mutex_lock(&kvm->lock);
arch/s390/kvm/pci.c
702
if (kzdev->kvm != kvm) {
arch/s390/kvm/pci.c
724
mutex_unlock(&kvm->lock);
arch/s390/kvm/pci.h
22
struct kvm *kvm;
arch/s390/kvm/pci.h
48
static inline struct kvm *kvm_s390_pci_si_to_kvm(struct zpci_aift *aift,
arch/s390/kvm/pci.h
54
return aift->kzdev[si]->kvm;
arch/s390/kvm/pci.h
60
void kvm_s390_pci_init_list(struct kvm *kvm);
arch/s390/kvm/pci.h
61
void kvm_s390_pci_clear_list(struct kvm *kvm);
arch/s390/kvm/pci.h
63
int kvm_s390_pci_zpci_op(struct kvm *kvm, struct kvm_s390_zpci_op *args);
arch/s390/kvm/priv.c
1056
if (!test_kvm_facility(vcpu->kvm, 8))
arch/s390/kvm/priv.c
1067
!test_kvm_facility(vcpu->kvm, 14))
arch/s390/kvm/priv.c
1072
test_kvm_facility(vcpu->kvm, 10)) {
arch/s390/kvm/priv.c
1099
if (!test_kvm_facility(vcpu->kvm, 78) ||
arch/s390/kvm/priv.c
1110
if (kvm_clear_guest(vcpu->kvm, start, PAGE_SIZE))
arch/s390/kvm/priv.c
1119
scoped_guard(read_lock, &vcpu->kvm->mmu_lock) {
arch/s390/kvm/priv.c
114
if (!kvm_s390_try_set_tod_clock(vcpu->kvm, &gtod)) {
arch/s390/kvm/priv.c
1185
atomic64_inc(&vcpu->kvm->arch.cmma_dirty_pages);
arch/s390/kvm/priv.c
1198
lockdep_assert_held(&vcpu->kvm->mmu_lock);
arch/s390/kvm/priv.c
1208
hva = gpa_to_hva(vcpu->kvm, cbrl[i]);
arch/s390/kvm/priv.c
1210
gmap_helper_zap_one_page(vcpu->kvm->mm, hva);
arch/s390/kvm/priv.c
1218
lockdep_assert_held(&vcpu->kvm->srcu);
arch/s390/kvm/priv.c
1227
if (!vcpu->kvm->arch.use_cmma)
arch/s390/kvm/priv.c
1235
if (orc > (test_kvm_facility(vcpu->kvm, 147) ? ESSA_SET_STABLE_NODAT
arch/s390/kvm/priv.c
1239
if (!READ_ONCE(vcpu->kvm->arch.migration_mode)) {
arch/s390/kvm/priv.c
1263
scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
arch/s390/kvm/priv.c
1274
mmap_read_lock(vcpu->kvm->mm);
arch/s390/kvm/priv.c
1275
scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
arch/s390/kvm/priv.c
1277
mmap_read_unlock(vcpu->kvm->mm);
arch/s390/kvm/priv.c
1481
ipte_lock(vcpu->kvm);
arch/s390/kvm/priv.c
1486
gfn_to_hva_prot(vcpu->kvm, gpa_to_gfn(gpa), &writable);
arch/s390/kvm/priv.c
1518
ipte_unlock(vcpu->kvm);
arch/s390/kvm/priv.c
153
if (!kvm_is_gpa_in_memslot(vcpu->kvm, address))
arch/s390/kvm/priv.c
233
if (!vcpu->kvm->arch.use_skf)
arch/s390/kvm/priv.c
248
if (vcpu->kvm->arch.use_skf) {
arch/s390/kvm/priv.c
278
scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
arch/s390/kvm/priv.c
309
scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
arch/s390/kvm/priv.c
340
if (!test_kvm_facility(vcpu->kvm, 8))
arch/s390/kvm/priv.c
342
if (!test_kvm_facility(vcpu->kvm, 10))
arch/s390/kvm/priv.c
344
if (!test_kvm_facility(vcpu->kvm, 14))
arch/s390/kvm/priv.c
361
scoped_guard(read_lock, &vcpu->kvm->mmu_lock) {
arch/s390/kvm/priv.c
37
if (test_kvm_facility(vcpu->kvm, 64)) {
arch/s390/kvm/priv.c
405
wait_event(vcpu->kvm->arch.ipte_wq, !ipte_lock_held(vcpu->kvm));
arch/s390/kvm/priv.c
428
if (!kvm_is_gpa_in_memslot(vcpu->kvm, addr))
arch/s390/kvm/priv.c
434
if (kvm_clear_guest(vcpu->kvm, addr, PAGE_SIZE))
arch/s390/kvm/priv.c
456
inti = kvm_s390_get_io_int(vcpu->kvm, vcpu->arch.sie_block->gcr[6], 0);
arch/s390/kvm/priv.c
499
if (kvm_s390_reinject_io_int(vcpu->kvm, inti)) {
arch/s390/kvm/priv.c
516
inti = kvm_s390_get_io_int(vcpu->kvm, isc_mask,
arch/s390/kvm/priv.c
547
if (vcpu->kvm->arch.css_support) {
arch/s390/kvm/priv.c
570
bool kvm_s390_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa)
arch/s390/kvm/priv.c
572
return kvm_is_gpa_in_memslot(kvm, gpa);
arch/s390/kvm/priv.c
58
if (test_kvm_facility(vcpu->kvm, 133)) {
arch/s390/kvm/priv.c
628
if (!test_kvm_facility(vcpu->kvm, 15) && (reg0 & 0x00800000UL))
arch/s390/kvm/priv.c
631
if (!(vcpu->kvm->arch.crypto.crycbd & 0x02) && (reg0 & 0x0000c0f0UL))
arch/s390/kvm/priv.c
636
if (!test_kvm_facility(vcpu->kvm, 65))
arch/s390/kvm/priv.c
644
down_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem);
arch/s390/kvm/priv.c
645
if (vcpu->kvm->arch.crypto.pqap_hook) {
arch/s390/kvm/priv.c
646
pqap_hook = *vcpu->kvm->arch.crypto.pqap_hook;
arch/s390/kvm/priv.c
654
up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem);
arch/s390/kvm/priv.c
657
up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem);
arch/s390/kvm/priv.c
683
fac = *vcpu->kvm->arch.model.fac_list >> 32;
arch/s390/kvm/priv.c
782
if (!test_kvm_facility(vcpu->kvm, 193))
arch/s390/kvm/priv.c
805
u64 stidp_data = vcpu->kvm->arch.model.cpuid;
arch/s390/kvm/priv.c
833
cpus = atomic_read(&vcpu->kvm->online_vcpus);
arch/s390/kvm/priv.c
891
if (fc == 15 && (!test_kvm_facility(vcpu->kvm, 11) ||
arch/s390/kvm/priv.c
892
!vcpu->kvm->arch.user_stsi))
arch/s390/kvm/priv.c
942
if (vcpu->kvm->arch.user_stsi) {
arch/s390/kvm/pv.c
1023
int kvm_s390_pv_dump_complete(struct kvm *kvm, void __user *buff_user,
arch/s390/kvm/pv.c
1029
.config_handle = kvm_s390_pv_get_handle(kvm),
arch/s390/kvm/pv.c
1043
KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP COMPLETE: rc %x rrc %x",
arch/s390/kvm/pv.c
1052
kvm->arch.pv.dumping = false;
arch/s390/kvm/pv.c
1053
kvm_s390_vcpu_unblock_all(kvm);
arch/s390/kvm/pv.c
114
if (!mmap_read_trylock(priv->kvm->mm))
arch/s390/kvm/pv.c
117
ptep = try_get_locked_pte(priv->kvm->mm, gfn_to_hva(priv->kvm, f->gfn), &ptl);
arch/s390/kvm/pv.c
135
mmap_read_unlock(priv->kvm->mm);
arch/s390/kvm/pv.c
147
int kvm_s390_pv_make_secure(struct kvm *kvm, unsigned long gaddr, void *uvcb)
arch/s390/kvm/pv.c
149
struct pv_make_secure priv = { .uvcb = uvcb, .kvm = kvm, };
arch/s390/kvm/pv.c
158
lockdep_assert_held(&kvm->srcu);
arch/s390/kvm/pv.c
160
priv.needs_export = should_export_before_import(uvcb, kvm->mm);
arch/s390/kvm/pv.c
162
scoped_guard(mutex, &kvm->arch.pv.import_lock) {
arch/s390/kvm/pv.c
163
rc = kvm_s390_faultin_gfn(NULL, kvm, &f);
arch/s390/kvm/pv.c
168
rc = s390_wiggle_split_folio(kvm->mm, priv.folio);
arch/s390/kvm/pv.c
179
int kvm_s390_pv_convert_to_secure(struct kvm *kvm, unsigned long gaddr)
arch/s390/kvm/pv.c
184
.guest_handle = kvm_s390_pv_get_handle(kvm),
arch/s390/kvm/pv.c
188
return kvm_s390_pv_make_secure(kvm, gaddr, &uvcb);
arch/s390/kvm/pv.c
202
int kvm_s390_pv_destroy_page(struct kvm *kvm, unsigned long gaddr)
arch/s390/kvm/pv.c
207
mmap_read_lock(kvm->mm);
arch/s390/kvm/pv.c
208
page = gfn_to_page(kvm, gpa_to_gfn(gaddr));
arch/s390/kvm/pv.c
212
mmap_read_unlock(kvm->mm);
arch/s390/kvm/pv.c
239
static void kvm_s390_clear_pv_state(struct kvm *kvm)
arch/s390/kvm/pv.c
241
kvm->arch.pv.handle = 0;
arch/s390/kvm/pv.c
242
kvm->arch.pv.guest_len = 0;
arch/s390/kvm/pv.c
243
kvm->arch.pv.stor_base = 0;
arch/s390/kvm/pv.c
244
kvm->arch.pv.stor_var = NULL;
arch/s390/kvm/pv.c
256
KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT DESTROY VCPU %d: rc %x rrc %x",
arch/s390/kvm/pv.c
27
bool kvm_s390_pv_is_protected(struct kvm *kvm)
arch/s390/kvm/pv.c
29
lockdep_assert_held(&kvm->lock);
arch/s390/kvm/pv.c
299
uvcb.guest_handle = kvm_s390_pv_get_handle(vcpu->kvm);
arch/s390/kvm/pv.c
30
return !!kvm_s390_pv_get_handle(kvm);
arch/s390/kvm/pv.c
316
KVM_UV_EVENT(vcpu->kvm, 3,
arch/s390/kvm/pv.c
331
vcpu->arch.sie_block->pv_handle_config = kvm_s390_pv_get_handle(vcpu->kvm);
arch/s390/kvm/pv.c
338
static void kvm_s390_pv_dealloc_vm(struct kvm *kvm)
arch/s390/kvm/pv.c
340
vfree(kvm->arch.pv.stor_var);
arch/s390/kvm/pv.c
341
free_pages(kvm->arch.pv.stor_base,
arch/s390/kvm/pv.c
343
kvm_s390_clear_pv_state(kvm);
arch/s390/kvm/pv.c
346
static int kvm_s390_pv_alloc_vm(struct kvm *kvm)
arch/s390/kvm/pv.c
352
kvm->arch.pv.stor_var = NULL;
arch/s390/kvm/pv.c
353
kvm->arch.pv.stor_base = __get_free_pages(GFP_KERNEL_ACCOUNT, get_order(base));
arch/s390/kvm/pv.c
354
if (!kvm->arch.pv.stor_base)
arch/s390/kvm/pv.c
363
mutex_lock(&kvm->slots_lock);
arch/s390/kvm/pv.c
364
npages = kvm_s390_get_gfn_end(kvm_memslots(kvm));
arch/s390/kvm/pv.c
365
mutex_unlock(&kvm->slots_lock);
arch/s390/kvm/pv.c
367
kvm->arch.pv.guest_len = npages * PAGE_SIZE;
arch/s390/kvm/pv.c
372
kvm->arch.pv.stor_var = vzalloc(vlen);
arch/s390/kvm/pv.c
373
if (!kvm->arch.pv.stor_var)
arch/s390/kvm/pv.c
378
kvm_s390_pv_dealloc_vm(kvm);
arch/s390/kvm/pv.c
395
static int kvm_s390_pv_dispose_one_leftover(struct kvm *kvm,
arch/s390/kvm/pv.c
405
KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY LEFTOVER VM: rc %x rrc %x", *rc, *rrc);
arch/s390/kvm/pv.c
419
atomic_dec(&kvm->mm->context.protected_count);
arch/s390/kvm/pv.c
423
static int kvm_s390_pv_deinit_vm_fast(struct kvm *kvm, u16 *rc, u16 *rrc)
arch/s390/kvm/pv.c
428
.handle = kvm_s390_pv_get_handle(kvm),
arch/s390/kvm/pv.c
437
KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY VM FAST: rc %x rrc %x",
arch/s390/kvm/pv.c
441
kvm_s390_pv_get_handle(kvm), uvcb.header.rc, uvcb.header.rrc);
arch/s390/kvm/pv.c
444
kvm_s390_pv_dealloc_vm(kvm);
arch/s390/kvm/pv.c
471
int kvm_s390_pv_set_aside(struct kvm *kvm, u16 *rc, u16 *rrc)
arch/s390/kvm/pv.c
476
lockdep_assert_held(&kvm->lock);
arch/s390/kvm/pv.c
481
if (kvm->arch.pv.set_aside)
arch/s390/kvm/pv.c
485
if (kvm->arch.gmap->asce.dt == TABLE_TYPE_SEGMENT)
arch/s390/kvm/pv.c
493
res = kvm_s390_pv_deinit_vm_fast(kvm, rc, rrc);
arch/s390/kvm/pv.c
495
priv->stor_var = kvm->arch.pv.stor_var;
arch/s390/kvm/pv.c
496
priv->stor_base = kvm->arch.pv.stor_base;
arch/s390/kvm/pv.c
497
priv->handle = kvm_s390_pv_get_handle(kvm);
arch/s390/kvm/pv.c
498
priv->old_gmap_table = (unsigned long)dereference_asce(kvm->arch.gmap->asce);
arch/s390/kvm/pv.c
499
if (s390_replace_asce(kvm->arch.gmap))
arch/s390/kvm/pv.c
508
gmap_pv_destroy_range(kvm->arch.gmap, 0, gpa_to_gfn(SZ_2G), false);
arch/s390/kvm/pv.c
509
kvm_s390_clear_pv_state(kvm);
arch/s390/kvm/pv.c
510
kvm->arch.pv.set_aside = priv;
arch/s390/kvm/pv.c
536
int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
arch/s390/kvm/pv.c
540
cc = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
arch/s390/kvm/pv.c
543
atomic_dec(&kvm->mm->context.protected_count);
arch/s390/kvm/pv.c
544
kvm_s390_pv_dealloc_vm(kvm);
arch/s390/kvm/pv.c
547
s390_replace_asce(kvm->arch.gmap);
arch/s390/kvm/pv.c
549
KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY VM: rc %x rrc %x", *rc, *rrc);
arch/s390/kvm/pv.c
571
int kvm_s390_pv_deinit_cleanup_all(struct kvm *kvm, u16 *rc, u16 *rrc)
arch/s390/kvm/pv.c
582
if (!atomic_inc_not_zero(&kvm->mm->context.protected_count))
arch/s390/kvm/pv.c
587
if (kvm_s390_pv_get_handle(kvm)) {
arch/s390/kvm/pv.c
588
cc = kvm_s390_pv_deinit_vm(kvm, rc, rrc);
arch/s390/kvm/pv.c
593
if (kvm->arch.pv.set_aside) {
arch/s390/kvm/pv.c
594
list_add(kvm->arch.pv.set_aside, &kvm->arch.pv.need_cleanup);
arch/s390/kvm/pv.c
595
kvm->arch.pv.set_aside = NULL;
arch/s390/kvm/pv.c
599
while (!list_empty(&kvm->arch.pv.need_cleanup)) {
arch/s390/kvm/pv.c
600
cur = list_first_entry(&kvm->arch.pv.need_cleanup, typeof(*cur), list);
arch/s390/kvm/pv.c
602
if (kvm_s390_pv_dispose_one_leftover(kvm, cur, &_rc, &_rrc)) {
arch/s390/kvm/pv.c
623
if (need_zap && mmget_not_zero(kvm->mm)) {
arch/s390/kvm/pv.c
624
gmap_pv_destroy_range(kvm->arch.gmap, 0, asce_end(kvm->arch.gmap->asce), false);
arch/s390/kvm/pv.c
625
mmput(kvm->mm);
arch/s390/kvm/pv.c
629
atomic_dec(&kvm->mm->context.protected_count);
arch/s390/kvm/pv.c
648
int kvm_s390_pv_deinit_aside_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
arch/s390/kvm/pv.c
653
lockdep_assert_not_held(&kvm->lock);
arch/s390/kvm/pv.c
654
mutex_lock(&kvm->lock);
arch/s390/kvm/pv.c
655
p = kvm->arch.pv.set_aside;
arch/s390/kvm/pv.c
656
kvm->arch.pv.set_aside = NULL;
arch/s390/kvm/pv.c
657
mutex_unlock(&kvm->lock);
arch/s390/kvm/pv.c
662
if (gmap_pv_destroy_range(kvm->arch.gmap, 0, asce_end(kvm->arch.gmap->asce), true))
arch/s390/kvm/pv.c
664
if (kvm_s390_pv_dispose_one_leftover(kvm, p, rc, rrc))
arch/s390/kvm/pv.c
674
mutex_lock(&kvm->lock);
arch/s390/kvm/pv.c
675
list_add(&p->list, &kvm->arch.pv.need_cleanup);
arch/s390/kvm/pv.c
676
mutex_unlock(&kvm->lock);
arch/s390/kvm/pv.c
687
struct kvm *kvm = container_of(subscription, struct kvm, arch.pv.mmu_notifier);
arch/s390/kvm/pv.c
698
r = kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
arch/s390/kvm/pv.c
699
if (!r && is_destroy_fast_available() && kvm_s390_pv_get_handle(kvm))
arch/s390/kvm/pv.c
700
kvm_s390_pv_deinit_vm_fast(kvm, &dummy, &dummy);
arch/s390/kvm/pv.c
701
set_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &kvm->arch.gmap->flags);
arch/s390/kvm/pv.c
708
int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
arch/s390/kvm/pv.c
718
if (kvm->arch.pv.mmu_notifier.ops != &kvm_s390_pv_mmu_notifier_ops) {
arch/s390/kvm/pv.c
720
kvm->arch.pv.mmu_notifier.ops = &kvm_s390_pv_mmu_notifier_ops;
arch/s390/kvm/pv.c
721
ret = mmu_notifier_register(&kvm->arch.pv.mmu_notifier, kvm->mm);
arch/s390/kvm/pv.c
723
kvm->arch.pv.mmu_notifier.ops = NULL;
arch/s390/kvm/pv.c
728
ret = kvm_s390_pv_alloc_vm(kvm);
arch/s390/kvm/pv.c
734
uvcb.guest_stor_len = kvm->arch.pv.guest_len;
arch/s390/kvm/pv.c
735
uvcb.guest_asce = kvm->arch.gmap->asce.val;
arch/s390/kvm/pv.c
736
uvcb.guest_sca = virt_to_phys(kvm->arch.sca);
arch/s390/kvm/pv.c
738
virt_to_phys((void *)kvm->arch.pv.stor_base);
arch/s390/kvm/pv.c
739
uvcb.conf_virt_stor_origin = (u64)kvm->arch.pv.stor_var;
arch/s390/kvm/pv.c
740
uvcb.flags.ap_allow_instr = kvm->arch.model.uv_feat_guest.ap;
arch/s390/kvm/pv.c
741
uvcb.flags.ap_instr_intr = kvm->arch.model.uv_feat_guest.ap_intr;
arch/s390/kvm/pv.c
743
scoped_guard(write_lock, &kvm->mmu_lock) {
arch/s390/kvm/pv.c
744
clear_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags);
arch/s390/kvm/pv.c
745
clear_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &kvm->arch.gmap->flags);
arch/s390/kvm/pv.c
747
gmap_split_huge_pages(kvm->arch.gmap);
arch/s390/kvm/pv.c
752
KVM_UV_EVENT(kvm, 3, "PROTVIRT CREATE VM: handle %llx len %llx rc %x rrc %x flags %04x",
arch/s390/kvm/pv.c
756
kvm->arch.pv.handle = uvcb.guest_handle;
arch/s390/kvm/pv.c
758
atomic_inc(&kvm->mm->context.protected_count);
arch/s390/kvm/pv.c
761
kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
arch/s390/kvm/pv.c
763
atomic_dec(&kvm->mm->context.protected_count);
arch/s390/kvm/pv.c
764
kvm_s390_pv_dealloc_vm(kvm);
arch/s390/kvm/pv.c
77
struct kvm *kvm;
arch/s390/kvm/pv.c
771
int kvm_s390_pv_set_sec_parms(struct kvm *kvm, void *hdr, u64 length, u16 *rc,
arch/s390/kvm/pv.c
779
.guest_handle = kvm_s390_pv_get_handle(kvm),
arch/s390/kvm/pv.c
785
KVM_UV_EVENT(kvm, 3, "PROTVIRT VM SET PARMS: rc %x rrc %x",
arch/s390/kvm/pv.c
790
static int unpack_one(struct kvm *kvm, unsigned long addr, u64 tweak,
arch/s390/kvm/pv.c
796
.guest_handle = kvm_s390_pv_get_handle(kvm),
arch/s390/kvm/pv.c
801
int ret = kvm_s390_pv_make_secure(kvm, addr, &uvcb);
arch/s390/kvm/pv.c
807
ret = kvm_s390_faultin_gfn_simple(NULL, kvm, gpa_to_gfn(addr), true);
arch/s390/kvm/pv.c
813
KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: failed addr %llx with rc %x rrc %x",
arch/s390/kvm/pv.c
818
int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size,
arch/s390/kvm/pv.c
827
KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: start addr %lx size %lx",
arch/s390/kvm/pv.c
830
guard(srcu)(&kvm->srcu);
arch/s390/kvm/pv.c
833
ret = unpack_one(kvm, addr, tweak, offset, rc, rrc);
arch/s390/kvm/pv.c
846
KVM_UV_EVENT(kvm, 3, "%s", "PROTVIRT VM UNPACK: successful");
arch/s390/kvm/pv.c
861
KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT SET CPU %d STATE %d rc %x rrc %x",
arch/s390/kvm/pv.c
915
int kvm_s390_pv_dump_stor_state(struct kvm *kvm, void __user *buff_user,
arch/s390/kvm/pv.c
921
.config_handle = kvm->arch.pv.handle,
arch/s390/kvm/pv.c
994
KVM_UV_EVENT(kvm, 3,
arch/s390/kvm/sigp.c
175
if (!kvm_is_gpa_in_memslot(vcpu->kvm, irq.u.prefix.address)) {
arch/s390/kvm/sigp.c
218
if (!test_kvm_facility(vcpu->kvm, 9)) {
arch/s390/kvm/sigp.c
274
struct kvm_vcpu *dst_vcpu = kvm_get_vcpu_by_id(vcpu->kvm, cpu_addr);
arch/s390/kvm/sigp.c
378
if (!vcpu->kvm->arch.user_sigp)
arch/s390/kvm/sigp.c
486
dest_vcpu = kvm_get_vcpu_by_id(vcpu->kvm, cpu_addr);
arch/s390/kvm/trace-s390.h
247
TP_PROTO(void *kvm),
arch/s390/kvm/trace-s390.h
248
TP_ARGS(kvm),
arch/s390/kvm/trace-s390.h
251
__field(void *, kvm)
arch/s390/kvm/trace-s390.h
255
__entry->kvm = kvm;
arch/s390/kvm/trace-s390.h
259
__entry->kvm)
arch/s390/kvm/trace-s390.h
8
#define TRACE_SYSTEM kvm-s390
arch/s390/kvm/trace.h
11
#define TRACE_SYSTEM kvm
arch/s390/kvm/vsie.c
1054
bool has_astfleie2 = test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_ASTFLEIE2);
arch/s390/kvm/vsie.c
1063
if (fac && test_kvm_facility(vcpu->kvm, 7)) {
arch/s390/kvm/vsie.c
1117
edat = cr0.edat && test_kvm_facility(vcpu->kvm, 8);
arch/s390/kvm/vsie.c
1188
__releases(vcpu->kvm->srcu)
arch/s390/kvm/vsie.c
1189
__acquires(vcpu->kvm->srcu)
arch/s390/kvm/vsie.c
1210
if (test_kvm_facility(vcpu->kvm, 82) &&
arch/s390/kvm/vsie.c
1288
lockdep_assert_held(&gmap->kvm->arch.gmap->children_lock);
arch/s390/kvm/vsie.c
1309
edat = cr0.edat && test_kvm_facility(vcpu->kvm, 8);
arch/s390/kvm/vsie.c
1310
edat += edat && test_kvm_facility(vcpu->kvm, 78);
arch/s390/kvm/vsie.c
1312
scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock) {
arch/s390/kvm/vsie.c
132
if (!(cpuflags & CPUSTAT_ZARCH) && !vcpu->kvm->arch.allow_vsie_esamode)
arch/s390/kvm/vsie.c
1321
vcpu->kvm->stat.gmap_shadow_reuse++;
arch/s390/kvm/vsie.c
1330
gmap = gmap_create_shadow(vcpu->arch.mc, vcpu->kvm->arch.gmap, asce, edat);
arch/s390/kvm/vsie.c
1333
scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock) {
arch/s390/kvm/vsie.c
1341
vcpu->kvm->stat.gmap_shadow_create++;
arch/s390/kvm/vsie.c
1368
scb_s->epoch += vcpu->kvm->arch.epoch;
arch/s390/kvm/vsie.c
1371
scb_s->epdx += vcpu->kvm->arch.epdx;
arch/s390/kvm/vsie.c
1372
if (scb_s->epoch < vcpu->kvm->arch.epoch)
arch/s390/kvm/vsie.c
144
if (cpuflags & CPUSTAT_GED && test_kvm_facility(vcpu->kvm, 8))
arch/s390/kvm/vsie.c
146
if (cpuflags & CPUSTAT_GED2 && test_kvm_facility(vcpu->kvm, 78)) {
arch/s390/kvm/vsie.c
1483
static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr)
arch/s390/kvm/vsie.c
1489
vsie_page = radix_tree_lookup(&kvm->arch.vsie.addr_to_page, addr >> 9);
arch/s390/kvm/vsie.c
1507
nr_vcpus = atomic_read(&kvm->online_vcpus);
arch/s390/kvm/vsie.c
1509
mutex_lock(&kvm->arch.vsie.mutex);
arch/s390/kvm/vsie.c
151
if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_GPERE))
arch/s390/kvm/vsie.c
1510
if (kvm->arch.vsie.page_count < nr_vcpus) {
arch/s390/kvm/vsie.c
1513
mutex_unlock(&kvm->arch.vsie.mutex);
arch/s390/kvm/vsie.c
1517
kvm->arch.vsie.pages[kvm->arch.vsie.page_count] = vsie_page;
arch/s390/kvm/vsie.c
1518
kvm->arch.vsie.page_count++;
arch/s390/kvm/vsie.c
1522
vsie_page = kvm->arch.vsie.pages[kvm->arch.vsie.next];
arch/s390/kvm/vsie.c
1525
kvm->arch.vsie.next++;
arch/s390/kvm/vsie.c
1526
kvm->arch.vsie.next %= nr_vcpus;
arch/s390/kvm/vsie.c
1529
radix_tree_delete(&kvm->arch.vsie.addr_to_page,
arch/s390/kvm/vsie.c
153
if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_GSLS))
arch/s390/kvm/vsie.c
1536
if (radix_tree_insert(&kvm->arch.vsie.addr_to_page, addr >> 9, vsie_page)) {
arch/s390/kvm/vsie.c
1538
mutex_unlock(&kvm->arch.vsie.mutex);
arch/s390/kvm/vsie.c
1542
mutex_unlock(&kvm->arch.vsie.mutex);
arch/s390/kvm/vsie.c
1546
scoped_guard(spinlock, &kvm->arch.gmap->children_lock)
arch/s390/kvm/vsie.c
155
if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_IBS))
arch/s390/kvm/vsie.c
1563
if (!test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_SIEF2))
arch/s390/kvm/vsie.c
157
if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_KSS))
arch/s390/kvm/vsie.c
1580
vsie_page = get_vsie_page(vcpu->kvm, scb_addr);
arch/s390/kvm/vsie.c
1613
void kvm_s390_vsie_init(struct kvm *kvm)
arch/s390/kvm/vsie.c
1615
mutex_init(&kvm->arch.vsie.mutex);
arch/s390/kvm/vsie.c
1616
INIT_RADIX_TREE(&kvm->arch.vsie.addr_to_page, GFP_KERNEL_ACCOUNT);
arch/s390/kvm/vsie.c
1620
void kvm_s390_vsie_destroy(struct kvm *kvm)
arch/s390/kvm/vsie.c
1625
mutex_lock(&kvm->arch.vsie.mutex);
arch/s390/kvm/vsie.c
1626
for (i = 0; i < kvm->arch.vsie.page_count; i++) {
arch/s390/kvm/vsie.c
1627
vsie_page = kvm->arch.vsie.pages[i];
arch/s390/kvm/vsie.c
1628
scoped_guard(spinlock, &kvm->arch.gmap->children_lock)
arch/s390/kvm/vsie.c
1631
kvm->arch.vsie.pages[i] = NULL;
arch/s390/kvm/vsie.c
1634
radix_tree_delete(&kvm->arch.vsie.addr_to_page,
arch/s390/kvm/vsie.c
1638
kvm->arch.vsie.page_count = 0;
arch/s390/kvm/vsie.c
1639
mutex_unlock(&kvm->arch.vsie.mutex);
arch/s390/kvm/vsie.c
329
int key_msk = test_kvm_facility(vcpu->kvm, 76);
arch/s390/kvm/vsie.c
351
vcpu->kvm->arch.crypto.crycb,
arch/s390/kvm/vsie.c
377
vcpu->kvm->arch.crypto.crycb->dea_wrapping_key_mask;
arch/s390/kvm/vsie.c
416
if (vcpu->kvm->arch.model.ibc && new_ibc) {
arch/s390/kvm/vsie.c
422
if (scb_s->ibc > vcpu->kvm->arch.model.ibc)
arch/s390/kvm/vsie.c
423
scb_s->ibc = vcpu->kvm->arch.model.ibc;
arch/s390/kvm/vsie.c
453
if (test_kvm_facility(vcpu->kvm, 82)) {
arch/s390/kvm/vsie.c
555
if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_ESOP))
arch/s390/kvm/vsie.c
563
if (test_kvm_facility(vcpu->kvm, 11))
arch/s390/kvm/vsie.c
566
if (test_kvm_facility(vcpu->kvm, 73) && wants_tx) {
arch/s390/kvm/vsie.c
575
if (test_kvm_facility(vcpu->kvm, 82))
arch/s390/kvm/vsie.c
578
if (test_kvm_facility(vcpu->kvm, 129)) {
arch/s390/kvm/vsie.c
583
if (test_kvm_facility(vcpu->kvm, 64))
arch/s390/kvm/vsie.c
586
if (test_kvm_facility(vcpu->kvm, 130))
arch/s390/kvm/vsie.c
589
if (test_kvm_facility(vcpu->kvm, 133)) {
arch/s390/kvm/vsie.c
593
if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_SIIF))
arch/s390/kvm/vsie.c
595
if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_IB))
arch/s390/kvm/vsie.c
597
if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_CEI))
arch/s390/kvm/vsie.c
600
if (test_kvm_facility(vcpu->kvm, 139)) {
arch/s390/kvm/vsie.c
606
if (test_kvm_facility(vcpu->kvm, 156))
arch/s390/kvm/vsie.c
628
KVM_BUG_ON(!test_bit(GMAP_FLAG_SHADOW, &gmap->flags), gmap->kvm);
arch/s390/kvm/vsie.c
690
static int pin_guest_page(struct kvm *kvm, gpa_t gpa, hpa_t *hpa)
arch/s390/kvm/vsie.c
694
page = gfn_to_page(kvm, gpa_to_gfn(gpa));
arch/s390/kvm/vsie.c
702
static void unpin_guest_page(struct kvm *kvm, gpa_t gpa, hpa_t hpa)
arch/s390/kvm/vsie.c
706
mark_page_dirty(kvm, gpa_to_gfn(gpa));
arch/s390/kvm/vsie.c
717
unpin_guest_page(vcpu->kvm, vsie_page->sca_gpa, hpa);
arch/s390/kvm/vsie.c
725
unpin_guest_page(vcpu->kvm, vsie_page->itdba_gpa, hpa);
arch/s390/kvm/vsie.c
732
unpin_guest_page(vcpu->kvm, vsie_page->gvrd_gpa, hpa);
arch/s390/kvm/vsie.c
739
unpin_guest_page(vcpu->kvm, vsie_page->riccbd_gpa, hpa);
arch/s390/kvm/vsie.c
746
unpin_guest_page(vcpu->kvm, vsie_page->sdnx_gpa, hpa);
arch/s390/kvm/vsie.c
775
if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_64BSCAO))
arch/s390/kvm/vsie.c
786
rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
arch/s390/kvm/vsie.c
804
rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
arch/s390/kvm/vsie.c
823
rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
arch/s390/kvm/vsie.c
839
rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
arch/s390/kvm/vsie.c
869
rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
arch/s390/kvm/vsie.c
890
unpin_guest_page(vcpu->kvm, gpa, hpa);
arch/s390/kvm/vsie.c
906
rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
arch/x86/include/asm/kvm_host.h
1795
bool (*has_emulated_msr)(struct kvm *kvm, u32 index);
arch/x86/include/asm/kvm_host.h
1799
int (*vm_init)(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
1800
void (*vm_destroy)(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
1801
void (*vm_pre_destroy)(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
1804
int (*vcpu_precreate)(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
1850
int (*flush_remote_tlbs)(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
1851
int (*flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn,
arch/x86/include/asm/kvm_host.h
1910
int (*set_tss_addr)(struct kvm *kvm, unsigned int addr);
arch/x86/include/asm/kvm_host.h
1911
int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr);
arch/x86/include/asm/kvm_host.h
1913
bool (*tdp_has_smep)(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
1919
int (*set_external_spte)(struct kvm *kvm, gfn_t gfn, u64 old_spte,
arch/x86/include/asm/kvm_host.h
1923
void (*free_external_spt)(struct kvm *kvm, struct kvm_mmu_page *sp);
arch/x86/include/asm/kvm_host.h
1957
int (*pi_update_irte)(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
arch/x86/include/asm/kvm_host.h
1960
void (*pi_start_bypass)(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
1980
int (*mem_enc_ioctl)(struct kvm *kvm, void __user *argp);
arch/x86/include/asm/kvm_host.h
1983
int (*mem_enc_register_region)(struct kvm *kvm, struct kvm_enc_region *argp);
arch/x86/include/asm/kvm_host.h
1984
int (*mem_enc_unregister_region)(struct kvm *kvm, struct kvm_enc_region *argp);
arch/x86/include/asm/kvm_host.h
1985
int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
arch/x86/include/asm/kvm_host.h
1986
int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
arch/x86/include/asm/kvm_host.h
1987
void (*guest_memory_reclaimed)(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
2010
int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order);
arch/x86/include/asm/kvm_host.h
2012
int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private);
arch/x86/include/asm/kvm_host.h
2076
static inline struct kvm *kvm_arch_alloc_vm(void)
arch/x86/include/asm/kvm_host.h
2082
void kvm_arch_free_vm(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
2086
static inline int kvm_arch_flush_remote_tlbs(struct kvm *kvm)
arch/x86/include/asm/kvm_host.h
2089
!kvm_x86_call(flush_remote_tlbs)(kvm))
arch/x86/include/asm/kvm_host.h
2096
static inline int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn,
arch/x86/include/asm/kvm_host.h
2102
return kvm_x86_call(flush_remote_tlbs_range)(kvm, gfn, nr_pages);
arch/x86/include/asm/kvm_host.h
2123
int kvm_mmu_init_vm(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
2124
void kvm_mmu_uninit_vm(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
2126
void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
arch/x86/include/asm/kvm_host.h
2131
void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
arch/x86/include/asm/kvm_host.h
2134
void kvm_mmu_slot_try_split_huge_pages(struct kvm *kvm,
arch/x86/include/asm/kvm_host.h
2137
void kvm_mmu_try_split_huge_pages(struct kvm *kvm,
arch/x86/include/asm/kvm_host.h
2141
void kvm_mmu_recover_huge_pages(struct kvm *kvm,
arch/x86/include/asm/kvm_host.h
2143
void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
arch/x86/include/asm/kvm_host.h
2145
void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen);
arch/x86/include/asm/kvm_host.h
2146
void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long kvm_nr_mmu_pages);
arch/x86/include/asm/kvm_host.h
2147
void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end);
arch/x86/include/asm/kvm_host.h
2342
void kvm_mmu_free_roots(struct kvm *kvm, struct kvm_mmu *mmu,
arch/x86/include/asm/kvm_host.h
2344
void kvm_mmu_free_guest_mode_roots(struct kvm *kvm, struct kvm_mmu *mmu);
arch/x86/include/asm/kvm_host.h
2352
bool kvm_apicv_activated(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
2355
void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
arch/x86/include/asm/kvm_host.h
2357
void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
arch/x86/include/asm/kvm_host.h
2360
static inline void kvm_set_apicv_inhibit(struct kvm *kvm,
arch/x86/include/asm/kvm_host.h
2363
kvm_set_or_clear_apicv_inhibit(kvm, reason, true);
arch/x86/include/asm/kvm_host.h
2366
static inline void kvm_clear_apicv_inhibit(struct kvm *kvm,
arch/x86/include/asm/kvm_host.h
2369
kvm_set_or_clear_apicv_inhibit(kvm, reason, false);
arch/x86/include/asm/kvm_host.h
2372
void kvm_inc_or_dec_irq_window_inhibit(struct kvm *kvm, bool inc);
arch/x86/include/asm/kvm_host.h
2374
static inline void kvm_inc_apicv_irq_window_req(struct kvm *kvm)
arch/x86/include/asm/kvm_host.h
2376
kvm_inc_or_dec_irq_window_inhibit(kvm, true);
arch/x86/include/asm/kvm_host.h
2379
static inline void kvm_dec_apicv_irq_window_req(struct kvm *kvm)
arch/x86/include/asm/kvm_host.h
2381
kvm_inc_or_dec_irq_window_inhibit(kvm, false);
arch/x86/include/asm/kvm_host.h
2398
#define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem)
arch/x86/include/asm/kvm_host.h
2401
#define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state)
arch/x86/include/asm/kvm_host.h
2452
# define kvm_arch_nr_memslot_as_ids(kvm) (kvm_arch_has_private_mem(kvm) ? 1 : 2)
arch/x86/include/asm/kvm_host.h
2454
# define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm)
arch/x86/include/asm/kvm_host.h
2456
# define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, 0)
arch/x86/include/asm/kvm_host.h
2467
int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
arch/x86/include/asm/kvm_host.h
2489
void kvm_make_scan_ioapic_request(struct kvm *kvm);
arch/x86/include/asm/kvm_host.h
2490
void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
arch/x86/include/asm/kvm_host.h
2506
void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
arch/x86/include/asm/kvm_page_track.h
47
int kvm_page_track_register_notifier(struct kvm *kvm,
arch/x86/include/asm/kvm_page_track.h
49
void kvm_page_track_unregister_notifier(struct kvm *kvm,
arch/x86/include/asm/kvm_page_track.h
52
int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn);
arch/x86/include/asm/kvm_page_track.h
53
int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn);
arch/x86/include/asm/kvm_types.h
10
#define KVM_SUB_MODULES kvm-intel
arch/x86/include/asm/kvm_types.h
6
#define KVM_SUB_MODULES kvm-amd,kvm-intel
arch/x86/include/asm/kvm_types.h
8
#define KVM_SUB_MODULES kvm-amd
arch/x86/kvm/cpuid.c
248
if (kvm_hlt_in_guest(vcpu->kvm))
arch/x86/kvm/cpuid.c
303
if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT))
arch/x86/kvm/debugfs.c
100
if (!kvm_memslots_have_rmaps(kvm))
arch/x86/kvm/debugfs.c
111
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/debugfs.c
112
write_lock(&kvm->mmu_lock);
arch/x86/kvm/debugfs.c
114
for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
arch/x86/kvm/debugfs.c
117
slots = __kvm_memslots(kvm, i);
arch/x86/kvm/debugfs.c
132
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/debugfs.c
133
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/debugfs.c
162
struct kvm *kvm = inode->i_private;
arch/x86/kvm/debugfs.c
165
if (!kvm_get_kvm_safe(kvm))
arch/x86/kvm/debugfs.c
168
r = single_open(file, kvm_mmu_rmaps_stat_show, kvm);
arch/x86/kvm/debugfs.c
170
kvm_put_kvm(kvm);
arch/x86/kvm/debugfs.c
177
struct kvm *kvm = inode->i_private;
arch/x86/kvm/debugfs.c
179
kvm_put_kvm(kvm);
arch/x86/kvm/debugfs.c
192
void kvm_arch_create_vm_debugfs(struct kvm *kvm)
arch/x86/kvm/debugfs.c
194
debugfs_create_file("mmu_rmaps_stat", 0644, kvm->debugfs_dentry, kvm,
arch/x86/kvm/debugfs.c
92
struct kvm *kvm = m->private;
arch/x86/kvm/hyperv.c
1036
static int kvm_hv_msr_get_crash_data(struct kvm *kvm, u32 index, u64 *pdata)
arch/x86/kvm/hyperv.c
1038
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
1048
static int kvm_hv_msr_get_crash_ctl(struct kvm *kvm, u64 *pdata)
arch/x86/kvm/hyperv.c
1050
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
1056
static int kvm_hv_msr_set_crash_ctl(struct kvm *kvm, u64 data)
arch/x86/kvm/hyperv.c
1058
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
1065
static int kvm_hv_msr_set_crash_data(struct kvm *kvm, u32 index, u64 data)
arch/x86/kvm/hyperv.c
1067
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
111
struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
arch/x86/kvm/hyperv.c
1161
void kvm_hv_setup_tsc_page(struct kvm *kvm,
arch/x86/kvm/hyperv.c
1164
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
1186
if (unlikely(kvm_read_guest(kvm, gfn_to_gpa(gfn),
arch/x86/kvm/hyperv.c
1191
if (kvm_read_guest(kvm, gfn_to_gpa(gfn), &hv->tsc_ref, sizeof(hv->tsc_ref)))
arch/x86/kvm/hyperv.c
1203
if (kvm_write_guest(kvm, gfn_to_gpa(gfn),
arch/x86/kvm/hyperv.c
1212
if (kvm_write_guest(kvm, gfn_to_gpa(gfn), &hv->tsc_ref, sizeof(hv->tsc_ref)))
arch/x86/kvm/hyperv.c
1226
if (kvm_write_guest(kvm, gfn_to_gpa(gfn),
arch/x86/kvm/hyperv.c
1237
void kvm_hv_request_tsc_page_update(struct kvm *kvm)
arch/x86/kvm/hyperv.c
1239
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
1338
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/hyperv.c
1339
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
1352
if (atomic_read(&kvm->online_vcpus) < 2)
arch/x86/kvm/hyperv.c
1365
struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
arch/x86/kvm/hyperv.c
137
down_write(&vcpu->kvm->arch.apicv_update_lock);
arch/x86/kvm/hyperv.c
1379
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/hyperv.c
1380
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
1411
if (kvm_xen_hypercall_enabled(kvm)) {
arch/x86/kvm/hyperv.c
1446
return kvm_hv_msr_set_crash_data(kvm,
arch/x86/kvm/hyperv.c
1451
return kvm_hv_msr_set_crash_ctl(kvm, data);
arch/x86/kvm/hyperv.c
148
__kvm_set_or_clear_apicv_inhibit(vcpu->kvm,
arch/x86/kvm/hyperv.c
152
up_write(&vcpu->kvm->arch.apicv_update_lock);
arch/x86/kvm/hyperv.c
1528
struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
arch/x86/kvm/hyperv.c
1635
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/hyperv.c
1636
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
1649
data = get_time_ref_counter(kvm);
arch/x86/kvm/hyperv.c
1655
return kvm_hv_msr_get_crash_data(kvm,
arch/x86/kvm/hyperv.c
1659
return kvm_hv_msr_get_crash_ctl(kvm, pdata);
arch/x86/kvm/hyperv.c
1742
vcpu->kvm->arch.apic_bus_cycle_ns);
arch/x86/kvm/hyperv.c
1754
struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
arch/x86/kvm/hyperv.c
1775
struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
arch/x86/kvm/hyperv.c
1794
static void sparse_set_to_vcpu_mask(struct kvm *kvm, u64 *sparse_banks,
arch/x86/kvm/hyperv.c
1797
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
1831
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/hyperv.c
1889
static int kvm_hv_get_hc_data(struct kvm *kvm, struct kvm_hv_hcall *hc,
arch/x86/kvm/hyperv.c
190
static struct kvm_vcpu *get_vcpu_by_vpidx(struct kvm *kvm, u32 vpidx)
arch/x86/kvm/hyperv.c
1918
return kvm_read_guest(kvm, hc->ingpa + hc->data_offset, data,
arch/x86/kvm/hyperv.c
1922
static u64 kvm_get_sparse_vp_set(struct kvm *kvm, struct kvm_hv_hcall *hc,
arch/x86/kvm/hyperv.c
1929
return kvm_hv_get_hc_data(kvm, hc, hc->var_cnt, KVM_HV_MAX_SPARSE_VCPU_SET_BITS,
arch/x86/kvm/hyperv.c
1933
static int kvm_hv_get_tlb_flush_entries(struct kvm *kvm, struct kvm_hv_hcall *hc, u64 entries[])
arch/x86/kvm/hyperv.c
1935
return kvm_hv_get_hc_data(kvm, hc, hc->rep_cnt, hc->rep_cnt, entries);
arch/x86/kvm/hyperv.c
198
vcpu = kvm_get_vcpu(kvm, vpidx);
arch/x86/kvm/hyperv.c
201
kvm_for_each_vcpu(i, vcpu, kvm)
arch/x86/kvm/hyperv.c
2017
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/hyperv.c
2064
if (unlikely(kvm_read_guest(kvm, hc->ingpa,
arch/x86/kvm/hyperv.c
207
static struct kvm_vcpu_hv_synic *synic_get(struct kvm *kvm, u32 vpidx)
arch/x86/kvm/hyperv.c
2094
if (unlikely(kvm_read_guest(kvm, hc->ingpa, &flush_ex,
arch/x86/kvm/hyperv.c
2116
if (kvm_get_sparse_vp_set(kvm, hc, sparse_banks))
arch/x86/kvm/hyperv.c
212
vcpu = get_vcpu_by_vpidx(kvm, vpidx);
arch/x86/kvm/hyperv.c
2138
if (kvm_hv_get_tlb_flush_entries(kvm, hc, __tlb_flush_entries))
arch/x86/kvm/hyperv.c
2148
kvm_for_each_vcpu(i, v, kvm) {
arch/x86/kvm/hyperv.c
2154
kvm_make_all_cpus_request(kvm, KVM_REQ_HV_TLB_FLUSH);
arch/x86/kvm/hyperv.c
2156
sparse_set_to_vcpu_mask(kvm, sparse_banks, valid_bank_mask, vcpu_mask);
arch/x86/kvm/hyperv.c
2159
v = kvm_get_vcpu(kvm, i);
arch/x86/kvm/hyperv.c
2167
kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask);
arch/x86/kvm/hyperv.c
2173
kvm_for_each_vcpu(i, v, kvm) {
arch/x86/kvm/hyperv.c
2200
kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask);
arch/x86/kvm/hyperv.c
2209
static void kvm_hv_send_ipi_to_many(struct kvm *kvm, u32 vector,
arch/x86/kvm/hyperv.c
221
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/hyperv.c
2219
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/hyperv.c
2234
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/hyperv.c
2246
if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi,
arch/x86/kvm/hyperv.c
2264
if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi_ex,
arch/x86/kvm/hyperv.c
2296
if (kvm_get_sparse_vp_set(kvm, hc, sparse_banks))
arch/x86/kvm/hyperv.c
2305
kvm_hv_send_ipi_to_many(kvm, vector, NULL, 0);
arch/x86/kvm/hyperv.c
2307
kvm_hv_send_ipi_to_many(kvm, vector, sparse_banks, valid_bank_mask);
arch/x86/kvm/hyperv.c
238
idx = srcu_read_lock(&kvm->irq_srcu);
arch/x86/kvm/hyperv.c
2399
kvm_read_guest(vcpu->kvm, to_hv_vcpu(vcpu)->nested.pa_page_gpa,
arch/x86/kvm/hyperv.c
241
kvm_notify_acked_gsi(kvm, gsi);
arch/x86/kvm/hyperv.c
242
srcu_read_unlock(&kvm->irq_srcu, idx);
arch/x86/kvm/hyperv.c
2422
struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
arch/x86/kvm/hyperv.c
2702
void kvm_hv_init_vm(struct kvm *kvm)
arch/x86/kvm/hyperv.c
2704
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
2710
void kvm_hv_destroy_vm(struct kvm *kvm)
arch/x86/kvm/hyperv.c
2712
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
2721
static int kvm_hv_eventfd_assign(struct kvm *kvm, u32 conn_id, int fd)
arch/x86/kvm/hyperv.c
2723
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
2745
static int kvm_hv_eventfd_deassign(struct kvm *kvm, u32 conn_id)
arch/x86/kvm/hyperv.c
2747
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
2757
synchronize_srcu(&kvm->srcu);
arch/x86/kvm/hyperv.c
2762
int kvm_vm_ioctl_hv_eventfd(struct kvm *kvm, struct kvm_hyperv_eventfd *args)
arch/x86/kvm/hyperv.c
2769
return kvm_hv_eventfd_deassign(kvm, args->conn_id);
arch/x86/kvm/hyperv.c
2770
return kvm_hv_eventfd_assign(kvm, args->conn_id, args->fd);
arch/x86/kvm/hyperv.c
287
if (kvm_clear_guest(vcpu->kvm,
arch/x86/kvm/hyperv.c
299
if (kvm_clear_guest(vcpu->kvm,
arch/x86/kvm/hyperv.c
338
struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
arch/x86/kvm/hyperv.c
478
if (KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm))
arch/x86/kvm/hyperv.c
495
ret = kvm_irq_delivery_to_apic(vcpu->kvm, vcpu->arch.apic, &irq);
arch/x86/kvm/hyperv.c
500
int kvm_hv_synic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
arch/x86/kvm/hyperv.c
508
synic = synic_get(kvm, e->hv_sint.vcpu);
arch/x86/kvm/hyperv.c
527
static int kvm_hv_set_sint_gsi(struct kvm *kvm, u32 vpidx, u32 sint, int gsi)
arch/x86/kvm/hyperv.c
531
synic = synic_get(kvm, vpidx);
arch/x86/kvm/hyperv.c
542
void kvm_hv_irq_routing_update(struct kvm *kvm)
arch/x86/kvm/hyperv.c
548
irq_rt = srcu_dereference_check(kvm->irq_routing, &kvm->irq_srcu,
arch/x86/kvm/hyperv.c
549
lockdep_is_held(&kvm->irq_lock));
arch/x86/kvm/hyperv.c
554
kvm_hv_set_sint_gsi(kvm, e->hv_sint.vcpu,
arch/x86/kvm/hyperv.c
572
static u64 get_time_ref_counter(struct kvm *kvm)
arch/x86/kvm/hyperv.c
574
struct kvm_hv *hv = to_kvm_hv(kvm);
arch/x86/kvm/hyperv.c
583
return div_u64(get_kvmclock_ns(kvm), 100);
arch/x86/kvm/hyperv.c
585
vcpu = kvm_get_vcpu(kvm, 0);
arch/x86/kvm/hyperv.c
639
time_now = get_time_ref_counter(hv_stimer_to_vcpu(stimer)->kvm);
arch/x86/kvm/hyperv.c
830
payload->delivery_time = get_time_ref_counter(vcpu->kvm);
arch/x86/kvm/hyperv.c
885
get_time_ref_counter(vcpu->kvm);
arch/x86/kvm/hyperv.c
935
return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data,
arch/x86/kvm/hyperv.h
100
return vcpu->arch.hyperv_enabled && to_kvm_hv(vcpu->kvm)->hv_guest_os_id;
arch/x86/kvm/hyperv.h
105
void kvm_hv_irq_routing_update(struct kvm *kvm);
arch/x86/kvm/hyperv.h
106
int kvm_hv_synic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
arch/x86/kvm/hyperv.h
177
return !(to_kvm_hv(vcpu->kvm)->hv_invtsc_control & HV_EXPOSE_INVARIANT_TSC);
arch/x86/kvm/hyperv.h
182
void kvm_hv_setup_tsc_page(struct kvm *kvm,
arch/x86/kvm/hyperv.h
184
void kvm_hv_request_tsc_page_update(struct kvm *kvm);
arch/x86/kvm/hyperv.h
188
void kvm_hv_init_vm(struct kvm *kvm);
arch/x86/kvm/hyperv.h
189
void kvm_hv_destroy_vm(struct kvm *kvm);
arch/x86/kvm/hyperv.h
193
int kvm_vm_ioctl_hv_eventfd(struct kvm *kvm, struct kvm_hyperv_eventfd *args);
arch/x86/kvm/hyperv.h
270
static inline void kvm_hv_setup_tsc_page(struct kvm *kvm,
arch/x86/kvm/hyperv.h
272
static inline void kvm_hv_request_tsc_page_update(struct kvm *kvm) {}
arch/x86/kvm/hyperv.h
274
static inline void kvm_hv_init_vm(struct kvm *kvm) {}
arch/x86/kvm/hyperv.h
275
static inline void kvm_hv_destroy_vm(struct kvm *kvm) {}
arch/x86/kvm/hyperv.h
59
static inline struct kvm_hv *to_kvm_hv(struct kvm *kvm)
arch/x86/kvm/hyperv.h
61
return &kvm->arch.hyperv;
arch/x86/kvm/hyperv.h
85
return &vcpu->kvm->arch.hyperv.hv_syndbg;
arch/x86/kvm/i8254.c
220
struct kvm_pit *pit = vcpu->kvm->arch.vpit;
arch/x86/kvm/i8254.c
243
struct kvm *kvm = pit->kvm;
arch/x86/kvm/i8254.c
251
kvm_set_irq(kvm, KVM_PIT_IRQ_SOURCE_ID, 0, 1, false);
arch/x86/kvm/i8254.c
252
kvm_set_irq(kvm, KVM_PIT_IRQ_SOURCE_ID, 0, 0, false);
arch/x86/kvm/i8254.c
263
if (atomic_read(&kvm->arch.vapics_in_nmi_mode) > 0)
arch/x86/kvm/i8254.c
264
kvm_for_each_vcpu(i, vcpu, kvm)
arch/x86/kvm/i8254.c
294
struct kvm *kvm = pit->kvm;
arch/x86/kvm/i8254.c
308
kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PIT_REINJ);
arch/x86/kvm/i8254.c
311
kvm_register_irq_ack_notifier(kvm, &ps->irq_ack_notifier);
arch/x86/kvm/i8254.c
312
kvm_register_irq_mask_notifier(kvm, 0, &pit->mask_notifier);
arch/x86/kvm/i8254.c
314
kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PIT_REINJ);
arch/x86/kvm/i8254.c
315
kvm_unregister_irq_ack_notifier(kvm, &ps->irq_ack_notifier);
arch/x86/kvm/i8254.c
316
kvm_unregister_irq_mask_notifier(kvm, 0, &pit->mask_notifier);
arch/x86/kvm/i8254.c
325
struct kvm *kvm = pit->kvm;
arch/x86/kvm/i8254.c
328
if (!ioapic_in_kernel(kvm) ||
arch/x86/kvm/i8254.c
652
int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
arch/x86/kvm/i8254.c
654
struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;
arch/x86/kvm/i8254.c
664
int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
arch/x86/kvm/i8254.c
667
struct kvm_pit *pit = kvm->arch.vpit;
arch/x86/kvm/i8254.c
677
int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
arch/x86/kvm/i8254.c
679
mutex_lock(&kvm->arch.vpit->pit_state.lock);
arch/x86/kvm/i8254.c
680
memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
arch/x86/kvm/i8254.c
682
ps->flags = kvm->arch.vpit->pit_state.flags;
arch/x86/kvm/i8254.c
683
mutex_unlock(&kvm->arch.vpit->pit_state.lock);
arch/x86/kvm/i8254.c
688
int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
arch/x86/kvm/i8254.c
693
struct kvm_pit *pit = kvm->arch.vpit;
arch/x86/kvm/i8254.c
710
int kvm_vm_ioctl_reinject(struct kvm *kvm, struct kvm_reinject_control *control)
arch/x86/kvm/i8254.c
712
struct kvm_pit *pit = kvm->arch.vpit;
arch/x86/kvm/i8254.c
735
struct kvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags)
arch/x86/kvm/i8254.c
759
pit->kvm = kvm;
arch/x86/kvm/i8254.c
772
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/i8254.c
774
ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, KVM_PIT_BASE_ADDRESS,
arch/x86/kvm/i8254.c
781
ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS,
arch/x86/kvm/i8254.c
787
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/i8254.c
792
kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS, &pit->dev);
arch/x86/kvm/i8254.c
794
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/i8254.c
802
void kvm_free_pit(struct kvm *kvm)
arch/x86/kvm/i8254.c
804
struct kvm_pit *pit = kvm->arch.vpit;
arch/x86/kvm/i8254.c
807
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/i8254.c
808
kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS, &pit->dev);
arch/x86/kvm/i8254.c
809
kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS, &pit->speaker_dev);
arch/x86/kvm/i8254.c
810
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/i8254.h
48
struct kvm *kvm;
arch/x86/kvm/i8254.h
62
int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps);
arch/x86/kvm/i8254.h
63
int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps);
arch/x86/kvm/i8254.h
64
int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps);
arch/x86/kvm/i8254.h
65
int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps);
arch/x86/kvm/i8254.h
66
int kvm_vm_ioctl_reinject(struct kvm *kvm, struct kvm_reinject_control *control);
arch/x86/kvm/i8254.h
68
struct kvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags);
arch/x86/kvm/i8254.h
69
void kvm_free_pit(struct kvm *kvm);
arch/x86/kvm/i8259.c
180
pic_irq_request(s->kvm, irq >= 0);
arch/x86/kvm/i8259.c
190
int kvm_pic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
arch/x86/kvm/i8259.c
193
struct kvm_pic *s = kvm->arch.vpic;
arch/x86/kvm/i8259.c
232
int kvm_pic_read_irq(struct kvm *kvm)
arch/x86/kvm/i8259.c
235
struct kvm_pic *s = kvm->arch.vpic;
arch/x86/kvm/i8259.c
288
kvm_for_each_vcpu(i, vcpu, s->pics_state->kvm)
arch/x86/kvm/i8259.c
371
s->pics_state->kvm,
arch/x86/kvm/i8259.c
44
static void pic_irq_request(struct kvm *kvm, int level);
arch/x86/kvm/i8259.c
562
static void pic_irq_request(struct kvm *kvm, int level)
arch/x86/kvm/i8259.c
564
struct kvm_pic *s = kvm->arch.vpic;
arch/x86/kvm/i8259.c
586
int kvm_pic_init(struct kvm *kvm)
arch/x86/kvm/i8259.c
595
s->kvm = kvm;
arch/x86/kvm/i8259.c
607
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/i8259.c
608
ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, 0x20, 2,
arch/x86/kvm/i8259.c
613
ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, 0xa0, 2, &s->dev_slave);
arch/x86/kvm/i8259.c
617
ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, 0x4d0, 2, &s->dev_elcr);
arch/x86/kvm/i8259.c
621
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/i8259.c
623
kvm->arch.vpic = s;
arch/x86/kvm/i8259.c
628
kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS, &s->dev_slave);
arch/x86/kvm/i8259.c
631
kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS, &s->dev_master);
arch/x86/kvm/i8259.c
634
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/i8259.c
64
kvm_for_each_vcpu(i, vcpu, s->kvm) {
arch/x86/kvm/i8259.c
641
void kvm_pic_destroy(struct kvm *kvm)
arch/x86/kvm/i8259.c
643
struct kvm_pic *vpic = kvm->arch.vpic;
arch/x86/kvm/i8259.c
648
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/i8259.c
649
kvm_io_bus_unregister_dev(vpic->kvm, KVM_PIO_BUS, &vpic->dev_master);
arch/x86/kvm/i8259.c
650
kvm_io_bus_unregister_dev(vpic->kvm, KVM_PIO_BUS, &vpic->dev_slave);
arch/x86/kvm/i8259.c
651
kvm_io_bus_unregister_dev(vpic->kvm, KVM_PIO_BUS, &vpic->dev_elcr);
arch/x86/kvm/i8259.c
652
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/i8259.c
654
kvm->arch.vpic = NULL;
arch/x86/kvm/i8259.c
86
kvm_notify_acked_irq(s->pics_state->kvm, SELECT_PIC(irq), irq);
arch/x86/kvm/ioapic.c
123
struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;
arch/x86/kvm/ioapic.c
139
kvm_for_each_vcpu(i, vcpu, ioapic->kvm)
arch/x86/kvm/ioapic.c
171
kvm_for_each_vcpu(i, vcpu, ioapic->kvm) {
arch/x86/kvm/ioapic.c
210
if (edge && kvm_apicv_activated(ioapic->kvm))
arch/x86/kvm/ioapic.c
261
struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;
arch/x86/kvm/ioapic.c
276
kvm_irq_has_notifier(ioapic->kvm, KVM_IRQCHIP_IOAPIC, index) ||
arch/x86/kvm/ioapic.c
287
void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm)
arch/x86/kvm/ioapic.c
289
if (!ioapic_in_kernel(kvm))
arch/x86/kvm/ioapic.c
291
kvm_make_scan_ioapic_request(kvm);
arch/x86/kvm/ioapic.c
294
void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
arch/x86/kvm/ioapic.c
297
struct kvm_ioapic *ioapic = kvm->arch.vioapic;
arch/x86/kvm/ioapic.c
299
mutex_lock(&kvm->irq_lock);
arch/x86/kvm/ioapic.c
302
mutex_unlock(&kvm->irq_lock);
arch/x86/kvm/ioapic.c
305
void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
arch/x86/kvm/ioapic.c
308
mutex_lock(&kvm->irq_lock);
arch/x86/kvm/ioapic.c
310
mutex_unlock(&kvm->irq_lock);
arch/x86/kvm/ioapic.c
311
synchronize_srcu(&kvm->irq_srcu);
arch/x86/kvm/ioapic.c
314
void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
arch/x86/kvm/ioapic.c
317
struct kvm_ioapic *ioapic = kvm->arch.vioapic;
arch/x86/kvm/ioapic.c
321
idx = srcu_read_lock(&kvm->irq_srcu);
arch/x86/kvm/ioapic.c
322
gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
arch/x86/kvm/ioapic.c
325
srcu_read_lock_held(&kvm->irq_srcu))
arch/x86/kvm/ioapic.c
328
srcu_read_unlock(&kvm->irq_srcu, idx);
arch/x86/kvm/ioapic.c
385
kvm_fire_mask_notifiers(ioapic->kvm, KVM_IRQCHIP_IOAPIC, index, mask_after);
arch/x86/kvm/ioapic.c
414
if (kvm_notify_irqfd_resampler(ioapic->kvm, KVM_IRQCHIP_IOAPIC, index))
arch/x86/kvm/ioapic.c
432
kvm_bitmap_or_dest_vcpus(ioapic->kvm, &irq,
arch/x86/kvm/ioapic.c
445
kvm_bitmap_or_dest_vcpus(ioapic->kvm, &irq,
arch/x86/kvm/ioapic.c
448
kvm_make_scan_ioapic_request_mask(ioapic->kvm,
arch/x86/kvm/ioapic.c
451
kvm_make_scan_ioapic_request(ioapic->kvm);
arch/x86/kvm/ioapic.c
488
ret = __kvm_irq_delivery_to_apic(ioapic->kvm, NULL, &irqe,
arch/x86/kvm/ioapic.c
492
ret = kvm_irq_delivery_to_apic(ioapic->kvm, NULL, &irqe);
arch/x86/kvm/ioapic.c
500
int kvm_ioapic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
arch/x86/kvm/ioapic.c
503
struct kvm_ioapic *ioapic = kvm->arch.vioapic;
arch/x86/kvm/ioapic.c
556
kvm_notify_acked_irq(ioapic->kvm, KVM_IRQCHIP_IOAPIC, pin);
arch/x86/kvm/ioapic.c
589
struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;
arch/x86/kvm/ioapic.c
717
int kvm_ioapic_init(struct kvm *kvm)
arch/x86/kvm/ioapic.c
728
kvm->arch.vioapic = ioapic;
arch/x86/kvm/ioapic.c
731
ioapic->kvm = kvm;
arch/x86/kvm/ioapic.c
732
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/ioapic.c
733
ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, ioapic->base_address,
arch/x86/kvm/ioapic.c
735
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/ioapic.c
737
kvm->arch.vioapic = NULL;
arch/x86/kvm/ioapic.c
744
void kvm_ioapic_destroy(struct kvm *kvm)
arch/x86/kvm/ioapic.c
746
struct kvm_ioapic *ioapic = kvm->arch.vioapic;
arch/x86/kvm/ioapic.c
752
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/ioapic.c
753
kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &ioapic->dev);
arch/x86/kvm/ioapic.c
754
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/ioapic.c
755
kvm->arch.vioapic = NULL;
arch/x86/kvm/ioapic.c
759
void kvm_get_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state)
arch/x86/kvm/ioapic.c
761
struct kvm_ioapic *ioapic = kvm->arch.vioapic;
arch/x86/kvm/ioapic.c
769
void kvm_set_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state)
arch/x86/kvm/ioapic.c
771
struct kvm_ioapic *ioapic = kvm->arch.vioapic;
arch/x86/kvm/ioapic.c
777
kvm_make_scan_ioapic_request(kvm);
arch/x86/kvm/ioapic.c
94
struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;
arch/x86/kvm/ioapic.h
101
void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
arch/x86/kvm/ioapic.h
107
int kvm_ioapic_init(struct kvm *kvm);
arch/x86/kvm/ioapic.h
108
void kvm_ioapic_destroy(struct kvm *kvm);
arch/x86/kvm/ioapic.h
109
int kvm_ioapic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
arch/x86/kvm/ioapic.h
11
struct kvm;
arch/x86/kvm/ioapic.h
112
void kvm_get_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state);
arch/x86/kvm/ioapic.h
113
void kvm_set_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state);
arch/x86/kvm/ioapic.h
118
static inline int ioapic_in_kernel(struct kvm *kvm)
arch/x86/kvm/ioapic.h
120
return irqchip_full(kvm);
arch/x86/kvm/ioapic.h
80
struct kvm *kvm;
arch/x86/kvm/ioapic.h
97
void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
arch/x86/kvm/ioapic.h
99
void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
arch/x86/kvm/irq.c
140
return v->kvm->arch.xen.upcall_vector;
arch/x86/kvm/irq.c
144
if (pic_in_kernel(v->kvm))
arch/x86/kvm/irq.c
145
return kvm_pic_read_irq(v->kvm); /* PIC */
arch/x86/kvm/irq.c
148
WARN_ON_ONCE(!irqchip_split(v->kvm));
arch/x86/kvm/irq.c
186
bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args)
arch/x86/kvm/irq.c
190
return resample ? irqchip_full(kvm) : irqchip_in_kernel(kvm);
arch/x86/kvm/irq.c
193
bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
arch/x86/kvm/irq.c
195
return irqchip_in_kernel(kvm);
arch/x86/kvm/irq.c
198
static void kvm_msi_to_lapic_irq(struct kvm *kvm,
arch/x86/kvm/irq.c
206
trace_kvm_msi_set_irq(msg.address_lo | (kvm->arch.x2apic_format ?
arch/x86/kvm/irq.c
209
irq->dest_id = x86_msi_msg_get_destid(&msg, kvm->arch.x2apic_format);
arch/x86/kvm/irq.c
219
static inline bool kvm_msi_route_invalid(struct kvm *kvm,
arch/x86/kvm/irq.c
222
return kvm->arch.x2apic_format && (e->msi.address_hi & 0xff);
arch/x86/kvm/irq.c
226
struct kvm *kvm, int irq_source_id, int level, bool line_status)
arch/x86/kvm/irq.c
230
if (kvm_msi_route_invalid(kvm, e))
arch/x86/kvm/irq.c
236
kvm_msi_to_lapic_irq(kvm, e, &irq);
arch/x86/kvm/irq.c
238
return kvm_irq_delivery_to_apic(kvm, NULL, &irq);
arch/x86/kvm/irq.c
242
struct kvm *kvm, int irq_source_id, int level,
arch/x86/kvm/irq.c
251
return kvm_hv_synic_set_irq(e, kvm, irq_source_id, level,
arch/x86/kvm/irq.c
256
if (kvm_msi_route_invalid(kvm, e))
arch/x86/kvm/irq.c
259
kvm_msi_to_lapic_irq(kvm, e, &irq);
arch/x86/kvm/irq.c
261
if (kvm_irq_delivery_to_apic_fast(kvm, NULL, &irq, &r))
arch/x86/kvm/irq.c
270
return kvm_xen_set_evtchn_fast(&e->xen_evtchn, kvm);
arch/x86/kvm/irq.c
279
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
arch/x86/kvm/irq.c
282
if (!irqchip_in_kernel(kvm))
arch/x86/kvm/irq.c
285
irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
arch/x86/kvm/irq.c
291
bool kvm_arch_can_set_irq_routing(struct kvm *kvm)
arch/x86/kvm/irq.c
293
return irqchip_in_kernel(kvm);
arch/x86/kvm/irq.c
296
int kvm_set_routing_entry(struct kvm *kvm,
arch/x86/kvm/irq.c
307
if (irqchip_split(kvm))
arch/x86/kvm/irq.c
336
if (kvm_msi_route_invalid(kvm, e))
arch/x86/kvm/irq.c
348
return kvm_xen_setup_evtchn(kvm, e, ue);
arch/x86/kvm/irq.c
387
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/irq.c
393
idx = srcu_read_lock(&kvm->irq_srcu);
arch/x86/kvm/irq.c
394
table = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
arch/x86/kvm/irq.c
396
kvm->arch.nr_reserved_ioapic_pins);
arch/x86/kvm/irq.c
404
kvm_msi_to_lapic_irq(vcpu->kvm, entry, &irq);
arch/x86/kvm/irq.c
413
srcu_read_unlock(&kvm->irq_srcu, idx);
arch/x86/kvm/irq.c
416
void kvm_arch_irq_routing_update(struct kvm *kvm)
arch/x86/kvm/irq.c
419
kvm_hv_irq_routing_update(kvm);
arch/x86/kvm/irq.c
422
if (irqchip_split(kvm))
arch/x86/kvm/irq.c
423
kvm_make_scan_ioapic_request(kvm);
arch/x86/kvm/irq.c
430
struct kvm *kvm = irqfd->kvm;
arch/x86/kvm/irq.c
435
if (WARN_ON_ONCE(!irqchip_in_kernel(kvm) || !kvm_arch_has_irq_bypass()))
arch/x86/kvm/irq.c
439
kvm_msi_to_lapic_irq(kvm, entry, &irq);
arch/x86/kvm/irq.c
453
if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
arch/x86/kvm/irq.c
461
r = kvm_x86_call(pi_update_irte)(irqfd, irqfd->kvm, host_irq, irqfd->gsi,
arch/x86/kvm/irq.c
480
struct kvm *kvm = irqfd->kvm;
arch/x86/kvm/irq.c
483
spin_lock_irq(&kvm->irqfds.lock);
arch/x86/kvm/irq.c
486
if (!kvm->arch.nr_possible_bypass_irqs++)
arch/x86/kvm/irq.c
487
kvm_x86_call(pi_start_bypass)(kvm);
arch/x86/kvm/irq.c
492
kvm->arch.nr_possible_bypass_irqs--;
arch/x86/kvm/irq.c
496
spin_unlock_irq(&kvm->irqfds.lock);
arch/x86/kvm/irq.c
506
struct kvm *kvm = irqfd->kvm;
arch/x86/kvm/irq.c
517
spin_lock_irq(&kvm->irqfds.lock);
arch/x86/kvm/irq.c
528
kvm->arch.nr_possible_bypass_irqs--;
arch/x86/kvm/irq.c
530
spin_unlock_irq(&kvm->irqfds.lock);
arch/x86/kvm/irq.c
576
int kvm_setup_default_ioapic_and_pic_routing(struct kvm *kvm)
arch/x86/kvm/irq.c
578
return kvm_set_irq_routing(kvm, default_routing,
arch/x86/kvm/irq.c
582
int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
arch/x86/kvm/irq.c
584
struct kvm_pic *pic = kvm->arch.vpic;
arch/x86/kvm/irq.c
602
kvm_get_ioapic(kvm, &chip->chip.ioapic);
arch/x86/kvm/irq.c
611
int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
arch/x86/kvm/irq.c
613
struct kvm_pic *pic = kvm->arch.vpic;
arch/x86/kvm/irq.c
631
kvm_set_ioapic(kvm, &chip->chip.ioapic);
arch/x86/kvm/irq.c
82
if (pic_in_kernel(v->kvm))
arch/x86/kvm/irq.c
83
return v->kvm->arch.vpic->output;
arch/x86/kvm/irq.c
86
WARN_ON_ONCE(!irqchip_split(v->kvm));
arch/x86/kvm/irq.h
106
static inline int irqchip_in_kernel(struct kvm *kvm)
arch/x86/kvm/irq.h
108
int mode = kvm->arch.irqchip_mode;
arch/x86/kvm/irq.h
27
struct kvm;
arch/x86/kvm/irq.h
55
struct kvm *kvm;
arch/x86/kvm/irq.h
64
int kvm_pic_init(struct kvm *kvm);
arch/x86/kvm/irq.h
65
void kvm_pic_destroy(struct kvm *kvm);
arch/x86/kvm/irq.h
66
int kvm_pic_read_irq(struct kvm *kvm);
arch/x86/kvm/irq.h
68
int kvm_pic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
arch/x86/kvm/irq.h
71
int kvm_setup_default_ioapic_and_pic_routing(struct kvm *kvm);
arch/x86/kvm/irq.h
73
int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip);
arch/x86/kvm/irq.h
74
int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip);
arch/x86/kvm/irq.h
76
static inline int irqchip_full(struct kvm *kvm)
arch/x86/kvm/irq.h
78
int mode = kvm->arch.irqchip_mode;
arch/x86/kvm/irq.h
85
static __always_inline int irqchip_full(struct kvm *kvm)
arch/x86/kvm/irq.h
91
static inline int pic_in_kernel(struct kvm *kvm)
arch/x86/kvm/irq.h
93
return irqchip_full(kvm);
arch/x86/kvm/irq.h
97
static inline int irqchip_split(struct kvm *kvm)
arch/x86/kvm/irq.h
99
int mode = kvm->arch.irqchip_mode;
arch/x86/kvm/kvm_onhyperv.c
102
return __hv_flush_remote_tlbs_range(kvm, &range);
arch/x86/kvm/kvm_onhyperv.c
106
int hv_flush_remote_tlbs(struct kvm *kvm)
arch/x86/kvm/kvm_onhyperv.c
108
return __hv_flush_remote_tlbs_range(kvm, NULL);
arch/x86/kvm/kvm_onhyperv.c
114
struct kvm_arch *kvm_arch = &vcpu->kvm->arch;
arch/x86/kvm/kvm_onhyperv.c
37
static int __hv_flush_remote_tlbs_range(struct kvm *kvm,
arch/x86/kvm/kvm_onhyperv.c
40
struct kvm_arch *kvm_arch = &kvm->arch;
arch/x86/kvm/kvm_onhyperv.c
56
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/kvm_onhyperv.c
95
int hv_flush_remote_tlbs_range(struct kvm *kvm, gfn_t start_gfn, gfn_t nr_pages)
arch/x86/kvm/kvm_onhyperv.h
10
int hv_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, gfn_t nr_pages);
arch/x86/kvm/kvm_onhyperv.h
11
int hv_flush_remote_tlbs(struct kvm *kvm);
arch/x86/kvm/kvm_onhyperv.h
23
&vcpu->kvm->arch.hv_pa_pg;
arch/x86/kvm/kvm_onhyperv.h
34
static inline int hv_flush_remote_tlbs(struct kvm *kvm)
arch/x86/kvm/lapic.c
1072
if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
arch/x86/kvm/lapic.c
108
static bool kvm_lapic_advertise_suppress_eoi_broadcast(struct kvm *kvm)
arch/x86/kvm/lapic.c
110
switch (kvm->arch.suppress_eoi_broadcast_mode) {
arch/x86/kvm/lapic.c
1112
static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
arch/x86/kvm/lapic.c
1114
if (!kvm->arch.disabled_lapic_found) {
arch/x86/kvm/lapic.c
1115
kvm->arch.disabled_lapic_found = true;
arch/x86/kvm/lapic.c
1120
static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
arch/x86/kvm/lapic.c
1123
if (kvm->arch.x2apic_broadcast_quirk_disabled) {
arch/x86/kvm/lapic.c
1156
static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
arch/x86/kvm/lapic.c
1170
if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
arch/x86/kvm/lapic.c
1211
kvm_apic_disabled_lapic_found(kvm);
arch/x86/kvm/lapic.c
1222
static bool __kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
arch/x86/kvm/lapic.c
1235
if (KVM_BUG_ON(!src, kvm)) {
arch/x86/kvm/lapic.c
1244
map = rcu_dereference(kvm->arch.apic_map);
arch/x86/kvm/lapic.c
1246
ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
arch/x86/kvm/lapic.c
1261
bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
arch/x86/kvm/lapic.c
1264
return __kvm_irq_delivery_to_apic_fast(kvm, src, irq, r, NULL);
arch/x86/kvm/lapic.c
1281
static bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm,
arch/x86/kvm/lapic.c
1294
map = rcu_dereference(kvm->arch.apic_map);
arch/x86/kvm/lapic.c
1296
if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
arch/x86/kvm/lapic.c
130
return !ioapic_in_kernel(kvm);
arch/x86/kvm/lapic.c
1310
bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq,
arch/x86/kvm/lapic.c
1317
if (kvm_intr_is_single_vcpu_fast(kvm, irq, dest_vcpu))
arch/x86/kvm/lapic.c
1320
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/lapic.c
1338
int __kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src,
arch/x86/kvm/lapic.c
1347
if (__kvm_irq_delivery_to_apic_fast(kvm, src, irq, &r, rtc_status))
arch/x86/kvm/lapic.c
1358
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/lapic.c
1387
lowest = kvm_get_vcpu(kvm, idx);
arch/x86/kvm/lapic.c
139
struct kvm *kvm = apic->vcpu->kvm;
arch/x86/kvm/lapic.c
144
switch (kvm->arch.suppress_eoi_broadcast_mode) {
arch/x86/kvm/lapic.c
1503
void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq,
arch/x86/kvm/lapic.c
1515
map = rcu_dereference(kvm->arch.apic_map);
arch/x86/kvm/lapic.c
1517
ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dest_vcpu,
arch/x86/kvm/lapic.c
1527
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/lapic.c
1563
if (irqchip_split(apic->vcpu->kvm)) {
arch/x86/kvm/lapic.c
158
return ioapic_in_kernel(kvm);
arch/x86/kvm/lapic.c
1654
kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq);
arch/x86/kvm/lapic.c
1674
return div64_u64(ns, (apic->vcpu->kvm->arch.apic_bus_cycle_ns *
arch/x86/kvm/lapic.c
1843
if (!kvm_check_has_quirk(vcpu->kvm,
arch/x86/kvm/lapic.c
191
(kvm_mwait_in_guest(vcpu->kvm) || kvm_hlt_in_guest(vcpu->kvm));
arch/x86/kvm/lapic.c
197
&& !(kvm_mwait_in_guest(vcpu->kvm) ||
arch/x86/kvm/lapic.c
2117
return (u64)tmict * apic->vcpu->kvm->arch.apic_bus_cycle_ns *
arch/x86/kvm/lapic.c
2410
atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
arch/x86/kvm/lapic.c
2412
atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
arch/x86/kvm/lapic.c
2572
kvm_recalculate_apic_map(apic->vcpu->kvm);
arch/x86/kvm/lapic.c
2588
if (!kvm_check_has_quirk(vcpu->kvm,
arch/x86/kvm/lapic.c
2639
if (!kvm_irq_delivery_to_apic_fast(apic->vcpu->kvm, apic, &irq,
arch/x86/kvm/lapic.c
2681
if (KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm))
arch/x86/kvm/lapic.c
2814
atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
arch/x86/kvm/lapic.c
2835
kvm_set_apicv_inhibit(apic->vcpu->kvm,
arch/x86/kvm/lapic.c
2861
kvm_recalculate_apic_map(vcpu->kvm);
arch/x86/kvm/lapic.c
2901
int kvm_alloc_apic_access_page(struct kvm *kvm)
arch/x86/kvm/lapic.c
2905
guard(mutex)(&kvm->slots_lock);
arch/x86/kvm/lapic.c
2907
if (kvm->arch.apic_access_memslot_enabled ||
arch/x86/kvm/lapic.c
2908
kvm->arch.apic_access_memslot_inhibited)
arch/x86/kvm/lapic.c
2911
hva = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
arch/x86/kvm/lapic.c
2916
kvm->arch.apic_access_memslot_enabled = true;
arch/x86/kvm/lapic.c
2924
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/lapic.c
2926
if (!kvm->arch.apic_access_memslot_enabled)
arch/x86/kvm/lapic.c
2931
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/lapic.c
2933
if (kvm->arch.apic_access_memslot_enabled) {
arch/x86/kvm/lapic.c
2934
__x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 0, 0);
arch/x86/kvm/lapic.c
2943
kvm->arch.apic_access_memslot_enabled = false;
arch/x86/kvm/lapic.c
2949
kvm->arch.apic_access_memslot_inhibited = true;
arch/x86/kvm/lapic.c
2952
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/lapic.c
2994
kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
arch/x86/kvm/lapic.c
300
if (vcpu->kvm->arch.x2apic_format) {
arch/x86/kvm/lapic.c
3030
kvm_recalculate_apic_map(vcpu->kvm);
arch/x86/kvm/lapic.c
3106
if (!irqchip_in_kernel(vcpu->kvm)) {
arch/x86/kvm/lapic.c
3250
if (vcpu->kvm->arch.x2apic_format) {
arch/x86/kvm/lapic.c
3318
kvm_recalculate_apic_map(vcpu->kvm);
arch/x86/kvm/lapic.c
3323
atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
arch/x86/kvm/lapic.c
3324
kvm_recalculate_apic_map(vcpu->kvm);
arch/x86/kvm/lapic.c
3341
if (ioapic_in_kernel(vcpu->kvm))
arch/x86/kvm/lapic.c
3407
if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
arch/x86/kvm/lapic.c
3460
kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
arch/x86/kvm/lapic.c
3467
if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
arch/x86/kvm/lapic.c
3568
ret = kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len);
arch/x86/kvm/lapic.c
405
static void kvm_recalculate_apic_map(struct kvm *kvm)
arch/x86/kvm/lapic.c
415
if (atomic_read_acquire(&kvm->arch.apic_map_dirty) == CLEAN)
arch/x86/kvm/lapic.c
418
WARN_ONCE(!irqchip_in_kernel(kvm),
arch/x86/kvm/lapic.c
421
mutex_lock(&kvm->arch.apic_map_lock);
arch/x86/kvm/lapic.c
431
if (atomic_cmpxchg_acquire(&kvm->arch.apic_map_dirty,
arch/x86/kvm/lapic.c
434
mutex_unlock(&kvm->arch.apic_map_lock);
arch/x86/kvm/lapic.c
447
kvm_for_each_vcpu(i, vcpu, kvm)
arch/x86/kvm/lapic.c
461
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/lapic.c
486
kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED);
arch/x86/kvm/lapic.c
488
kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED);
arch/x86/kvm/lapic.c
491
kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED);
arch/x86/kvm/lapic.c
493
kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED);
arch/x86/kvm/lapic.c
496
kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_APIC_ID_MODIFIED);
arch/x86/kvm/lapic.c
498
kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_APIC_ID_MODIFIED);
arch/x86/kvm/lapic.c
500
old = rcu_dereference_protected(kvm->arch.apic_map,
arch/x86/kvm/lapic.c
501
lockdep_is_held(&kvm->arch.apic_map_lock));
arch/x86/kvm/lapic.c
502
rcu_assign_pointer(kvm->arch.apic_map, new);
arch/x86/kvm/lapic.c
507
atomic_cmpxchg_release(&kvm->arch.apic_map_dirty,
arch/x86/kvm/lapic.c
509
mutex_unlock(&kvm->arch.apic_map_lock);
arch/x86/kvm/lapic.c
514
kvm_make_scan_ioapic_request(kvm);
arch/x86/kvm/lapic.c
530
atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
arch/x86/kvm/lapic.c
543
atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
arch/x86/kvm/lapic.c
549
atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
arch/x86/kvm/lapic.c
555
atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
arch/x86/kvm/lapic.c
566
atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
arch/x86/kvm/lapic.c
616
kvm_lapic_advertise_suppress_eoi_broadcast(vcpu->kvm))
arch/x86/kvm/lapic.c
770
KVM_BUG_ON(apic->isr_count > MAX_APIC_VECTOR, apic->vcpu->kvm);
arch/x86/kvm/lapic.c
811
KVM_BUG_ON(apic->isr_count < 0, apic->vcpu->kvm);
arch/x86/kvm/lapic.c
862
int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
arch/x86/kvm/lapic.c
880
map = rcu_dereference(kvm->arch.apic_map);
arch/x86/kvm/lapic.c
896
return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
arch/x86/kvm/lapic.c
903
return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
arch/x86/kvm/lapic.h
117
int kvm_alloc_apic_access_page(struct kvm *kvm);
arch/x86/kvm/lapic.h
120
bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
arch/x86/kvm/lapic.h
122
int __kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src,
arch/x86/kvm/lapic.h
126
static inline int kvm_irq_delivery_to_apic(struct kvm *kvm,
arch/x86/kvm/lapic.h
130
return __kvm_irq_delivery_to_apic(kvm, src, irq, NULL);
arch/x86/kvm/lapic.h
246
void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq,
arch/x86/kvm/lapic.h
249
bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq,
arch/x86/kvm/mmu.h
250
int kvm_mmu_post_init_vm(struct kvm *kvm);
arch/x86/kvm/mmu.h
251
void kvm_mmu_pre_destroy_vm(struct kvm *kvm);
arch/x86/kvm/mmu.h
253
static inline bool kvm_shadow_root_allocated(struct kvm *kvm)
arch/x86/kvm/mmu.h
261
return smp_load_acquire(&kvm->arch.shadow_root_allocated);
arch/x86/kvm/mmu.h
272
static inline bool kvm_memslots_have_rmaps(struct kvm *kvm)
arch/x86/kvm/mmu.h
274
return !tdp_mmu_enabled || kvm_shadow_root_allocated(kvm);
arch/x86/kvm/mmu.h
298
static inline void kvm_update_page_stats(struct kvm *kvm, int level, int count)
arch/x86/kvm/mmu.h
300
atomic64_add(count, &kvm->stat.pages[level - 1]);
arch/x86/kvm/mmu.h
316
static inline bool kvm_has_mirrored_tdp(const struct kvm *kvm)
arch/x86/kvm/mmu.h
318
return kvm->arch.vm_type == KVM_X86_TDX_VM;
arch/x86/kvm/mmu.h
321
static inline gfn_t kvm_gfn_direct_bits(const struct kvm *kvm)
arch/x86/kvm/mmu.h
323
return kvm->arch.gfn_direct_bits;
arch/x86/kvm/mmu.h
326
static inline bool kvm_is_addr_direct(struct kvm *kvm, gpa_t gpa)
arch/x86/kvm/mmu.h
328
gpa_t gpa_direct_bits = gfn_to_gpa(kvm_gfn_direct_bits(kvm));
arch/x86/kvm/mmu.h
333
static inline bool kvm_is_gfn_alias(struct kvm *kvm, gfn_t gfn)
arch/x86/kvm/mmu.h
335
return gfn & kvm_gfn_direct_bits(kvm);
arch/x86/kvm/mmu.h
94
void kvm_mmu_set_mmio_spte_value(struct kvm *kvm, u64 mmio_value);
arch/x86/kvm/mmu/mmu.c
1036
static int pte_list_add(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
arch/x86/kvm/mmu/mmu.c
1043
old_val = kvm_rmap_lock(kvm, rmap_head);
arch/x86/kvm/mmu/mmu.c
1075
kvm_rmap_unlock(kvm, rmap_head, new_val);
arch/x86/kvm/mmu/mmu.c
1080
static void pte_list_desc_remove_entry(struct kvm *kvm, unsigned long *rmap_val,
arch/x86/kvm/mmu/mmu.c
1091
KVM_BUG_ON_DATA_CORRUPTION(j < 0, kvm);
arch/x86/kvm/mmu/mmu.c
1116
static void pte_list_remove(struct kvm *kvm, u64 *spte,
arch/x86/kvm/mmu/mmu.c
1123
rmap_val = kvm_rmap_lock(kvm, rmap_head);
arch/x86/kvm/mmu/mmu.c
1124
if (KVM_BUG_ON_DATA_CORRUPTION(!rmap_val, kvm))
arch/x86/kvm/mmu/mmu.c
1128
if (KVM_BUG_ON_DATA_CORRUPTION((u64 *)rmap_val != spte, kvm))
arch/x86/kvm/mmu/mmu.c
1137
pte_list_desc_remove_entry(kvm, &rmap_val,
arch/x86/kvm/mmu/mmu.c
1145
KVM_BUG_ON_DATA_CORRUPTION(true, kvm);
arch/x86/kvm/mmu/mmu.c
1149
kvm_rmap_unlock(kvm, rmap_head, rmap_val);
arch/x86/kvm/mmu/mmu.c
1152
static void kvm_zap_one_rmap_spte(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1155
mmu_spte_clear_track_bits(kvm, sptep);
arch/x86/kvm/mmu/mmu.c
1156
pte_list_remove(kvm, sptep, rmap_head);
arch/x86/kvm/mmu/mmu.c
1160
static bool kvm_zap_all_rmap_sptes(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1167
rmap_val = kvm_rmap_lock(kvm, rmap_head);
arch/x86/kvm/mmu/mmu.c
1172
mmu_spte_clear_track_bits(kvm, (u64 *)rmap_val);
arch/x86/kvm/mmu/mmu.c
1180
mmu_spte_clear_track_bits(kvm, desc->sptes[i]);
arch/x86/kvm/mmu/mmu.c
1186
kvm_rmap_unlock(kvm, rmap_head, 0);
arch/x86/kvm/mmu/mmu.c
1213
static void rmap_remove(struct kvm *kvm, u64 *spte)
arch/x86/kvm/mmu/mmu.c
1229
slots = kvm_memslots_for_spte_role(kvm, sp->role);
arch/x86/kvm/mmu/mmu.c
1234
pte_list_remove(kvm, spte, rmap_head);
arch/x86/kvm/mmu/mmu.c
1311
static void drop_spte(struct kvm *kvm, u64 *sptep)
arch/x86/kvm/mmu/mmu.c
1313
u64 old_spte = mmu_spte_clear_track_bits(kvm, sptep);
arch/x86/kvm/mmu/mmu.c
1316
rmap_remove(kvm, sptep);
arch/x86/kvm/mmu/mmu.c
1375
static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
arch/x86/kvm/mmu/mmu.c
1393
static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1400
kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
arch/x86/kvm/mmu/mmu.c
1403
if (!kvm_memslots_have_rmaps(kvm))
arch/x86/kvm/mmu/mmu.c
1416
static void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1423
kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
arch/x86/kvm/mmu/mmu.c
1426
if (!kvm_memslots_have_rmaps(kvm))
arch/x86/kvm/mmu/mmu.c
1432
__rmap_clear_dirty(kvm, rmap_head, slot);
arch/x86/kvm/mmu/mmu.c
1439
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1454
if (kvm_dirty_log_manual_protect_and_init_set(kvm)) {
arch/x86/kvm/mmu/mmu.c
1459
kvm_mmu_try_split_huge_pages(kvm, slot, start, end + 1, PG_LEVEL_4K);
arch/x86/kvm/mmu/mmu.c
1461
kvm_mmu_slot_gfn_write_protect(kvm, slot, start, PG_LEVEL_2M);
arch/x86/kvm/mmu/mmu.c
1466
kvm_mmu_slot_gfn_write_protect(kvm, slot, end,
arch/x86/kvm/mmu/mmu.c
1480
if (kvm->arch.cpu_dirty_log_size)
arch/x86/kvm/mmu/mmu.c
1481
kvm_mmu_clear_dirty_pt_masked(kvm, slot, gfn_offset, mask);
arch/x86/kvm/mmu/mmu.c
1483
kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
arch/x86/kvm/mmu/mmu.c
1486
int kvm_cpu_dirty_log_size(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
1488
return kvm->arch.cpu_dirty_log_size;
arch/x86/kvm/mmu/mmu.c
1491
bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1499
if (kvm_memslots_have_rmaps(kvm)) {
arch/x86/kvm/mmu/mmu.c
1508
kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level);
arch/x86/kvm/mmu/mmu.c
1518
return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn, PG_LEVEL_4K);
arch/x86/kvm/mmu/mmu.c
1521
static bool kvm_zap_rmap(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
arch/x86/kvm/mmu/mmu.c
1524
return kvm_zap_all_rmap_sptes(kvm, rmap_head);
arch/x86/kvm/mmu/mmu.c
1597
typedef bool (*slot_rmaps_handler) (struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1601
static __always_inline bool __walk_slot_rmaps(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1611
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
1616
flush |= fn(kvm, iterator.rmap, slot);
arch/x86/kvm/mmu/mmu.c
1621
if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
arch/x86/kvm/mmu/mmu.c
1623
kvm_flush_remote_tlbs_range(kvm, start_gfn,
arch/x86/kvm/mmu/mmu.c
1627
cond_resched_rwlock_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
1634
static __always_inline bool walk_slot_rmaps(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1640
return __walk_slot_rmaps(kvm, slot, fn, start_level, end_level,
arch/x86/kvm/mmu/mmu.c
1645
static __always_inline bool walk_slot_rmaps_4k(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1650
return walk_slot_rmaps(kvm, slot, fn, PG_LEVEL_4K, PG_LEVEL_4K, flush_on_yield);
arch/x86/kvm/mmu/mmu.c
1653
static bool __kvm_rmap_zap_gfn_range(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1658
return __walk_slot_rmaps(kvm, slot, kvm_zap_rmap,
arch/x86/kvm/mmu/mmu.c
1663
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
arch/x86/kvm/mmu/mmu.c
1675
lockdep_assert_once(kvm->mmu_invalidate_in_progress ||
arch/x86/kvm/mmu/mmu.c
1676
lockdep_is_held(&kvm->slots_lock));
arch/x86/kvm/mmu/mmu.c
1678
if (kvm_memslots_have_rmaps(kvm))
arch/x86/kvm/mmu/mmu.c
1679
flush = __kvm_rmap_zap_gfn_range(kvm, range->slot,
arch/x86/kvm/mmu/mmu.c
1684
flush = kvm_tdp_mmu_unmap_gfn_range(kvm, range, flush);
arch/x86/kvm/mmu/mmu.c
1688
kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
arch/x86/kvm/mmu/mmu.c
1695
static void __rmap_add(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1706
kvm_update_page_stats(kvm, sp->role.level, 1);
arch/x86/kvm/mmu/mmu.c
1709
rmap_count = pte_list_add(kvm, cache, spte, rmap_head);
arch/x86/kvm/mmu/mmu.c
1711
if (rmap_count > kvm->stat.max_mmu_rmap_size)
arch/x86/kvm/mmu/mmu.c
1712
kvm->stat.max_mmu_rmap_size = rmap_count;
arch/x86/kvm/mmu/mmu.c
1714
kvm_zap_all_rmap_sptes(kvm, rmap_head);
arch/x86/kvm/mmu/mmu.c
1715
kvm_flush_remote_tlbs_gfn(kvm, gfn, sp->role.level);
arch/x86/kvm/mmu/mmu.c
1724
__rmap_add(vcpu->kvm, cache, slot, spte, gfn, access);
arch/x86/kvm/mmu/mmu.c
1727
static bool kvm_rmap_age_gfn_range(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1775
static bool kvm_may_have_shadow_mmu_sptes(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
1777
return !tdp_mmu_enabled || READ_ONCE(kvm->arch.indirect_shadow_pages);
arch/x86/kvm/mmu/mmu.c
1780
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/x86/kvm/mmu/mmu.c
1785
young = kvm_tdp_mmu_age_gfn_range(kvm, range);
arch/x86/kvm/mmu/mmu.c
1787
if (kvm_may_have_shadow_mmu_sptes(kvm))
arch/x86/kvm/mmu/mmu.c
1788
young |= kvm_rmap_age_gfn_range(kvm, range, false);
arch/x86/kvm/mmu/mmu.c
1793
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/x86/kvm/mmu/mmu.c
1798
young = kvm_tdp_mmu_test_age_gfn(kvm, range);
arch/x86/kvm/mmu/mmu.c
1803
if (kvm_may_have_shadow_mmu_sptes(kvm))
arch/x86/kvm/mmu/mmu.c
1804
young |= kvm_rmap_age_gfn_range(kvm, range, true);
arch/x86/kvm/mmu/mmu.c
1823
static void kvm_account_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/mmu.c
1825
kvm->stat.mmu_shadow_pages++;
arch/x86/kvm/mmu/mmu.c
1829
static void kvm_unaccount_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/mmu.c
1831
kvm->stat.mmu_shadow_pages--;
arch/x86/kvm/mmu/mmu.c
1851
static void mmu_page_add_parent_pte(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
1858
pte_list_add(kvm, cache, parent_pte, &sp->parent_ptes);
arch/x86/kvm/mmu/mmu.c
1861
static void mmu_page_remove_parent_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/mmu.c
1864
pte_list_remove(kvm, parent_pte, &sp->parent_ptes);
arch/x86/kvm/mmu/mmu.c
1867
static void drop_parent_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/mmu.c
187
static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/mmu.c
1870
mmu_page_remove_parent_pte(kvm, sp, parent_pte);
arch/x86/kvm/mmu/mmu.c
1982
static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/mmu.c
1987
--kvm->stat.mmu_unsync;
arch/x86/kvm/mmu/mmu.c
1990
static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/mmu.c
1992
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
2008
static struct hlist_head *kvm_get_mmu_page_hash(struct kvm *kvm, gfn_t gfn)
arch/x86/kvm/mmu/mmu.c
2018
struct hlist_head *page_hash = smp_load_acquire(&kvm->arch.mmu_page_hash);
arch/x86/kvm/mmu/mmu.c
2020
lockdep_assert_held(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
2113
kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
arch/x86/kvm/mmu/mmu.c
2117
static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
2125
kvm_mmu_commit_zap_page(kvm, invalid_list);
arch/x86/kvm/mmu/mmu.c
2127
kvm_flush_remote_tlbs(kvm);
arch/x86/kvm/mmu/mmu.c
2131
static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/mmu.c
2138
unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
arch/x86/kvm/mmu/mmu.c
2230
kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, true);
arch/x86/kvm/mmu/mmu.c
2235
kvm_unlink_unsync_page(vcpu->kvm, sp);
arch/x86/kvm/mmu/mmu.c
2239
if (need_resched() || rwlock_needbreak(&vcpu->kvm->mmu_lock)) {
arch/x86/kvm/mmu/mmu.c
2240
kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
arch/x86/kvm/mmu/mmu.c
2246
cond_resched_rwlock_write(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
2251
kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
arch/x86/kvm/mmu/mmu.c
2271
static struct kvm_mmu_page *kvm_mmu_find_shadow_page(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
2282
for_each_valid_sp(kvm, sp, sp_list) {
arch/x86/kvm/mmu/mmu.c
2299
kvm_mmu_prepare_zap_page(kvm, sp,
arch/x86/kvm/mmu/mmu.c
2309
if (KVM_BUG_ON(!vcpu, kvm))
arch/x86/kvm/mmu/mmu.c
2330
kvm_flush_remote_tlbs(kvm);
arch/x86/kvm/mmu/mmu.c
2339
++kvm->stat.mmu_cache_miss;
arch/x86/kvm/mmu/mmu.c
2342
kvm_mmu_commit_zap_page(kvm, &invalid_list);
arch/x86/kvm/mmu/mmu.c
2344
if (collisions > kvm->stat.max_mmu_page_hash_collisions)
arch/x86/kvm/mmu/mmu.c
2345
kvm->stat.max_mmu_page_hash_collisions = collisions;
arch/x86/kvm/mmu/mmu.c
2356
static struct kvm_mmu_page *kvm_mmu_alloc_shadow_page(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
2378
sp->mmu_valid_gen = kvm->arch.mmu_valid_gen;
arch/x86/kvm/mmu/mmu.c
2379
list_add(&sp->link, &kvm->arch.active_mmu_pages);
arch/x86/kvm/mmu/mmu.c
2380
kvm_account_mmu_page(kvm, sp);
arch/x86/kvm/mmu/mmu.c
2386
account_shadowed(kvm, sp);
arch/x86/kvm/mmu/mmu.c
2392
static struct kvm_mmu_page *__kvm_mmu_get_shadow_page(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
2407
BUG_ON(!kvm->arch.mmu_page_hash);
arch/x86/kvm/mmu/mmu.c
2408
sp_list = &kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)];
arch/x86/kvm/mmu/mmu.c
2410
sp = kvm_mmu_find_shadow_page(kvm, vcpu, gfn, sp_list, role);
arch/x86/kvm/mmu/mmu.c
2413
sp = kvm_mmu_alloc_shadow_page(kvm, caches, gfn, sp_list, role);
arch/x86/kvm/mmu/mmu.c
2430
return __kvm_mmu_get_shadow_page(vcpu->kvm, vcpu, &caches, gfn, role);
arch/x86/kvm/mmu/mmu.c
2575
static void __link_shadow_page(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
2590
if (mmu_page_zap_pte(kvm, parent_sp, sptep, &invalid_list))
arch/x86/kvm/mmu/mmu.c
2591
kvm_mmu_commit_zap_page(kvm, &invalid_list);
arch/x86/kvm/mmu/mmu.c
2593
kvm_flush_remote_tlbs_sptep(kvm, sptep);
arch/x86/kvm/mmu/mmu.c
2600
mmu_page_add_parent_pte(kvm, cache, sp, sptep);
arch/x86/kvm/mmu/mmu.c
2618
__link_shadow_page(vcpu->kvm, &vcpu->arch.mmu_pte_list_desc_cache, sptep, sp, true);
arch/x86/kvm/mmu/mmu.c
2638
drop_parent_pte(vcpu->kvm, child, sptep);
arch/x86/kvm/mmu/mmu.c
2639
kvm_flush_remote_tlbs_sptep(vcpu->kvm, sptep);
arch/x86/kvm/mmu/mmu.c
2644
static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/mmu.c
2653
drop_spte(kvm, spte);
arch/x86/kvm/mmu/mmu.c
2656
drop_parent_pte(kvm, child, spte);
arch/x86/kvm/mmu/mmu.c
2667
return kvm_mmu_prepare_zap_page(kvm, child,
arch/x86/kvm/mmu/mmu.c
2670
} else if (is_mmio_spte(kvm, pte)) {
arch/x86/kvm/mmu/mmu.c
2676
static int kvm_mmu_page_unlink_children(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
2684
zapped += mmu_page_zap_pte(kvm, sp, sp->spt + i, invalid_list);
arch/x86/kvm/mmu/mmu.c
2689
static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/mmu.c
2695
drop_parent_pte(kvm, sp, sptep);
arch/x86/kvm/mmu/mmu.c
2698
static int mmu_zap_unsync_children(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
2713
kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
arch/x86/kvm/mmu/mmu.c
2722
static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
2729
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
2731
++kvm->stat.mmu_shadow_zapped;
arch/x86/kvm/mmu/mmu.c
2732
*nr_zapped = mmu_zap_unsync_children(kvm, sp, invalid_list);
arch/x86/kvm/mmu/mmu.c
2733
*nr_zapped += kvm_mmu_page_unlink_children(kvm, sp, invalid_list);
arch/x86/kvm/mmu/mmu.c
2734
kvm_mmu_unlink_parents(kvm, sp);
arch/x86/kvm/mmu/mmu.c
2740
unaccount_shadowed(kvm, sp);
arch/x86/kvm/mmu/mmu.c
2743
kvm_unlink_unsync_page(kvm, sp);
arch/x86/kvm/mmu/mmu.c
2757
kvm_unaccount_mmu_page(kvm, sp);
arch/x86/kvm/mmu/mmu.c
2770
zapped_root = !is_obsolete_sp(kvm, sp);
arch/x86/kvm/mmu/mmu.c
2774
unaccount_nx_huge_page(kvm, sp);
arch/x86/kvm/mmu/mmu.c
2783
kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
arch/x86/kvm/mmu/mmu.c
2787
static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/mmu.c
2792
__kvm_mmu_prepare_zap_page(kvm, sp, invalid_list, &nr_zapped);
arch/x86/kvm/mmu/mmu.c
2796
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
2813
kvm_flush_remote_tlbs(kvm);
arch/x86/kvm/mmu/mmu.c
2821
static unsigned long kvm_mmu_zap_oldest_mmu_pages(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
2830
if (list_empty(&kvm->arch.active_mmu_pages))
arch/x86/kvm/mmu/mmu.c
2834
list_for_each_entry_safe_reverse(sp, tmp, &kvm->arch.active_mmu_pages, link) {
arch/x86/kvm/mmu/mmu.c
2842
unstable = __kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list,
arch/x86/kvm/mmu/mmu.c
2852
kvm_mmu_commit_zap_page(kvm, &invalid_list);
arch/x86/kvm/mmu/mmu.c
2854
kvm->stat.mmu_recycled += total_zapped;
arch/x86/kvm/mmu/mmu.c
2858
static inline unsigned long kvm_mmu_available_pages(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
2860
if (kvm->arch.n_max_mmu_pages > kvm->stat.mmu_shadow_pages)
arch/x86/kvm/mmu/mmu.c
2861
return kvm->arch.n_max_mmu_pages -
arch/x86/kvm/mmu/mmu.c
2862
kvm->stat.mmu_shadow_pages;
arch/x86/kvm/mmu/mmu.c
2869
unsigned long avail = kvm_mmu_available_pages(vcpu->kvm);
arch/x86/kvm/mmu/mmu.c
2874
kvm_mmu_zap_oldest_mmu_pages(vcpu->kvm, KVM_REFILL_PAGES - avail);
arch/x86/kvm/mmu/mmu.c
2885
if (!kvm_mmu_available_pages(vcpu->kvm))
arch/x86/kvm/mmu/mmu.c
2894
void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages)
arch/x86/kvm/mmu/mmu.c
2896
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
2898
if (kvm->stat.mmu_shadow_pages > goal_nr_mmu_pages) {
arch/x86/kvm/mmu/mmu.c
2899
kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->stat.mmu_shadow_pages -
arch/x86/kvm/mmu/mmu.c
2902
goal_nr_mmu_pages = kvm->stat.mmu_shadow_pages;
arch/x86/kvm/mmu/mmu.c
2905
kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
arch/x86/kvm/mmu/mmu.c
2907
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
2913
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/mmu/mmu.c
2927
if (!READ_ONCE(kvm->arch.indirect_shadow_pages))
arch/x86/kvm/mmu/mmu.c
293
static void kvm_flush_remote_tlbs_sptep(struct kvm *kvm, u64 *sptep)
arch/x86/kvm/mmu/mmu.c
2936
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
2937
for_each_gfn_valid_sp_with_gptes(kvm, sp, gpa_to_gfn(gpa))
arch/x86/kvm/mmu/mmu.c
2938
kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
arch/x86/kvm/mmu/mmu.c
2945
kvm_mmu_commit_zap_page(kvm, &invalid_list);
arch/x86/kvm/mmu/mmu.c
2946
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
2956
static void kvm_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/mmu.c
2959
++kvm->stat.mmu_unsync;
arch/x86/kvm/mmu/mmu.c
2971
int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot,
arch/x86/kvm/mmu/mmu.c
298
kvm_flush_remote_tlbs_gfn(kvm, gfn, sp->role.level);
arch/x86/kvm/mmu/mmu.c
2982
if (kvm_gfn_is_write_tracked(kvm, slot, gfn))
arch/x86/kvm/mmu/mmu.c
2991
if (kvm_gfn_is_lpage_allowed(kvm, slot, gfn, PG_LEVEL_2M))
arch/x86/kvm/mmu/mmu.c
3000
for_each_gfn_valid_sp_with_gptes(kvm, sp, gfn) {
arch/x86/kvm/mmu/mmu.c
3019
spin_lock(&kvm->arch.mmu_unsync_pages_lock);
arch/x86/kvm/mmu/mmu.c
3034
kvm_unsync_page(kvm, sp);
arch/x86/kvm/mmu/mmu.c
3037
spin_unlock(&kvm->arch.mmu_unsync_pages_lock);
arch/x86/kvm/mmu/mmu.c
3112
drop_parent_pte(vcpu->kvm, child, sptep);
arch/x86/kvm/mmu/mmu.c
3116
drop_spte(vcpu->kvm, sptep);
arch/x86/kvm/mmu/mmu.c
3126
kvm_flush_remote_tlbs_gfn(vcpu->kvm, gfn, level);
arch/x86/kvm/mmu/mmu.c
3144
kvm_flush_remote_tlbs_gfn(vcpu->kvm, gfn, level);
arch/x86/kvm/mmu/mmu.c
3251
if (unlikely(vcpu->kvm->mmu_invalidate_in_progress))
arch/x86/kvm/mmu/mmu.c
3282
static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn,
arch/x86/kvm/mmu/mmu.c
3317
pgd = READ_ONCE(*pgd_offset(kvm->mm, hva));
arch/x86/kvm/mmu/mmu.c
3363
static u8 kvm_gmem_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault,
arch/x86/kvm/mmu/mmu.c
3389
coco_level = kvm_x86_call(gmem_max_mapping_level)(kvm, pfn, is_private);
arch/x86/kvm/mmu/mmu.c
3396
int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault,
arch/x86/kvm/mmu/mmu.c
3403
lockdep_assert_held(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
3410
is_private = kvm_mem_is_private(kvm, gfn);
arch/x86/kvm/mmu/mmu.c
3424
host_level = kvm_gmem_max_mapping_level(kvm, fault, slot, gfn,
arch/x86/kvm/mmu/mmu.c
3427
host_level = host_pfn_mapping_level(kvm, gfn, slot);
arch/x86/kvm/mmu/mmu.c
3451
fault->req_level = kvm_mmu_max_mapping_level(vcpu->kvm, fault,
arch/x86/kvm/mmu/mmu.c
3516
account_nx_huge_page(vcpu->kvm, sp,
arch/x86/kvm/mmu/mmu.c
3601
static bool page_fault_can_be_fast(struct kvm *kvm, struct kvm_page_fault *fault)
arch/x86/kvm/mmu/mmu.c
3628
if (kvm->arch.has_private_mem &&
arch/x86/kvm/mmu/mmu.c
3629
fault->is_private != kvm_mem_is_private(kvm, fault->gfn))
arch/x86/kvm/mmu/mmu.c
3681
mark_page_dirty_in_slot(vcpu->kvm, fault->slot, fault->gfn);
arch/x86/kvm/mmu/mmu.c
3720
if (!page_fault_can_be_fast(vcpu->kvm, fault))
arch/x86/kvm/mmu/mmu.c
3835
static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa,
arch/x86/kvm/mmu/mmu.c
3848
lockdep_assert_held_read(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
3849
kvm_tdp_mmu_put_root(kvm, sp);
arch/x86/kvm/mmu/mmu.c
3851
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
3853
kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
arch/x86/kvm/mmu/mmu.c
3860
void kvm_mmu_free_roots(struct kvm *kvm, struct kvm_mmu *mmu,
arch/x86/kvm/mmu/mmu.c
3887
read_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
3889
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
3893
mmu_free_root_page(kvm, &mmu->prev_roots[i].hpa,
arch/x86/kvm/mmu/mmu.c
3900
mmu_free_root_page(kvm, &mmu->root.hpa, &invalid_list);
arch/x86/kvm/mmu/mmu.c
3906
mmu_free_root_page(kvm, &mmu->pae_root[i],
arch/x86/kvm/mmu/mmu.c
3916
read_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
3919
kvm_mmu_commit_zap_page(kvm, &invalid_list);
arch/x86/kvm/mmu/mmu.c
3920
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
3925
void kvm_mmu_free_guest_mode_roots(struct kvm *kvm, struct kvm_mmu *mmu)
arch/x86/kvm/mmu/mmu.c
3948
kvm_mmu_free_roots(kvm, mmu, roots_to_free);
arch/x86/kvm/mmu/mmu.c
3979
if (kvm_has_mirrored_tdp(vcpu->kvm) &&
arch/x86/kvm/mmu/mmu.c
3986
write_lock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
4018
write_unlock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
4022
static int kvm_mmu_alloc_page_hash(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
4026
if (kvm->arch.mmu_page_hash)
arch/x86/kvm/mmu/mmu.c
4040
smp_store_release(&kvm->arch.mmu_page_hash, h);
arch/x86/kvm/mmu/mmu.c
4044
static int mmu_first_shadow_root_alloc(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
4054
if (kvm_shadow_root_allocated(kvm))
arch/x86/kvm/mmu/mmu.c
4057
mutex_lock(&kvm->slots_arch_lock);
arch/x86/kvm/mmu/mmu.c
4060
if (kvm_shadow_root_allocated(kvm))
arch/x86/kvm/mmu/mmu.c
4063
r = kvm_mmu_alloc_page_hash(kvm);
arch/x86/kvm/mmu/mmu.c
4071
if (kvm_memslots_have_rmaps(kvm) &&
arch/x86/kvm/mmu/mmu.c
4072
kvm_page_track_write_tracking_enabled(kvm))
arch/x86/kvm/mmu/mmu.c
4075
for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
arch/x86/kvm/mmu/mmu.c
4076
slots = __kvm_memslots(kvm, i);
arch/x86/kvm/mmu/mmu.c
4102
smp_store_release(&kvm->arch.shadow_root_allocated, true);
arch/x86/kvm/mmu/mmu.c
4105
mutex_unlock(&kvm->slots_arch_lock);
arch/x86/kvm/mmu/mmu.c
4140
r = mmu_first_shadow_root_alloc(vcpu->kvm);
arch/x86/kvm/mmu/mmu.c
4144
write_lock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
4222
write_unlock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
4355
write_lock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
4357
write_unlock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
4361
write_lock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
4372
write_unlock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
4385
kvm_mmu_free_roots(vcpu->kvm, vcpu->arch.mmu, roots_to_free);
arch/x86/kvm/mmu/mmu.c
4514
if (is_mmio_spte(vcpu->kvm, spte)) {
arch/x86/kvm/mmu/mmu.c
4549
if (kvm_gfn_is_write_tracked(vcpu->kvm, fault->slot, fault->gfn))
arch/x86/kvm/mmu/mmu.c
4629
kvm_release_faultin_page(vcpu->kvm, fault->refcounted_page,
arch/x86/kvm/mmu/mmu.c
4643
r = kvm_gmem_get_pfn(vcpu->kvm, fault->slot, fault->gfn, &fault->pfn,
arch/x86/kvm/mmu/mmu.c
4705
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/mmu/mmu.c
4708
if (KVM_BUG_ON(kvm_is_gfn_alias(kvm, fault->gfn), kvm))
arch/x86/kvm/mmu/mmu.c
4717
fault->mmu_seq = vcpu->kvm->mmu_invalidate_seq;
arch/x86/kvm/mmu/mmu.c
4724
if (fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) {
arch/x86/kvm/mmu/mmu.c
4767
if (!kvm_apicv_activated(vcpu->kvm))
arch/x86/kvm/mmu/mmu.c
4792
if (mmu_invalidate_retry_gfn_unsafe(kvm, fault->mmu_seq, fault->gfn))
arch/x86/kvm/mmu/mmu.c
4812
if (mmu_invalidate_retry_gfn_unsafe(kvm, fault->mmu_seq, fault->gfn)) {
arch/x86/kvm/mmu/mmu.c
4830
if (sp && is_obsolete_sp(vcpu->kvm, sp))
arch/x86/kvm/mmu/mmu.c
4850
mmu_invalidate_retry_gfn(vcpu->kvm, fault->mmu_seq, fault->gfn);
arch/x86/kvm/mmu/mmu.c
4876
write_lock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
4891
write_unlock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
4969
read_lock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
4978
read_unlock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
5046
if (!vcpu->kvm->arch.pre_fault_allowed)
arch/x86/kvm/mmu/mmu.c
5049
if (kvm_is_gfn_alias(vcpu->kvm, gpa_to_gfn(range->gpa)))
arch/x86/kvm/mmu/mmu.c
5061
if (kvm_arch_has_private_mem(vcpu->kvm) &&
arch/x86/kvm/mmu/mmu.c
5062
kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(range->gpa)))
arch/x86/kvm/mmu/mmu.c
5065
direct_bits = gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm));
arch/x86/kvm/mmu/mmu.c
5103
.nx_huge_page_workaround_enabled = is_nx_huge_page_enabled(vcpu->kvm),
arch/x86/kvm/mmu/mmu.c
5115
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/mmu/mmu.c
5118
lockdep_assert_held(&kvm->slots_lock);
arch/x86/kvm/mmu/mmu.c
5127
if (KVM_BUG_ON(!tdp_mmu_enabled, kvm))
arch/x86/kvm/mmu/mmu.c
5130
if (kvm_gfn_is_write_tracked(kvm, fault.slot, fault.gfn))
arch/x86/kvm/mmu/mmu.c
5150
guard(read_lock)(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
5196
static bool cached_root_find_and_keep_current(struct kvm *kvm, struct kvm_mmu *mmu,
arch/x86/kvm/mmu/mmu.c
5219
kvm_mmu_free_roots(kvm, mmu, KVM_MMU_ROOT_CURRENT);
arch/x86/kvm/mmu/mmu.c
5230
static bool cached_root_find_without_current(struct kvm *kvm, struct kvm_mmu *mmu,
arch/x86/kvm/mmu/mmu.c
5251
static bool fast_pgd_switch(struct kvm *kvm, struct kvm_mmu *mmu,
arch/x86/kvm/mmu/mmu.c
5259
kvm_mmu_free_roots(kvm, mmu, KVM_MMU_ROOT_CURRENT);
arch/x86/kvm/mmu/mmu.c
5262
return cached_root_find_and_keep_current(kvm, mmu, new_pgd, new_role);
arch/x86/kvm/mmu/mmu.c
5264
return cached_root_find_without_current(kvm, mmu, new_pgd, new_role);
arch/x86/kvm/mmu/mmu.c
5276
if (!fast_pgd_switch(vcpu->kvm, mmu, new_pgd, new_role))
arch/x86/kvm/mmu/mmu.c
531
static u64 mmu_spte_clear_track_bits(struct kvm *kvm, u64 *sptep)
arch/x86/kvm/mmu/mmu.c
5316
if (unlikely(is_mmio_spte(vcpu->kvm, *sptep))) {
arch/x86/kvm/mmu/mmu.c
545
kvm_update_page_stats(kvm, level, -1);
arch/x86/kvm/mmu/mmu.c
5868
if (vcpu->kvm->arch.vm_type == KVM_X86_TDX_VM)
arch/x86/kvm/mmu/mmu.c
5896
role.cr4_smep = kvm_x86_call(tdp_has_smep)(vcpu->kvm);
arch/x86/kvm/mmu/mmu.c
612
if (kvm_has_mirrored_tdp(vcpu->kvm)) {
arch/x86/kvm/mmu/mmu.c
6158
KVM_BUG_ON(!kvm_can_set_cpuid_and_feature_msrs(vcpu), vcpu->kvm);
arch/x86/kvm/mmu/mmu.c
6203
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/mmu/mmu.c
6205
kvm_mmu_free_roots(kvm, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);
arch/x86/kvm/mmu/mmu.c
6207
kvm_mmu_free_roots(kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
arch/x86/kvm/mmu/mmu.c
6212
static bool is_obsolete_root(struct kvm *kvm, hpa_t root_hpa)
arch/x86/kvm/mmu/mmu.c
6236
return !sp || is_obsolete_sp(kvm, sp);
arch/x86/kvm/mmu/mmu.c
6239
static void __kvm_mmu_free_obsolete_roots(struct kvm *kvm, struct kvm_mmu *mmu)
arch/x86/kvm/mmu/mmu.c
6244
if (is_obsolete_root(kvm, mmu->root.hpa))
arch/x86/kvm/mmu/mmu.c
6248
if (is_obsolete_root(kvm, mmu->prev_roots[i].hpa))
arch/x86/kvm/mmu/mmu.c
6253
kvm_mmu_free_roots(kvm, mmu, roots_to_free);
arch/x86/kvm/mmu/mmu.c
6258
__kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.root_mmu);
arch/x86/kvm/mmu/mmu.c
6259
__kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.guest_mmu);
arch/x86/kvm/mmu/mmu.c
6381
if (!vcpu->kvm->arch.indirect_shadow_pages)
arch/x86/kvm/mmu/mmu.c
6384
write_lock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
6388
++vcpu->kvm->stat.mmu_pte_write;
arch/x86/kvm/mmu/mmu.c
6390
for_each_gfn_valid_sp_with_gptes(vcpu->kvm, sp, gfn) {
arch/x86/kvm/mmu/mmu.c
6393
kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
arch/x86/kvm/mmu/mmu.c
6394
++vcpu->kvm->stat.mmu_flooded;
arch/x86/kvm/mmu/mmu.c
6404
mmu_page_zap_pte(vcpu->kvm, sp, spte, &invalid_list);
arch/x86/kvm/mmu/mmu.c
6406
++vcpu->kvm->stat.mmu_pde_zapped;
arch/x86/kvm/mmu/mmu.c
6412
kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
arch/x86/kvm/mmu/mmu.c
6413
write_unlock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
6535
vcpu->kvm->arch.vm_type == KVM_X86_SW_PROTECTED_VM &&
arch/x86/kvm/mmu/mmu.c
6536
kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(cr2_or_gpa)))
arch/x86/kvm/mmu/mmu.c
6554
if (KVM_BUG_ON(r == RET_PF_INVALID, vcpu->kvm))
arch/x86/kvm/mmu/mmu.c
6623
write_lock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
6631
mmu_page_zap_pte(vcpu->kvm, sp, iterator.sptep, NULL);
arch/x86/kvm/mmu/mmu.c
6633
kvm_flush_remote_tlbs_sptep(vcpu->kvm, iterator.sptep);
arch/x86/kvm/mmu/mmu.c
6639
write_unlock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
6837
static void kvm_zap_obsolete_pages(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
6844
lockdep_assert_held(&kvm->slots_lock);
arch/x86/kvm/mmu/mmu.c
6848
&kvm->arch.active_mmu_pages, link) {
arch/x86/kvm/mmu/mmu.c
6853
if (!is_obsolete_sp(kvm, sp))
arch/x86/kvm/mmu/mmu.c
6871
cond_resched_rwlock_write(&kvm->mmu_lock)) {
arch/x86/kvm/mmu/mmu.c
6876
unstable = __kvm_mmu_prepare_zap_page(kvm, sp,
arch/x86/kvm/mmu/mmu.c
6893
kvm_mmu_commit_zap_page(kvm, &invalid_list);
arch/x86/kvm/mmu/mmu.c
6905
static void kvm_mmu_zap_all_fast(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
6907
lockdep_assert_held(&kvm->slots_lock);
arch/x86/kvm/mmu/mmu.c
6909
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
6910
trace_kvm_mmu_zap_all_fast(kvm);
arch/x86/kvm/mmu/mmu.c
6919
kvm->arch.mmu_valid_gen = kvm->arch.mmu_valid_gen ? 0 : 1;
arch/x86/kvm/mmu/mmu.c
6932
kvm_tdp_mmu_invalidate_roots(kvm, KVM_DIRECT_ROOTS);
arch/x86/kvm/mmu/mmu.c
6943
kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
arch/x86/kvm/mmu/mmu.c
6945
kvm_zap_obsolete_pages(kvm);
arch/x86/kvm/mmu/mmu.c
6947
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
6958
kvm_tdp_mmu_zap_invalidated_roots(kvm, true);
arch/x86/kvm/mmu/mmu.c
6961
int kvm_mmu_init_vm(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
6965
kvm->arch.shadow_mmio_value = shadow_mmio_value;
arch/x86/kvm/mmu/mmu.c
6966
INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
arch/x86/kvm/mmu/mmu.c
6968
INIT_LIST_HEAD(&kvm->arch.possible_nx_huge_pages[i].pages);
arch/x86/kvm/mmu/mmu.c
6969
spin_lock_init(&kvm->arch.mmu_unsync_pages_lock);
arch/x86/kvm/mmu/mmu.c
6972
kvm_mmu_init_tdp_mmu(kvm);
arch/x86/kvm/mmu/mmu.c
6974
r = kvm_mmu_alloc_page_hash(kvm);
arch/x86/kvm/mmu/mmu.c
6979
kvm->arch.split_page_header_cache.kmem_cache = mmu_page_header_cache;
arch/x86/kvm/mmu/mmu.c
6980
kvm->arch.split_page_header_cache.gfp_zero = __GFP_ZERO;
arch/x86/kvm/mmu/mmu.c
6982
kvm->arch.split_shadow_page_cache.gfp_zero = __GFP_ZERO;
arch/x86/kvm/mmu/mmu.c
6984
kvm->arch.split_desc_cache.kmem_cache = pte_list_desc_cache;
arch/x86/kvm/mmu/mmu.c
6985
kvm->arch.split_desc_cache.gfp_zero = __GFP_ZERO;
arch/x86/kvm/mmu/mmu.c
6989
static void mmu_free_vm_memory_caches(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
6991
kvm_mmu_free_memory_cache(&kvm->arch.split_desc_cache);
arch/x86/kvm/mmu/mmu.c
6992
kvm_mmu_free_memory_cache(&kvm->arch.split_page_header_cache);
arch/x86/kvm/mmu/mmu.c
6993
kvm_mmu_free_memory_cache(&kvm->arch.split_shadow_page_cache);
arch/x86/kvm/mmu/mmu.c
6996
void kvm_mmu_uninit_vm(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
6998
kvfree(kvm->arch.mmu_page_hash);
arch/x86/kvm/mmu/mmu.c
7001
kvm_mmu_uninit_tdp_mmu(kvm);
arch/x86/kvm/mmu/mmu.c
7003
mmu_free_vm_memory_caches(kvm);
arch/x86/kvm/mmu/mmu.c
7006
static bool kvm_rmap_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
arch/x86/kvm/mmu/mmu.c
7015
if (!kvm_memslots_have_rmaps(kvm))
arch/x86/kvm/mmu/mmu.c
7018
for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
arch/x86/kvm/mmu/mmu.c
7019
slots = __kvm_memslots(kvm, i);
arch/x86/kvm/mmu/mmu.c
7028
flush = __kvm_rmap_zap_gfn_range(kvm, memslot, start,
arch/x86/kvm/mmu/mmu.c
7040
void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
arch/x86/kvm/mmu/mmu.c
7047
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7049
kvm_mmu_invalidate_start(kvm);
arch/x86/kvm/mmu/mmu.c
7051
kvm_mmu_invalidate_range_add(kvm, gfn_start, gfn_end);
arch/x86/kvm/mmu/mmu.c
7053
flush = kvm_rmap_zap_gfn_range(kvm, gfn_start, gfn_end);
arch/x86/kvm/mmu/mmu.c
7056
flush = kvm_tdp_mmu_zap_leafs(kvm, gfn_start, gfn_end, flush);
arch/x86/kvm/mmu/mmu.c
7059
kvm_flush_remote_tlbs_range(kvm, gfn_start, gfn_end - gfn_start);
arch/x86/kvm/mmu/mmu.c
7061
kvm_mmu_invalidate_end(kvm);
arch/x86/kvm/mmu/mmu.c
7063
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7067
static bool slot_rmap_write_protect(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7074
void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7078
if (kvm_memslots_have_rmaps(kvm)) {
arch/x86/kvm/mmu/mmu.c
7079
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7080
walk_slot_rmaps(kvm, memslot, slot_rmap_write_protect,
arch/x86/kvm/mmu/mmu.c
7082
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7086
read_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7087
kvm_tdp_mmu_wrprot_slot(kvm, memslot, start_level);
arch/x86/kvm/mmu/mmu.c
7088
read_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7097
static bool need_topup_split_caches_or_resched(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
7099
if (need_resched() || rwlock_needbreak(&kvm->mmu_lock))
arch/x86/kvm/mmu/mmu.c
7107
return need_topup(&kvm->arch.split_desc_cache, SPLIT_DESC_CACHE_MIN_NR_OBJECTS) ||
arch/x86/kvm/mmu/mmu.c
7108
need_topup(&kvm->arch.split_page_header_cache, 1) ||
arch/x86/kvm/mmu/mmu.c
7109
need_topup(&kvm->arch.split_shadow_page_cache, 1);
arch/x86/kvm/mmu/mmu.c
7112
static int topup_split_caches(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
7131
lockdep_assert_held(&kvm->slots_lock);
arch/x86/kvm/mmu/mmu.c
7133
r = __kvm_mmu_topup_memory_cache(&kvm->arch.split_desc_cache, capacity,
arch/x86/kvm/mmu/mmu.c
7138
r = kvm_mmu_topup_memory_cache(&kvm->arch.split_page_header_cache, 1);
arch/x86/kvm/mmu/mmu.c
7142
return kvm_mmu_topup_memory_cache(&kvm->arch.split_shadow_page_cache, 1);
arch/x86/kvm/mmu/mmu.c
7145
static struct kvm_mmu_page *shadow_mmu_get_sp_for_split(struct kvm *kvm, u64 *huge_sptep)
arch/x86/kvm/mmu/mmu.c
7165
caches.page_header_cache = &kvm->arch.split_page_header_cache;
arch/x86/kvm/mmu/mmu.c
7166
caches.shadow_page_cache = &kvm->arch.split_shadow_page_cache;
arch/x86/kvm/mmu/mmu.c
7169
return __kvm_mmu_get_shadow_page(kvm, NULL, &caches, gfn, role);
arch/x86/kvm/mmu/mmu.c
7172
static void shadow_mmu_split_huge_page(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7177
struct kvm_mmu_memory_cache *cache = &kvm->arch.split_desc_cache;
arch/x86/kvm/mmu/mmu.c
7185
sp = shadow_mmu_get_sp_for_split(kvm, huge_sptep);
arch/x86/kvm/mmu/mmu.c
7210
spte = make_small_spte(kvm, huge_spte, sp->role, index);
arch/x86/kvm/mmu/mmu.c
7212
__rmap_add(kvm, cache, slot, sptep, gfn, sp->role.access);
arch/x86/kvm/mmu/mmu.c
7215
__link_shadow_page(kvm, cache, huge_sptep, sp, flush);
arch/x86/kvm/mmu/mmu.c
7218
static int shadow_mmu_try_split_huge_page(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7232
if (kvm_mmu_available_pages(kvm) <= KVM_MIN_FREE_MMU_PAGES) {
arch/x86/kvm/mmu/mmu.c
7237
if (need_topup_split_caches_or_resched(kvm)) {
arch/x86/kvm/mmu/mmu.c
7238
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7245
r = topup_split_caches(kvm) ?: -EAGAIN;
arch/x86/kvm/mmu/mmu.c
7246
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
725
static bool kvm_gfn_is_lpage_allowed(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7250
shadow_mmu_split_huge_page(kvm, slot, huge_sptep);
arch/x86/kvm/mmu/mmu.c
7257
static bool shadow_mmu_try_split_huge_pages(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7286
r = shadow_mmu_try_split_huge_page(kvm, slot, huge_sptep);
arch/x86/kvm/mmu/mmu.c
7303
static void kvm_shadow_mmu_try_split_huge_pages(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7317
__walk_slot_rmaps(kvm, slot, shadow_mmu_try_split_huge_pages,
arch/x86/kvm/mmu/mmu.c
7322
void kvm_mmu_try_split_huge_pages(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7330
if (kvm_memslots_have_rmaps(kvm))
arch/x86/kvm/mmu/mmu.c
7331
kvm_shadow_mmu_try_split_huge_pages(kvm, memslot, start, end, target_level);
arch/x86/kvm/mmu/mmu.c
7333
kvm_tdp_mmu_try_split_huge_pages(kvm, memslot, start, end, target_level, false);
arch/x86/kvm/mmu/mmu.c
7341
void kvm_mmu_slot_try_split_huge_pages(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7351
if (kvm_memslots_have_rmaps(kvm)) {
arch/x86/kvm/mmu/mmu.c
7352
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7353
kvm_shadow_mmu_try_split_huge_pages(kvm, memslot, start, end, target_level);
arch/x86/kvm/mmu/mmu.c
7354
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7357
read_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7358
kvm_tdp_mmu_try_split_huge_pages(kvm, memslot, start, end, target_level, true);
arch/x86/kvm/mmu/mmu.c
7359
read_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
736
if (kvm_arch_nr_memslot_as_ids(kvm) > 1) {
arch/x86/kvm/mmu/mmu.c
737
other_slot = __gfn_to_memslot(__kvm_memslots(kvm, slot->as_id ^ 1), gfn);
arch/x86/kvm/mmu/mmu.c
7372
static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7399
sp->role.level < kvm_mmu_max_mapping_level(kvm, NULL, slot, sp->gfn)) {
arch/x86/kvm/mmu/mmu.c
7400
kvm_zap_one_rmap_spte(kvm, rmap_head, sptep);
arch/x86/kvm/mmu/mmu.c
7403
kvm_flush_remote_tlbs_sptep(kvm, sptep);
arch/x86/kvm/mmu/mmu.c
7414
static void kvm_rmap_zap_collapsible_sptes(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7421
if (walk_slot_rmaps(kvm, slot, kvm_mmu_zap_collapsible_spte,
arch/x86/kvm/mmu/mmu.c
7423
kvm_flush_remote_tlbs_memslot(kvm, slot);
arch/x86/kvm/mmu/mmu.c
7426
void kvm_mmu_recover_huge_pages(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7429
if (kvm_memslots_have_rmaps(kvm)) {
arch/x86/kvm/mmu/mmu.c
7430
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7431
kvm_rmap_zap_collapsible_sptes(kvm, slot);
arch/x86/kvm/mmu/mmu.c
7432
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7436
read_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7437
kvm_tdp_mmu_recover_huge_pages(kvm, slot);
arch/x86/kvm/mmu/mmu.c
7438
read_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7442
void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7445
if (kvm_memslots_have_rmaps(kvm)) {
arch/x86/kvm/mmu/mmu.c
7446
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7451
walk_slot_rmaps_4k(kvm, memslot, __rmap_clear_dirty, false);
arch/x86/kvm/mmu/mmu.c
7452
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7456
read_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7457
kvm_tdp_mmu_clear_dirty_slot(kvm, memslot);
arch/x86/kvm/mmu/mmu.c
7458
read_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7471
static void kvm_mmu_zap_all(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
7477
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7479
list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {
arch/x86/kvm/mmu/mmu.c
7482
if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign))
arch/x86/kvm/mmu/mmu.c
7484
if (cond_resched_rwlock_write(&kvm->mmu_lock))
arch/x86/kvm/mmu/mmu.c
7488
kvm_mmu_commit_zap_page(kvm, &invalid_list);
arch/x86/kvm/mmu/mmu.c
7491
kvm_tdp_mmu_zap_all(kvm);
arch/x86/kvm/mmu/mmu.c
7493
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7496
void kvm_arch_flush_shadow_all(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
7498
kvm_mmu_zap_all(kvm);
arch/x86/kvm/mmu/mmu.c
7501
static void kvm_mmu_zap_memslot_pages_and_flush(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7508
if (list_empty(&kvm->arch.active_mmu_pages))
arch/x86/kvm/mmu/mmu.c
7521
for_each_gfn_valid_sp_with_gptes(kvm, sp, gfn)
arch/x86/kvm/mmu/mmu.c
7522
kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
arch/x86/kvm/mmu/mmu.c
7524
if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
arch/x86/kvm/mmu/mmu.c
7525
kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
arch/x86/kvm/mmu/mmu.c
7527
cond_resched_rwlock_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7532
kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
arch/x86/kvm/mmu/mmu.c
7535
static void kvm_mmu_zap_memslot(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7547
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7548
flush = kvm_unmap_gfn_range(kvm, &range);
arch/x86/kvm/mmu/mmu.c
7549
kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush);
arch/x86/kvm/mmu/mmu.c
7550
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7553
static inline bool kvm_memslot_flush_zap_all(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
7555
return kvm->arch.vm_type == KVM_X86_DEFAULT_VM &&
arch/x86/kvm/mmu/mmu.c
7556
kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL);
arch/x86/kvm/mmu/mmu.c
7559
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7562
if (kvm_memslot_flush_zap_all(kvm))
arch/x86/kvm/mmu/mmu.c
7563
kvm_mmu_zap_all_fast(kvm);
arch/x86/kvm/mmu/mmu.c
7565
kvm_mmu_zap_memslot(kvm, slot);
arch/x86/kvm/mmu/mmu.c
7568
void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)
arch/x86/kvm/mmu/mmu.c
7584
gen &= ~((u64)kvm_arch_nr_memslot_as_ids(kvm) - 1);
arch/x86/kvm/mmu/mmu.c
7592
kvm_mmu_zap_all_fast(kvm);
arch/x86/kvm/mmu/mmu.c
7604
static void kvm_wake_nx_recovery_thread(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
7611
struct vhost_task *nx_thread = READ_ONCE(kvm->arch.nx_huge_page_recovery_thread);
arch/x86/kvm/mmu/mmu.c
7673
struct kvm *kvm;
arch/x86/kvm/mmu/mmu.c
7677
list_for_each_entry(kvm, &vm_list, vm_list) {
arch/x86/kvm/mmu/mmu.c
7678
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/mmu/mmu.c
7679
kvm_mmu_zap_all_fast(kvm);
arch/x86/kvm/mmu/mmu.c
7680
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/mmu/mmu.c
7682
kvm_wake_nx_recovery_thread(kvm);
arch/x86/kvm/mmu/mmu.c
7753
read_lock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7754
mmu_free_root_page(vcpu->kvm, &vcpu->arch.mmu->mirror_root_hpa,
arch/x86/kvm/mmu/mmu.c
7756
read_unlock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
778
static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/mmu.c
7812
struct kvm *kvm;
arch/x86/kvm/mmu/mmu.c
7816
list_for_each_entry(kvm, &vm_list, vm_list)
arch/x86/kvm/mmu/mmu.c
7817
kvm_wake_nx_recovery_thread(kvm);
arch/x86/kvm/mmu/mmu.c
7825
static unsigned long nx_huge_pages_to_zap(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7828
unsigned long pages = READ_ONCE(kvm->arch.possible_nx_huge_pages[mmu_type].nr_pages);
arch/x86/kvm/mmu/mmu.c
7834
static bool kvm_mmu_sp_dirty_logging_enabled(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
784
kvm->arch.indirect_shadow_pages++;
arch/x86/kvm/mmu/mmu.c
7849
if (!atomic_read(&kvm->nr_memslots_dirty_logging))
arch/x86/kvm/mmu/mmu.c
7852
slot = __gfn_to_memslot(kvm_memslots_for_spte_role(kvm, sp->role), sp->gfn);
arch/x86/kvm/mmu/mmu.c
7859
static void kvm_recover_nx_huge_pages(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
7864
spinlock_t *tdp_mmu_pages_lock = &kvm->arch.tdp_mmu_pages_lock;
arch/x86/kvm/mmu/mmu.c
7869
unsigned long to_zap = nx_huge_pages_to_zap(kvm, mmu_type);
arch/x86/kvm/mmu/mmu.c
7876
nx_huge_pages = &kvm->arch.possible_nx_huge_pages[mmu_type].pages;
arch/x86/kvm/mmu/mmu.c
7878
rcu_idx = srcu_read_lock(&kvm->srcu);
arch/x86/kvm/mmu/mmu.c
7880
read_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7882
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7914
unaccount_nx_huge_page(kvm, sp);
arch/x86/kvm/mmu/mmu.c
7926
if (!kvm_mmu_sp_dirty_logging_enabled(kvm, sp)) {
arch/x86/kvm/mmu/mmu.c
7928
flush |= kvm_tdp_mmu_zap_possible_nx_huge_page(kvm, sp);
arch/x86/kvm/mmu/mmu.c
7930
kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
arch/x86/kvm/mmu/mmu.c
7936
if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
arch/x86/kvm/mmu/mmu.c
7937
kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
arch/x86/kvm/mmu/mmu.c
7941
cond_resched_rwlock_read(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7943
cond_resched_rwlock_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7949
kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
arch/x86/kvm/mmu/mmu.c
795
slots = kvm_memslots_for_spte_role(kvm, sp->role);
arch/x86/kvm/mmu/mmu.c
7954
read_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7956
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
7957
srcu_read_unlock(&kvm->srcu, rcu_idx);
arch/x86/kvm/mmu/mmu.c
7966
struct kvm *kvm = data;
arch/x86/kvm/mmu/mmu.c
7976
remaining_time = kvm->arch.nx_huge_page_last + msecs_to_jiffies(period)
arch/x86/kvm/mmu/mmu.c
7986
kvm_recover_nx_huge_pages(kvm, i);
arch/x86/kvm/mmu/mmu.c
7987
kvm->arch.nx_huge_page_last = get_jiffies_64();
arch/x86/kvm/mmu/mmu.c
7994
struct kvm *kvm = container_of(ka, struct kvm, arch);
arch/x86/kvm/mmu/mmu.c
7997
kvm->arch.nx_huge_page_last = get_jiffies_64();
arch/x86/kvm/mmu/mmu.c
800
return __kvm_write_track_add_gfn(kvm, slot, gfn);
arch/x86/kvm/mmu/mmu.c
8000
kvm, "kvm-nx-lpage-recovery");
arch/x86/kvm/mmu/mmu.c
8008
WRITE_ONCE(kvm->arch.nx_huge_page_recovery_thread, nx_thread);
arch/x86/kvm/mmu/mmu.c
8012
int kvm_mmu_post_init_vm(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
8017
return call_once(&kvm->arch.nx_once, kvm_mmu_start_lpage_recovery);
arch/x86/kvm/mmu/mmu.c
8020
void kvm_mmu_pre_destroy_vm(struct kvm *kvm)
arch/x86/kvm/mmu/mmu.c
8022
if (kvm->arch.nx_huge_page_recovery_thread)
arch/x86/kvm/mmu/mmu.c
8023
vhost_task_stop(kvm->arch.nx_huge_page_recovery_thread);
arch/x86/kvm/mmu/mmu.c
804
if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K))
arch/x86/kvm/mmu/mmu.c
8045
bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
805
kvm_flush_remote_tlbs_gfn(kvm, gfn, PG_LEVEL_4K);
arch/x86/kvm/mmu/mmu.c
8062
if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm)))
arch/x86/kvm/mmu/mmu.c
808
void track_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/mmu.c
8086
kvm_mmu_invalidate_range_add(kvm, start, start + nr_pages);
arch/x86/kvm/mmu/mmu.c
8094
kvm_mmu_invalidate_range_add(kvm, end, end + nr_pages);
arch/x86/kvm/mmu/mmu.c
8103
return kvm_unmap_gfn_range(kvm, range);
arch/x86/kvm/mmu/mmu.c
8108
static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot,
arch/x86/kvm/mmu/mmu.c
8115
return kvm_range_has_memory_attributes(kvm, start, end, ~0, attrs);
arch/x86/kvm/mmu/mmu.c
8119
attrs != kvm_get_memory_attributes(kvm, gfn))
arch/x86/kvm/mmu/mmu.c
8125
bool kvm_arch_post_set_memory_attributes(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
8132
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
8133
lockdep_assert_held(&kvm->slots_lock);
arch/x86/kvm/mmu/mmu.c
8141
if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm)))
arch/x86/kvm/mmu/mmu.c
8161
if (hugepage_has_attrs(kvm, slot, gfn, level, attrs))
arch/x86/kvm/mmu/mmu.c
8183
if (hugepage_has_attrs(kvm, slot, gfn, level, attrs))
arch/x86/kvm/mmu/mmu.c
8192
void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
8197
if (!kvm_arch_has_private_mem(kvm))
arch/x86/kvm/mmu/mmu.c
8219
unsigned long attrs = kvm_get_memory_attributes(kvm, gfn);
arch/x86/kvm/mmu/mmu.c
822
++kvm->stat.nx_lpage_splits;
arch/x86/kvm/mmu/mmu.c
8221
if (hugepage_has_attrs(kvm, slot, gfn, level, attrs))
arch/x86/kvm/mmu/mmu.c
823
++kvm->arch.possible_nx_huge_pages[mmu_type].nr_pages;
arch/x86/kvm/mmu/mmu.c
825
&kvm->arch.possible_nx_huge_pages[mmu_type].pages);
arch/x86/kvm/mmu/mmu.c
828
static void account_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/mmu.c
834
track_possible_nx_huge_page(kvm, sp, KVM_SHADOW_MMU);
arch/x86/kvm/mmu/mmu.c
837
static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/mmu.c
843
kvm->arch.indirect_shadow_pages--;
arch/x86/kvm/mmu/mmu.c
845
slots = kvm_memslots_for_spte_role(kvm, sp->role);
arch/x86/kvm/mmu/mmu.c
848
return __kvm_write_track_remove_gfn(kvm, slot, gfn);
arch/x86/kvm/mmu/mmu.c
853
void untrack_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/mmu.c
859
--kvm->stat.nx_lpage_splits;
arch/x86/kvm/mmu/mmu.c
860
--kvm->arch.possible_nx_huge_pages[mmu_type].nr_pages;
arch/x86/kvm/mmu/mmu.c
864
static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/mmu.c
868
untrack_possible_nx_huge_page(kvm, sp, KVM_SHADOW_MMU);
arch/x86/kvm/mmu/mmu.c
968
static unsigned long kvm_rmap_lock(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
971
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu.c
989
static void kvm_rmap_unlock(struct kvm *kvm,
arch/x86/kvm/mmu/mmu.c
993
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/mmu_internal.h
171
static inline gfn_t kvm_gfn_root_bits(const struct kvm *kvm, const struct kvm_mmu_page *root)
arch/x86/kvm/mmu/mmu_internal.h
180
return kvm_gfn_direct_bits(kvm);
arch/x86/kvm/mmu/mmu_internal.h
183
static inline bool kvm_mmu_page_ad_need_write_protect(struct kvm *kvm,
arch/x86/kvm/mmu/mmu_internal.h
194
return kvm->arch.cpu_dirty_log_size && sp->role.guest_mode;
arch/x86/kvm/mmu/mmu_internal.h
202
int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot,
arch/x86/kvm/mmu/mmu_internal.h
207
bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
arch/x86/kvm/mmu/mmu_internal.h
212
static inline void kvm_flush_remote_tlbs_gfn(struct kvm *kvm, gfn_t gfn, int level)
arch/x86/kvm/mmu/mmu_internal.h
214
kvm_flush_remote_tlbs_range(kvm, gfn_round_for_level(gfn, level),
arch/x86/kvm/mmu/mmu_internal.h
221
static inline bool is_nx_huge_page_enabled(struct kvm *kvm)
arch/x86/kvm/mmu/mmu_internal.h
223
return READ_ONCE(nx_huge_pages) && !kvm->arch.disable_nx_huge_pages;
arch/x86/kvm/mmu/mmu_internal.h
355
is_nx_huge_page_enabled(vcpu->kvm),
arch/x86/kvm/mmu/mmu_internal.h
372
fault.gfn = gpa_to_gfn(fault.addr) & ~kvm_gfn_direct_bits(vcpu->kvm);
arch/x86/kvm/mmu/mmu_internal.h
404
int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault,
arch/x86/kvm/mmu/mmu_internal.h
409
void track_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/mmu_internal.h
411
void untrack_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/mmutrace.h
296
TP_PROTO(struct kvm *kvm),
arch/x86/kvm/mmu/mmutrace.h
297
TP_ARGS(kvm),
arch/x86/kvm/mmu/mmutrace.h
305
__entry->mmu_valid_gen = kvm->arch.mmu_valid_gen;
arch/x86/kvm/mmu/mmutrace.h
306
__entry->mmu_used_pages = kvm->stat.mmu_shadow_pages;
arch/x86/kvm/mmu/page_track.c
109
if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K))
arch/x86/kvm/mmu/page_track.c
110
kvm_flush_remote_tlbs(kvm);
arch/x86/kvm/mmu/page_track.c
113
void __kvm_write_track_remove_gfn(struct kvm *kvm,
arch/x86/kvm/mmu/page_track.c
116
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/page_track.c
118
lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) ||
arch/x86/kvm/mmu/page_track.c
119
srcu_read_lock_held(&kvm->srcu));
arch/x86/kvm/mmu/page_track.c
121
if (KVM_BUG_ON(!kvm_page_track_write_tracking_enabled(kvm), kvm))
arch/x86/kvm/mmu/page_track.c
146
bool kvm_gfn_is_write_tracked(struct kvm *kvm,
arch/x86/kvm/mmu/page_track.c
154
if (!kvm_page_track_write_tracking_enabled(kvm))
arch/x86/kvm/mmu/page_track.c
162
if (kvm_arch_nr_memslot_as_ids(kvm) > 1) {
arch/x86/kvm/mmu/page_track.c
163
other_slot = __gfn_to_memslot(__kvm_memslots(kvm, slot->as_id ^ 1), gfn);
arch/x86/kvm/mmu/page_track.c
172
void kvm_page_track_cleanup(struct kvm *kvm)
arch/x86/kvm/mmu/page_track.c
176
head = &kvm->arch.track_notifier_head;
arch/x86/kvm/mmu/page_track.c
180
int kvm_page_track_init(struct kvm *kvm)
arch/x86/kvm/mmu/page_track.c
184
head = &kvm->arch.track_notifier_head;
arch/x86/kvm/mmu/page_track.c
189
static int kvm_enable_external_write_tracking(struct kvm *kvm)
arch/x86/kvm/mmu/page_track.c
195
if (kvm->arch.vm_type == KVM_X86_TDX_VM)
arch/x86/kvm/mmu/page_track.c
198
mutex_lock(&kvm->slots_arch_lock);
arch/x86/kvm/mmu/page_track.c
205
if (kvm_page_track_write_tracking_enabled(kvm))
arch/x86/kvm/mmu/page_track.c
208
for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
arch/x86/kvm/mmu/page_track.c
209
slots = __kvm_memslots(kvm, i);
arch/x86/kvm/mmu/page_track.c
23
static bool kvm_external_write_tracking_enabled(struct kvm *kvm)
arch/x86/kvm/mmu/page_track.c
230
smp_store_release(&kvm->arch.external_write_tracking_enabled, true);
arch/x86/kvm/mmu/page_track.c
232
mutex_unlock(&kvm->slots_arch_lock);
arch/x86/kvm/mmu/page_track.c
240
int kvm_page_track_register_notifier(struct kvm *kvm,
arch/x86/kvm/mmu/page_track.c
246
if (!kvm || kvm->mm != current->mm)
arch/x86/kvm/mmu/page_track.c
249
if (!kvm_external_write_tracking_enabled(kvm)) {
arch/x86/kvm/mmu/page_track.c
250
r = kvm_enable_external_write_tracking(kvm);
arch/x86/kvm/mmu/page_track.c
255
kvm_get_kvm(kvm);
arch/x86/kvm/mmu/page_track.c
257
head = &kvm->arch.track_notifier_head;
arch/x86/kvm/mmu/page_track.c
259
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/page_track.c
261
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/page_track.c
270
void kvm_page_track_unregister_notifier(struct kvm *kvm,
arch/x86/kvm/mmu/page_track.c
275
head = &kvm->arch.track_notifier_head;
arch/x86/kvm/mmu/page_track.c
277
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/page_track.c
279
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/page_track.c
282
kvm_put_kvm(kvm);
arch/x86/kvm/mmu/page_track.c
293
void __kvm_page_track_write(struct kvm *kvm, gpa_t gpa, const u8 *new, int bytes)
arch/x86/kvm/mmu/page_track.c
299
head = &kvm->arch.track_notifier_head;
arch/x86/kvm/mmu/page_track.c
30
return smp_load_acquire(&kvm->arch.external_write_tracking_enabled);
arch/x86/kvm/mmu/page_track.c
316
void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot)
arch/x86/kvm/mmu/page_track.c
322
head = &kvm->arch.track_notifier_head;
arch/x86/kvm/mmu/page_track.c
342
int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn)
arch/x86/kvm/mmu/page_track.c
347
idx = srcu_read_lock(&kvm->srcu);
arch/x86/kvm/mmu/page_track.c
349
slot = gfn_to_memslot(kvm, gfn);
arch/x86/kvm/mmu/page_track.c
351
srcu_read_unlock(&kvm->srcu, idx);
arch/x86/kvm/mmu/page_track.c
355
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/page_track.c
356
__kvm_write_track_add_gfn(kvm, slot, gfn);
arch/x86/kvm/mmu/page_track.c
357
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/page_track.c
359
srcu_read_unlock(&kvm->srcu, idx);
arch/x86/kvm/mmu/page_track.c
36
bool kvm_page_track_write_tracking_enabled(struct kvm *kvm)
arch/x86/kvm/mmu/page_track.c
372
int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn)
arch/x86/kvm/mmu/page_track.c
377
idx = srcu_read_lock(&kvm->srcu);
arch/x86/kvm/mmu/page_track.c
379
slot = gfn_to_memslot(kvm, gfn);
arch/x86/kvm/mmu/page_track.c
38
return kvm_external_write_tracking_enabled(kvm) ||
arch/x86/kvm/mmu/page_track.c
381
srcu_read_unlock(&kvm->srcu, idx);
arch/x86/kvm/mmu/page_track.c
385
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/page_track.c
386
__kvm_write_track_remove_gfn(kvm, slot, gfn);
arch/x86/kvm/mmu/page_track.c
387
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/page_track.c
389
srcu_read_unlock(&kvm->srcu, idx);
arch/x86/kvm/mmu/page_track.c
39
kvm_shadow_root_allocated(kvm) || !tdp_enabled;
arch/x86/kvm/mmu/page_track.c
60
int kvm_page_track_create_memslot(struct kvm *kvm,
arch/x86/kvm/mmu/page_track.c
64
if (!kvm_page_track_write_tracking_enabled(kvm))
arch/x86/kvm/mmu/page_track.c
90
void __kvm_write_track_add_gfn(struct kvm *kvm, struct kvm_memory_slot *slot,
arch/x86/kvm/mmu/page_track.c
93
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/page_track.c
95
lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) ||
arch/x86/kvm/mmu/page_track.c
96
srcu_read_lock_held(&kvm->srcu));
arch/x86/kvm/mmu/page_track.c
98
if (KVM_BUG_ON(!kvm_page_track_write_tracking_enabled(kvm), kvm))
arch/x86/kvm/mmu/page_track.h
10
bool kvm_page_track_write_tracking_enabled(struct kvm *kvm);
arch/x86/kvm/mmu/page_track.h
14
int kvm_page_track_create_memslot(struct kvm *kvm,
arch/x86/kvm/mmu/page_track.h
18
void __kvm_write_track_add_gfn(struct kvm *kvm, struct kvm_memory_slot *slot,
arch/x86/kvm/mmu/page_track.h
20
void __kvm_write_track_remove_gfn(struct kvm *kvm,
arch/x86/kvm/mmu/page_track.h
23
bool kvm_gfn_is_write_tracked(struct kvm *kvm,
arch/x86/kvm/mmu/page_track.h
27
int kvm_page_track_init(struct kvm *kvm);
arch/x86/kvm/mmu/page_track.h
28
void kvm_page_track_cleanup(struct kvm *kvm);
arch/x86/kvm/mmu/page_track.h
30
void __kvm_page_track_write(struct kvm *kvm, gpa_t gpa, const u8 *new, int bytes);
arch/x86/kvm/mmu/page_track.h
31
void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot);
arch/x86/kvm/mmu/page_track.h
33
static inline bool kvm_page_track_has_external_user(struct kvm *kvm)
arch/x86/kvm/mmu/page_track.h
35
return !hlist_empty(&kvm->arch.track_notifier_head.track_notifier_list);
arch/x86/kvm/mmu/page_track.h
38
static inline int kvm_page_track_init(struct kvm *kvm) { return 0; }
arch/x86/kvm/mmu/page_track.h
39
static inline void kvm_page_track_cleanup(struct kvm *kvm) { }
arch/x86/kvm/mmu/page_track.h
41
static inline void __kvm_page_track_write(struct kvm *kvm, gpa_t gpa,
arch/x86/kvm/mmu/page_track.h
43
static inline void kvm_page_track_delete_slot(struct kvm *kvm,
arch/x86/kvm/mmu/page_track.h
46
static inline bool kvm_page_track_has_external_user(struct kvm *kvm) { return false; }
arch/x86/kvm/mmu/page_track.h
53
__kvm_page_track_write(vcpu->kvm, gpa, new, bytes);
arch/x86/kvm/mmu/paging_tmpl.h
174
drop_spte(vcpu->kvm, spte);
arch/x86/kvm/mmu/paging_tmpl.h
375
if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm))
arch/x86/kvm/mmu/paging_tmpl.h
625
if (unlikely(vcpu->kvm->mmu_invalidate_in_progress))
arch/x86/kvm/mmu/paging_tmpl.h
772
account_nx_huge_page(vcpu->kvm, sp,
arch/x86/kvm/mmu/paging_tmpl.h
867
write_lock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/paging_tmpl.h
882
write_unlock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/mmu/paging_tmpl.h
976
drop_spte(vcpu->kvm, &sp->spt[i]);
arch/x86/kvm/mmu/spte.c
101
spte |= vcpu->kvm->arch.shadow_mmio_value | access;
arch/x86/kvm/mmu/spte.c
150
WRITE_ONCE(vcpu->kvm->arch.has_mapped_host_mmio, true);
arch/x86/kvm/mmu/spte.c
156
kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_OUTSIDE_GUEST_MODE);
arch/x86/kvm/mmu/spte.c
203
else if (kvm_mmu_page_ad_need_write_protect(vcpu->kvm, sp))
arch/x86/kvm/mmu/spte.c
222
if (level > PG_LEVEL_4K && is_nx_huge_page_enabled(vcpu->kvm)) {
arch/x86/kvm/mmu/spte.c
274
mmu_try_to_unsync_pages(vcpu->kvm, slot, gfn, synchronizing, prefetch))
arch/x86/kvm/mmu/spte.c
296
mark_page_dirty_in_slot(vcpu->kvm, slot, gfn);
arch/x86/kvm/mmu/spte.c
345
u64 make_small_spte(struct kvm *kvm, u64 huge_spte,
arch/x86/kvm/mmu/spte.c
350
KVM_BUG_ON(!is_shadow_present_pte(huge_spte) || !is_large_pte(huge_spte), kvm);
arch/x86/kvm/mmu/spte.c
367
if (is_nx_huge_page_enabled(kvm))
arch/x86/kvm/mmu/spte.c
374
u64 make_huge_spte(struct kvm *kvm, u64 small_spte, int level)
arch/x86/kvm/mmu/spte.c
378
KVM_BUG_ON(!is_shadow_present_pte(small_spte) || level == PG_LEVEL_4K, kvm);
arch/x86/kvm/mmu/spte.c
389
if (is_nx_huge_page_enabled(kvm))
arch/x86/kvm/mmu/spte.c
484
void kvm_mmu_set_mmio_spte_value(struct kvm *kvm, u64 mmio_value)
arch/x86/kvm/mmu/spte.c
486
kvm->arch.shadow_mmio_value = mmio_value;
arch/x86/kvm/mmu/spte.h
302
return READ_ONCE(vcpu->kvm->arch.has_mapped_host_mmio);
arch/x86/kvm/mmu/spte.h
305
static inline bool is_mmio_spte(struct kvm *kvm, u64 spte)
arch/x86/kvm/mmu/spte.h
307
return (spte & shadow_mmio_mask) == kvm->arch.shadow_mmio_value &&
arch/x86/kvm/mmu/spte.h
558
u64 make_small_spte(struct kvm *kvm, u64 huge_spte,
arch/x86/kvm/mmu/spte.h
560
u64 make_huge_spte(struct kvm *kvm, u64 small_spte, int level);
arch/x86/kvm/mmu/tdp_iter.h
123
#define for_each_tdp_pte_min_level(iter, kvm, root, min_level, start, end) \
arch/x86/kvm/mmu/tdp_iter.h
124
for (tdp_iter_start(&iter, root, min_level, start, kvm_gfn_root_bits(kvm, root)); \
arch/x86/kvm/mmu/tdp_iter.h
133
#define for_each_tdp_pte(iter, kvm, root, start, end) \
arch/x86/kvm/mmu/tdp_iter.h
134
for_each_tdp_pte_min_level(iter, kvm, root, PG_LEVEL_4K, start, end)
arch/x86/kvm/mmu/tdp_mmu.c
1007
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1008
__for_each_tdp_mmu_root_yield_safe(kvm, root, -1,
arch/x86/kvm/mmu/tdp_mmu.c
1010
tdp_mmu_zap_root(kvm, root, false);
arch/x86/kvm/mmu/tdp_mmu.c
1017
void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm, bool shared)
arch/x86/kvm/mmu/tdp_mmu.c
1022
read_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1024
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1026
for_each_tdp_mmu_root_yield_safe(kvm, root) {
arch/x86/kvm/mmu/tdp_mmu.c
1031
KVM_BUG_ON(!root->role.invalid, kvm);
arch/x86/kvm/mmu/tdp_mmu.c
1042
tdp_mmu_zap_root(kvm, root, shared);
arch/x86/kvm/mmu/tdp_mmu.c
1049
kvm_tdp_mmu_put_root(kvm, root);
arch/x86/kvm/mmu/tdp_mmu.c
1053
read_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1055
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1068
void kvm_tdp_mmu_invalidate_roots(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
1092
refcount_read(&kvm->users_count) && kvm->created_vcpus)
arch/x86/kvm/mmu/tdp_mmu.c
1093
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1100
list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) {
arch/x86/kvm/mmu/tdp_mmu.c
1149
else if (tdp_mmu_set_spte_atomic(vcpu->kvm, iter, new_spte))
arch/x86/kvm/mmu/tdp_mmu.c
1154
kvm_flush_remote_tlbs_gfn(vcpu->kvm, iter->gfn, iter->level);
arch/x86/kvm/mmu/tdp_mmu.c
1165
if (unlikely(is_mmio_spte(vcpu->kvm, new_spte))) {
arch/x86/kvm/mmu/tdp_mmu.c
1190
static int tdp_mmu_link_sp(struct kvm *kvm, struct tdp_iter *iter,
arch/x86/kvm/mmu/tdp_mmu.c
1197
ret = tdp_mmu_set_spte_atomic(kvm, iter, spte);
arch/x86/kvm/mmu/tdp_mmu.c
1201
tdp_mmu_iter_set_spte(kvm, iter, spte);
arch/x86/kvm/mmu/tdp_mmu.c
1204
tdp_account_mmu_page(kvm, sp);
arch/x86/kvm/mmu/tdp_mmu.c
1209
static int tdp_mmu_split_huge_page(struct kvm *kvm, struct tdp_iter *iter,
arch/x86/kvm/mmu/tdp_mmu.c
1219
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/mmu/tdp_mmu.c
1232
for_each_tdp_pte(iter, kvm, root, fault->gfn, fault->gfn + 1) {
arch/x86/kvm/mmu/tdp_mmu.c
1266
KVM_BUG_ON(is_mirror_sptep(iter.sptep), vcpu->kvm);
arch/x86/kvm/mmu/tdp_mmu.c
1267
r = tdp_mmu_split_huge_page(kvm, &iter, sp, true);
arch/x86/kvm/mmu/tdp_mmu.c
1269
r = tdp_mmu_link_sp(kvm, &iter, sp, true);
arch/x86/kvm/mmu/tdp_mmu.c
127
static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
1283
spin_lock(&kvm->arch.tdp_mmu_pages_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1285
track_possible_nx_huge_page(kvm, sp, KVM_TDP_MMU);
arch/x86/kvm/mmu/tdp_mmu.c
1286
spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1306
bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range,
arch/x86/kvm/mmu/tdp_mmu.c
1312
types = kvm_gfn_range_filter_to_root_types(kvm, range->attr_filter) | KVM_INVALID_ROOTS;
arch/x86/kvm/mmu/tdp_mmu.c
1314
__for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id, types)
arch/x86/kvm/mmu/tdp_mmu.c
1315
flush = tdp_mmu_zap_leafs(kvm, root, range->start, range->end,
arch/x86/kvm/mmu/tdp_mmu.c
1329
static void kvm_tdp_mmu_age_spte(struct kvm *kvm, struct tdp_iter *iter)
arch/x86/kvm/mmu/tdp_mmu.c
1347
if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte))
arch/x86/kvm/mmu/tdp_mmu.c
1355
static bool __kvm_tdp_mmu_age_gfn_range(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
1364
types = kvm_gfn_range_filter_to_root_types(kvm, range->attr_filter);
arch/x86/kvm/mmu/tdp_mmu.c
137
lockdep_assert_held(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1375
for_each_tdp_mmu_root_rcu(kvm, root, range->slot->as_id, types) {
arch/x86/kvm/mmu/tdp_mmu.c
1376
tdp_root_for_each_leaf_pte(iter, kvm, root, range->start, range->end) {
arch/x86/kvm/mmu/tdp_mmu.c
1384
kvm_tdp_mmu_age_spte(kvm, &iter);
arch/x86/kvm/mmu/tdp_mmu.c
1391
bool kvm_tdp_mmu_age_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
arch/x86/kvm/mmu/tdp_mmu.c
1393
return __kvm_tdp_mmu_age_gfn_range(kvm, range, false);
arch/x86/kvm/mmu/tdp_mmu.c
1396
bool kvm_tdp_mmu_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
arch/x86/kvm/mmu/tdp_mmu.c
1398
return __kvm_tdp_mmu_age_gfn_range(kvm, range, true);
arch/x86/kvm/mmu/tdp_mmu.c
1406
static bool wrprot_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
arch/x86/kvm/mmu/tdp_mmu.c
1413
if (KVM_BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL, kvm))
arch/x86/kvm/mmu/tdp_mmu.c
1418
for_each_tdp_pte_min_level(iter, kvm, root, min_level, start, end) {
arch/x86/kvm/mmu/tdp_mmu.c
142
next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots,
arch/x86/kvm/mmu/tdp_mmu.c
1420
if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true))
arch/x86/kvm/mmu/tdp_mmu.c
1430
if (tdp_mmu_set_spte_atomic(kvm, &iter, new_spte))
arch/x86/kvm/mmu/tdp_mmu.c
1445
bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
1451
lockdep_assert_held_read(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1453
for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id)
arch/x86/kvm/mmu/tdp_mmu.c
1454
spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn,
arch/x86/kvm/mmu/tdp_mmu.c
146
next_root = list_first_or_null_rcu(&kvm->arch.tdp_mmu_roots,
arch/x86/kvm/mmu/tdp_mmu.c
1478
static int tdp_mmu_split_huge_page(struct kvm *kvm, struct tdp_iter *iter,
arch/x86/kvm/mmu/tdp_mmu.c
1490
sp->spt[i] = make_small_spte(kvm, huge_spte, sp->role, i);
arch/x86/kvm/mmu/tdp_mmu.c
15
void kvm_mmu_init_tdp_mmu(struct kvm *kvm)
arch/x86/kvm/mmu/tdp_mmu.c
1500
ret = tdp_mmu_link_sp(kvm, iter, sp, shared);
arch/x86/kvm/mmu/tdp_mmu.c
1509
kvm_update_page_stats(kvm, level - 1, SPTE_ENT_PER_PAGE);
arch/x86/kvm/mmu/tdp_mmu.c
1516
static int tdp_mmu_split_huge_pages_root(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
1537
for_each_tdp_pte_min_level(iter, kvm, root, target_level + 1, start, end) {
arch/x86/kvm/mmu/tdp_mmu.c
1539
if (tdp_mmu_iter_cond_resched(kvm, &iter, false, shared))
arch/x86/kvm/mmu/tdp_mmu.c
154
next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots,
arch/x86/kvm/mmu/tdp_mmu.c
1549
read_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1551
write_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1556
read_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1558
write_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1575
if (tdp_mmu_split_huge_page(kvm, &iter, sp, shared))
arch/x86/kvm/mmu/tdp_mmu.c
1598
void kvm_tdp_mmu_try_split_huge_pages(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
1606
kvm_lockdep_assert_mmu_lock_held(kvm, shared);
arch/x86/kvm/mmu/tdp_mmu.c
1607
for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id) {
arch/x86/kvm/mmu/tdp_mmu.c
1608
r = tdp_mmu_split_huge_pages_root(kvm, root, start, end, target_level, shared);
arch/x86/kvm/mmu/tdp_mmu.c
161
kvm_tdp_mmu_put_root(kvm, prev_root);
arch/x86/kvm/mmu/tdp_mmu.c
1610
kvm_tdp_mmu_put_root(kvm, root);
arch/x86/kvm/mmu/tdp_mmu.c
1616
static bool tdp_mmu_need_write_protect(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/tdp_mmu.c
1623
return kvm_mmu_page_ad_need_write_protect(kvm, sp) || !kvm_ad_enabled;
arch/x86/kvm/mmu/tdp_mmu.c
1626
static void clear_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
arch/x86/kvm/mmu/tdp_mmu.c
1629
const u64 dbit = tdp_mmu_need_write_protect(kvm, root) ?
arch/x86/kvm/mmu/tdp_mmu.c
1635
tdp_root_for_each_pte(iter, kvm, root, start, end) {
arch/x86/kvm/mmu/tdp_mmu.c
1641
if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true))
arch/x86/kvm/mmu/tdp_mmu.c
1650
if (tdp_mmu_set_spte_atomic(kvm, &iter, iter.old_spte & ~dbit))
arch/x86/kvm/mmu/tdp_mmu.c
1661
void kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
1666
lockdep_assert_held_read(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1667
for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id)
arch/x86/kvm/mmu/tdp_mmu.c
1668
clear_dirty_gfn_range(kvm, root, slot->base_gfn,
arch/x86/kvm/mmu/tdp_mmu.c
1672
static void clear_dirty_pt_masked(struct kvm *kvm, struct kvm_mmu_page *root,
arch/x86/kvm/mmu/tdp_mmu.c
1675
const u64 dbit = (wrprot || tdp_mmu_need_write_protect(kvm, root)) ?
arch/x86/kvm/mmu/tdp_mmu.c
1679
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1683
tdp_root_for_each_leaf_pte(iter, kvm, root, gfn + __ffs(mask),
arch/x86/kvm/mmu/tdp_mmu.c
17
INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots);
arch/x86/kvm/mmu/tdp_mmu.c
1717
void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
1724
for_each_valid_tdp_mmu_root(kvm, root, slot->as_id)
arch/x86/kvm/mmu/tdp_mmu.c
1725
clear_dirty_pt_masked(kvm, root, gfn, mask, wrprot);
arch/x86/kvm/mmu/tdp_mmu.c
1728
static int tdp_mmu_make_huge_spte(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
1737
tdp_root_for_each_leaf_pte(iter, kvm, root, start, end) {
arch/x86/kvm/mmu/tdp_mmu.c
1744
if (tdp_mmu_iter_need_resched(kvm, parent))
arch/x86/kvm/mmu/tdp_mmu.c
1747
*huge_spte = make_huge_spte(kvm, iter.old_spte, parent->level);
arch/x86/kvm/mmu/tdp_mmu.c
1754
static void recover_huge_pages_range(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
1771
for_each_tdp_pte_min_level(iter, kvm, root, PG_LEVEL_2M, start, end) {
arch/x86/kvm/mmu/tdp_mmu.c
1773
if (tdp_mmu_iter_cond_resched(kvm, &iter, flush, true)) {
arch/x86/kvm/mmu/tdp_mmu.c
18
spin_lock_init(&kvm->arch.tdp_mmu_pages_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1800
max_mapping_level = kvm_mmu_max_mapping_level(kvm, NULL, slot, iter.gfn);
arch/x86/kvm/mmu/tdp_mmu.c
1804
r = tdp_mmu_make_huge_spte(kvm, &iter, &huge_spte);
arch/x86/kvm/mmu/tdp_mmu.c
1810
if (tdp_mmu_set_spte_atomic(kvm, &iter, huge_spte))
arch/x86/kvm/mmu/tdp_mmu.c
1817
kvm_flush_remote_tlbs_memslot(kvm, slot);
arch/x86/kvm/mmu/tdp_mmu.c
1826
void kvm_tdp_mmu_recover_huge_pages(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
1831
lockdep_assert_held_read(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1832
for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id)
arch/x86/kvm/mmu/tdp_mmu.c
1833
recover_huge_pages_range(kvm, root, slot);
arch/x86/kvm/mmu/tdp_mmu.c
1841
static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
arch/x86/kvm/mmu/tdp_mmu.c
1848
if (KVM_BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL, kvm))
arch/x86/kvm/mmu/tdp_mmu.c
1853
for_each_tdp_pte_min_level(iter, kvm, root, min_level, gfn, gfn + 1) {
arch/x86/kvm/mmu/tdp_mmu.c
1864
tdp_mmu_iter_set_spte(kvm, &iter, new_spte);
arch/x86/kvm/mmu/tdp_mmu.c
1878
bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
1885
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
1886
for_each_valid_tdp_mmu_root(kvm, root, slot->as_id)
arch/x86/kvm/mmu/tdp_mmu.c
1887
spte_set |= write_protect_gfn(kvm, root, gfn, min_level);
arch/x86/kvm/mmu/tdp_mmu.c
1908
for_each_tdp_pte(iter, vcpu->kvm, root, gfn, gfn + 1) {
arch/x86/kvm/mmu/tdp_mmu.c
1935
for_each_tdp_pte(iter, vcpu->kvm, root, gfn, gfn + 1) {
arch/x86/kvm/mmu/tdp_mmu.c
22
static __always_inline bool kvm_lockdep_assert_mmu_lock_held(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
26
lockdep_assert_held_read(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
264
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/mmu/tdp_mmu.c
276
read_lock(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
278
for_each_valid_tdp_mmu_root_yield_safe(kvm, root, as_id) {
arch/x86/kvm/mmu/tdp_mmu.c
28
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
283
spin_lock(&kvm->arch.tdp_mmu_pages_lock);
arch/x86/kvm/mmu/tdp_mmu.c
293
list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) {
arch/x86/kvm/mmu/tdp_mmu.c
310
list_add_rcu(&root->link, &kvm->arch.tdp_mmu_roots);
arch/x86/kvm/mmu/tdp_mmu.c
313
spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
arch/x86/kvm/mmu/tdp_mmu.c
315
read_unlock(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
329
static void handle_changed_spte(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/tdp_mmu.c
33
void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm)
arch/x86/kvm/mmu/tdp_mmu.c
333
static void tdp_account_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/tdp_mmu.c
337
atomic64_inc(&kvm->arch.tdp_mmu_pages);
arch/x86/kvm/mmu/tdp_mmu.c
341
static void tdp_unaccount_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/tdp_mmu.c
345
atomic64_dec(&kvm->arch.tdp_mmu_pages);
arch/x86/kvm/mmu/tdp_mmu.c
355
static void tdp_mmu_unlink_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/mmu/tdp_mmu.c
357
tdp_unaccount_mmu_page(kvm, sp);
arch/x86/kvm/mmu/tdp_mmu.c
362
spin_lock(&kvm->arch.tdp_mmu_pages_lock);
arch/x86/kvm/mmu/tdp_mmu.c
364
untrack_possible_nx_huge_page(kvm, sp, KVM_TDP_MMU);
arch/x86/kvm/mmu/tdp_mmu.c
365
spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
arch/x86/kvm/mmu/tdp_mmu.c
385
static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared)
arch/x86/kvm/mmu/tdp_mmu.c
394
tdp_mmu_unlink_sp(kvm, sp);
arch/x86/kvm/mmu/tdp_mmu.c
40
kvm_tdp_mmu_invalidate_roots(kvm, KVM_VALID_ROOTS);
arch/x86/kvm/mmu/tdp_mmu.c
41
kvm_tdp_mmu_zap_invalidated_roots(kvm, false);
arch/x86/kvm/mmu/tdp_mmu.c
44
KVM_MMU_WARN_ON(atomic64_read(&kvm->arch.tdp_mmu_pages));
arch/x86/kvm/mmu/tdp_mmu.c
46
WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));
arch/x86/kvm/mmu/tdp_mmu.c
461
handle_changed_spte(kvm, sp, gfn, old_spte, FROZEN_SPTE, level, shared);
arch/x86/kvm/mmu/tdp_mmu.c
465
kvm_x86_call(free_external_spt)(kvm, sp);
arch/x86/kvm/mmu/tdp_mmu.c
486
static int __handle_changed_spte(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/tdp_mmu.c
542
if (WARN_ON_ONCE(!is_mmio_spte(kvm, old_spte) &&
arch/x86/kvm/mmu/tdp_mmu.c
543
!is_mmio_spte(kvm, new_spte) &&
arch/x86/kvm/mmu/tdp_mmu.c
569
handle_removed_pt(kvm, spte_to_child_pt(old_spte, level), shared);
arch/x86/kvm/mmu/tdp_mmu.c
573
r = kvm_x86_call(set_external_spte)(kvm, gfn, old_spte, new_spte, level);
arch/x86/kvm/mmu/tdp_mmu.c
580
kvm_update_page_stats(kvm, level, is_leaf ? 1 : -1);
arch/x86/kvm/mmu/tdp_mmu.c
585
static void handle_changed_spte(struct kvm *kvm, struct kvm_mmu_page *sp,
arch/x86/kvm/mmu/tdp_mmu.c
589
KVM_BUG_ON(__handle_changed_spte(kvm, sp, gfn, old_spte, new_spte,
arch/x86/kvm/mmu/tdp_mmu.c
590
level, shared), kvm);
arch/x86/kvm/mmu/tdp_mmu.c
593
static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
635
static inline int __must_check tdp_mmu_set_spte_atomic(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
642
lockdep_assert_held_read(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
654
ret = __tdp_mmu_set_spte_atomic(kvm, iter, FROZEN_SPTE);
arch/x86/kvm/mmu/tdp_mmu.c
656
ret = __tdp_mmu_set_spte_atomic(kvm, iter, new_spte);
arch/x86/kvm/mmu/tdp_mmu.c
669
ret = __handle_changed_spte(kvm, sp, iter->gfn, iter->old_spte,
arch/x86/kvm/mmu/tdp_mmu.c
686
KVM_BUG_ON(ret, kvm);
arch/x86/kvm/mmu/tdp_mmu.c
703
static u64 tdp_mmu_set_spte(struct kvm *kvm, tdp_ptep_t sptep, u64 old_spte,
arch/x86/kvm/mmu/tdp_mmu.c
708
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
721
handle_changed_spte(kvm, sp, gfn, old_spte, new_spte, level, false);
arch/x86/kvm/mmu/tdp_mmu.c
726
static inline void tdp_mmu_iter_set_spte(struct kvm *kvm, struct tdp_iter *iter,
arch/x86/kvm/mmu/tdp_mmu.c
730
iter->old_spte = tdp_mmu_set_spte(kvm, iter->sptep, iter->old_spte,
arch/x86/kvm/mmu/tdp_mmu.c
744
static inline bool __must_check tdp_mmu_iter_need_resched(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
747
if (!need_resched() && !rwlock_needbreak(&kvm->mmu_lock))
arch/x86/kvm/mmu/tdp_mmu.c
768
static inline bool __must_check tdp_mmu_iter_cond_resched(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
774
if (!tdp_mmu_iter_need_resched(kvm, iter))
arch/x86/kvm/mmu/tdp_mmu.c
778
kvm_flush_remote_tlbs(kvm);
arch/x86/kvm/mmu/tdp_mmu.c
783
cond_resched_rwlock_read(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
785
cond_resched_rwlock_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
806
static void __tdp_mmu_zap_root(struct kvm *kvm, struct kvm_mmu_page *root,
arch/x86/kvm/mmu/tdp_mmu.c
813
if (tdp_mmu_iter_cond_resched(kvm, &iter, false, shared))
arch/x86/kvm/mmu/tdp_mmu.c
823
tdp_mmu_iter_set_spte(kvm, &iter, SHADOW_NONPRESENT_VALUE);
arch/x86/kvm/mmu/tdp_mmu.c
824
else if (tdp_mmu_set_spte_atomic(kvm, &iter, SHADOW_NONPRESENT_VALUE))
arch/x86/kvm/mmu/tdp_mmu.c
829
static void tdp_mmu_zap_root(struct kvm *kvm, struct kvm_mmu_page *root,
arch/x86/kvm/mmu/tdp_mmu.c
845
kvm_lockdep_assert_mmu_lock_held(kvm, shared);
arch/x86/kvm/mmu/tdp_mmu.c
85
void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root)
arch/x86/kvm/mmu/tdp_mmu.c
867
__tdp_mmu_zap_root(kvm, root, shared, PG_LEVEL_4K);
arch/x86/kvm/mmu/tdp_mmu.c
868
__tdp_mmu_zap_root(kvm, root, shared, PG_LEVEL_2M);
arch/x86/kvm/mmu/tdp_mmu.c
870
__tdp_mmu_zap_root(kvm, root, shared, PG_LEVEL_1G);
arch/x86/kvm/mmu/tdp_mmu.c
871
__tdp_mmu_zap_root(kvm, root, shared, root->role.level);
arch/x86/kvm/mmu/tdp_mmu.c
876
bool kvm_tdp_mmu_zap_possible_nx_huge_page(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.c
887
lockdep_assert_held_read(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
918
if (tdp_mmu_set_spte_atomic(kvm, &iter, SHADOW_NONPRESENT_VALUE)) {
arch/x86/kvm/mmu/tdp_mmu.c
933
static bool tdp_mmu_zap_leafs(struct kvm *kvm, struct kvm_mmu_page *root,
arch/x86/kvm/mmu/tdp_mmu.c
940
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
944
for_each_tdp_pte_min_level(iter, kvm, root, PG_LEVEL_4K, start, end) {
arch/x86/kvm/mmu/tdp_mmu.c
946
tdp_mmu_iter_cond_resched(kvm, &iter, flush, false)) {
arch/x86/kvm/mmu/tdp_mmu.c
95
KVM_BUG_ON(!is_tdp_mmu_page(root) || !root->role.invalid, kvm);
arch/x86/kvm/mmu/tdp_mmu.c
955
tdp_mmu_iter_set_spte(kvm, &iter, SHADOW_NONPRESENT_VALUE);
arch/x86/kvm/mmu/tdp_mmu.c
97
spin_lock(&kvm->arch.tdp_mmu_pages_lock);
arch/x86/kvm/mmu/tdp_mmu.c
979
bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush)
arch/x86/kvm/mmu/tdp_mmu.c
983
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/mmu/tdp_mmu.c
984
for_each_valid_tdp_mmu_root_yield_safe(kvm, root, -1)
arch/x86/kvm/mmu/tdp_mmu.c
985
flush = tdp_mmu_zap_leafs(kvm, root, start, end, true, flush);
arch/x86/kvm/mmu/tdp_mmu.c
99
spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
arch/x86/kvm/mmu/tdp_mmu.c
990
void kvm_tdp_mmu_zap_all(struct kvm *kvm)
arch/x86/kvm/mmu/tdp_mmu.h
10
void kvm_mmu_init_tdp_mmu(struct kvm *kvm);
arch/x86/kvm/mmu/tdp_mmu.h
11
void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm);
arch/x86/kvm/mmu/tdp_mmu.h
20
void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root);
arch/x86/kvm/mmu/tdp_mmu.h
30
static inline enum kvm_tdp_mmu_root_types kvm_gfn_range_filter_to_root_types(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.h
35
if (!kvm_has_mirrored_tdp(kvm))
arch/x86/kvm/mmu/tdp_mmu.h
51
if (unlikely(!kvm_is_addr_direct(vcpu->kvm, fault->addr)))
arch/x86/kvm/mmu/tdp_mmu.h
66
bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush);
arch/x86/kvm/mmu/tdp_mmu.h
67
bool kvm_tdp_mmu_zap_possible_nx_huge_page(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.h
69
void kvm_tdp_mmu_zap_all(struct kvm *kvm);
arch/x86/kvm/mmu/tdp_mmu.h
70
void kvm_tdp_mmu_invalidate_roots(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.h
72
void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm, bool shared);
arch/x86/kvm/mmu/tdp_mmu.h
76
bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range,
arch/x86/kvm/mmu/tdp_mmu.h
78
bool kvm_tdp_mmu_age_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range);
arch/x86/kvm/mmu/tdp_mmu.h
79
bool kvm_tdp_mmu_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range);
arch/x86/kvm/mmu/tdp_mmu.h
81
bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.h
83
void kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.h
85
void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.h
89
void kvm_tdp_mmu_recover_huge_pages(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.h
92
bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm,
arch/x86/kvm/mmu/tdp_mmu.h
96
void kvm_tdp_mmu_try_split_huge_pages(struct kvm *kvm,
arch/x86/kvm/pmu.c
1138
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/x86/kvm/pmu.c
1145
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/x86/kvm/pmu.c
1241
int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp)
arch/x86/kvm/pmu.c
1283
mutex_lock(&kvm->lock);
arch/x86/kvm/pmu.c
1284
filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter,
arch/x86/kvm/pmu.c
1285
mutex_is_locked(&kvm->lock));
arch/x86/kvm/pmu.c
1286
mutex_unlock(&kvm->lock);
arch/x86/kvm/pmu.c
1287
synchronize_srcu_expedited(&kvm->srcu);
arch/x86/kvm/pmu.c
1292
kvm_for_each_vcpu(i, vcpu, kvm)
arch/x86/kvm/pmu.c
1295
kvm_make_all_cpus_request(kvm, KVM_REQ_PMU);
arch/x86/kvm/pmu.c
1347
KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm))
arch/x86/kvm/pmu.c
1408
KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm))
arch/x86/kvm/pmu.c
526
struct kvm *kvm = pmc->vcpu->kvm;
arch/x86/kvm/pmu.c
528
filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu);
arch/x86/kvm/pmu.c
730
vcpu->kvm->arch.kvmclock_offset;
arch/x86/kvm/pmu.c
968
if (KVM_BUG_ON(!kvm_can_set_cpuid_and_feature_msrs(vcpu), vcpu->kvm))
arch/x86/kvm/pmu.c
991
if (!vcpu->kvm->arch.enable_pmu)
arch/x86/kvm/pmu.h
284
int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp);
arch/x86/kvm/smm.c
370
kvm_vm_dead(vcpu->kvm);
arch/x86/kvm/smm.h
145
if (!kvm_x86_call(has_emulated_msr)(vcpu->kvm, MSR_IA32_SMBASE))
arch/x86/kvm/svm/avic.c
1007
struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
arch/x86/kvm/svm/avic.c
1019
PAGE_SIZE << avic_get_physical_id_table_order(vcpu->kvm)))
arch/x86/kvm/svm/avic.c
1074
struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
arch/x86/kvm/svm/avic.c
1082
PAGE_SIZE << avic_get_physical_id_table_order(vcpu->kvm)))
arch/x86/kvm/svm/avic.c
149
static u32 __avic_get_max_physical_id(struct kvm *kvm, struct kvm_vcpu *vcpu)
arch/x86/kvm/svm/avic.c
166
return min(kvm->arch.max_vcpu_ids - 1, arch_max);
arch/x86/kvm/svm/avic.c
171
return __avic_get_max_physical_id(vcpu->kvm, vcpu);
arch/x86/kvm/svm/avic.c
266
vcpu = kvm_get_vcpu(&kvm_svm->kvm, vcpu_idx);
arch/x86/kvm/svm/avic.c
282
static int avic_get_physical_id_table_order(struct kvm *kvm)
arch/x86/kvm/svm/avic.c
285
return get_order((__avic_get_max_physical_id(kvm, NULL) + 1) * sizeof(u64));
arch/x86/kvm/svm/avic.c
288
int avic_alloc_physical_id_table(struct kvm *kvm)
arch/x86/kvm/svm/avic.c
290
struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
arch/x86/kvm/svm/avic.c
292
if (!irqchip_in_kernel(kvm) || !enable_apicv)
arch/x86/kvm/svm/avic.c
299
avic_get_physical_id_table_order(kvm));
arch/x86/kvm/svm/avic.c
306
void avic_vm_destroy(struct kvm *kvm)
arch/x86/kvm/svm/avic.c
309
struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
arch/x86/kvm/svm/avic.c
316
avic_get_physical_id_table_order(kvm));
arch/x86/kvm/svm/avic.c
323
int avic_vm_init(struct kvm *kvm)
arch/x86/kvm/svm/avic.c
327
struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
arch/x86/kvm/svm/avic.c
359
avic_vm_destroy(kvm);
arch/x86/kvm/svm/avic.c
370
struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
arch/x86/kvm/svm/avic.c
386
struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
arch/x86/kvm/svm/avic.c
399
kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_TOO_BIG);
arch/x86/kvm/svm/avic.c
410
if (kvm_apicv_activated(vcpu->kvm)) {
arch/x86/kvm/svm/avic.c
419
ret = kvm_alloc_apic_access_page(vcpu->kvm);
arch/x86/kvm/svm/avic.c
471
static void avic_kick_vcpu_by_physical_id(struct kvm *kvm, u32 physical_id,
arch/x86/kvm/svm/avic.c
478
struct kvm_vcpu *target_vcpu = kvm_get_vcpu_by_id(kvm, physical_id);
arch/x86/kvm/svm/avic.c
487
static void avic_kick_vcpu_by_logical_id(struct kvm *kvm, u32 *avic_logical_id_table,
arch/x86/kvm/svm/avic.c
512
avic_kick_vcpu_by_physical_id(kvm, physical_id, icrl);
arch/x86/kvm/svm/avic.c
519
static int avic_kick_target_vcpus_fast(struct kvm *kvm, struct kvm_lapic *source,
arch/x86/kvm/svm/avic.c
524
struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
arch/x86/kvm/svm/avic.c
545
avic_kick_vcpu_by_physical_id(kvm, dest, icrl);
arch/x86/kvm/svm/avic.c
580
avic_kick_vcpu_by_logical_id(kvm, avic_logical_id_table,
arch/x86/kvm/svm/avic.c
587
static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source,
arch/x86/kvm/svm/avic.c
594
if (!avic_kick_target_vcpus_fast(kvm, source, icrl, icrh, index))
arch/x86/kvm/svm/avic.c
605
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/svm/avic.c
649
avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh, index);
arch/x86/kvm/svm/avic.c
673
struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
arch/x86/kvm/svm/avic.c
846
if (!enable_apicv || !irqchip_in_kernel(vcpu->kvm))
arch/x86/kvm/svm/avic.c
877
int avic_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
arch/x86/kvm/svm/avic.c
895
.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id,
arch/x86/kvm/svm/nested.c
1469
if (kvm_apicv_activated(vcpu->kvm))
arch/x86/kvm/svm/nested.c
1561
if (kvm_apicv_activated(vcpu->kvm))
arch/x86/kvm/svm/nested.c
2045
!kvm_check_has_quirk(vcpu->kvm,
arch/x86/kvm/svm/sev.c
1026
if (vcpu->kvm->arch.vm_type != KVM_X86_DEFAULT_VM) {
arch/x86/kvm/svm/sev.c
1066
static int __sev_launch_update_vmsa(struct kvm *kvm, struct kvm_vcpu *vcpu,
arch/x86/kvm/svm/sev.c
1091
vmsa.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1094
ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_VMSA, &vmsa, error);
arch/x86/kvm/svm/sev.c
111
static __always_inline void kvm_lockdep_assert_sev_lock_held(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
1117
static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
1123
if (!sev_es_guest(kvm))
arch/x86/kvm/svm/sev.c
1126
if (kvm_is_vcpu_creation_in_progress(kvm))
arch/x86/kvm/svm/sev.c
1129
ret = kvm_lock_all_vcpus(kvm);
arch/x86/kvm/svm/sev.c
1133
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/svm/sev.c
1134
ret = __sev_launch_update_vmsa(kvm, vcpu, &argp->error);
arch/x86/kvm/svm/sev.c
1139
kvm_unlock_all_vcpus(kvm);
arch/x86/kvm/svm/sev.c
1143
static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
1152
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
1178
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1179
ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, &data, &argp->error);
arch/x86/kvm/svm/sev.c
118
if (!refcount_read(&kvm->users_count))
arch/x86/kvm/svm/sev.c
1204
static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
1208
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
1211
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1212
return sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, &data, &argp->error);
arch/x86/kvm/svm/sev.c
1215
static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
1221
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
1226
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1227
ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, &data, &argp->error);
arch/x86/kvm/svm/sev.c
1241
static int sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src_pa,
arch/x86/kvm/svm/sev.c
1248
.handle = to_kvm_sev_info(kvm)->handle,
arch/x86/kvm/svm/sev.c
1254
return sev_issue_cmd(kvm, cmd, &data, error);
arch/x86/kvm/svm/sev.c
1287
static int sev_dbg_decrypt_slow(struct kvm *kvm, unsigned long src,
arch/x86/kvm/svm/sev.c
129
lockdep_assert_held(&kvm->lock);
arch/x86/kvm/svm/sev.c
1300
r = sev_issue_dbg_cmd(kvm, src_pa, __sme_set(__pa(buf)),
arch/x86/kvm/svm/sev.c
1312
static int sev_dbg_encrypt_slow(struct kvm *kvm, unsigned long src,
arch/x86/kvm/svm/sev.c
1326
r = sev_issue_dbg_cmd(kvm, dst_pa, __sme_set(__pa(buf)),
arch/x86/kvm/svm/sev.c
133
static bool sev_guest(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
1338
r = sev_issue_dbg_cmd(kvm, __sme_set(__pa(buf)), dst_pa,
arch/x86/kvm/svm/sev.c
1345
static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp,
arch/x86/kvm/svm/sev.c
135
kvm_lockdep_assert_sev_lock_held(kvm);
arch/x86/kvm/svm/sev.c
1351
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
136
return ____sev_guest(kvm);
arch/x86/kvm/svm/sev.c
138
static bool sev_es_guest(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
1382
src_p = sev_pin_page(kvm, src & PAGE_MASK, 0);
arch/x86/kvm/svm/sev.c
1386
dst_p = sev_pin_page(kvm, dst & PAGE_MASK, FOLL_WRITE);
arch/x86/kvm/svm/sev.c
1388
sev_unpin_page(kvm, src_p);
arch/x86/kvm/svm/sev.c
140
kvm_lockdep_assert_sev_lock_held(kvm);
arch/x86/kvm/svm/sev.c
1401
ret = sev_issue_dbg_cmd(kvm,
arch/x86/kvm/svm/sev.c
1406
ret = sev_dbg_decrypt_slow(kvm, src, src_p, dst,
arch/x86/kvm/svm/sev.c
1409
ret = sev_dbg_encrypt_slow(kvm, src, dst, dst_p,
arch/x86/kvm/svm/sev.c
141
return ____sev_es_guest(kvm);
arch/x86/kvm/svm/sev.c
1412
sev_unpin_page(kvm, src_p);
arch/x86/kvm/svm/sev.c
1413
sev_unpin_page(kvm, dst_p);
arch/x86/kvm/svm/sev.c
1421
static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
1430
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
1436
pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, FOLL_WRITE);
arch/x86/kvm/svm/sev.c
144
static bool sev_snp_guest(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
146
kvm_lockdep_assert_sev_lock_held(kvm);
arch/x86/kvm/svm/sev.c
147
return ____sev_snp_guest(kvm);
arch/x86/kvm/svm/sev.c
1478
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1479
ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, &data, &argp->error);
arch/x86/kvm/svm/sev.c
1491
sev_unpin_memory(kvm, pages, n);
arch/x86/kvm/svm/sev.c
1495
static int sev_get_attestation_report(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
150
static int snp_decommission_context(struct kvm *kvm);
arch/x86/kvm/svm/sev.c
1504
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
1530
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1531
ret = sev_issue_cmd(kvm, SEV_CMD_ATTESTATION_REPORT, &data, &argp->error);
arch/x86/kvm/svm/sev.c
1557
__sev_send_start_query_session_length(struct kvm *kvm, struct kvm_sev_cmd *argp,
arch/x86/kvm/svm/sev.c
1564
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1565
ret = sev_issue_cmd(kvm, SEV_CMD_SEND_START, &data, &argp->error);
arch/x86/kvm/svm/sev.c
1575
static int sev_send_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
1583
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
1592
return __sev_send_start_query_session_length(kvm, argp,
arch/x86/kvm/svm/sev.c
1637
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1639
ret = sev_issue_cmd(kvm, SEV_CMD_SEND_START, &data, &argp->error);
arch/x86/kvm/svm/sev.c
1666
__sev_send_update_data_query_lengths(struct kvm *kvm, struct kvm_sev_cmd *argp,
arch/x86/kvm/svm/sev.c
1673
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1674
ret = sev_issue_cmd(kvm, SEV_CMD_SEND_UPDATE_DATA, &data, &argp->error);
arch/x86/kvm/svm/sev.c
1686
static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
1694
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
1703
return __sev_send_update_data_query_lengths(kvm, argp, &params);
arch/x86/kvm/svm/sev.c
1715
guest_page = sev_pin_page(kvm, params.guest_uaddr & PAGE_MASK, 0);
arch/x86/kvm/svm/sev.c
1739
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1741
ret = sev_issue_cmd(kvm, SEV_CMD_SEND_UPDATE_DATA, &data, &argp->error);
arch/x86/kvm/svm/sev.c
1763
sev_unpin_page(kvm, guest_page);
arch/x86/kvm/svm/sev.c
1767
static int sev_send_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
1771
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
1774
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1775
return sev_issue_cmd(kvm, SEV_CMD_SEND_FINISH, &data, &argp->error);
arch/x86/kvm/svm/sev.c
1778
static int sev_send_cancel(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
1782
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
1785
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1786
return sev_issue_cmd(kvm, SEV_CMD_SEND_CANCEL, &data, &argp->error);
arch/x86/kvm/svm/sev.c
1789
static int sev_receive_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
1791
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
1799
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
1838
ret = sev_bind_asid(kvm, start.handle, error);
arch/x86/kvm/svm/sev.c
1848
sev_unbind_asid(kvm, start.handle);
arch/x86/kvm/svm/sev.c
1863
static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
1871
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
1905
guest_page = sev_pin_page(kvm, params.guest_uaddr & PAGE_MASK, FOLL_WRITE);
arch/x86/kvm/svm/sev.c
1922
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1924
ret = sev_issue_cmd(kvm, SEV_CMD_RECEIVE_UPDATE_DATA, &data,
arch/x86/kvm/svm/sev.c
1927
sev_unpin_page(kvm, guest_page);
arch/x86/kvm/svm/sev.c
1937
static int sev_receive_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
194
static inline bool is_mirroring_enc_context(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
1941
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
1944
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
1945
return sev_issue_cmd(kvm, SEV_CMD_RECEIVE_FINISH, &data, &argp->error);
arch/x86/kvm/svm/sev.c
196
return !!to_kvm_sev_info(kvm)->enc_context_owner;
arch/x86/kvm/svm/sev.c
1962
static int sev_lock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm)
arch/x86/kvm/svm/sev.c
1997
static void sev_unlock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm)
arch/x86/kvm/svm/sev.c
2008
static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm)
arch/x86/kvm/svm/sev.c
202
struct kvm_sev_info *sev = to_kvm_sev_info(vcpu->kvm);
arch/x86/kvm/svm/sev.c
207
static bool snp_is_secure_tsc_enabled(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
2089
static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src)
arch/x86/kvm/svm/sev.c
209
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2112
int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd)
arch/x86/kvm/svm/sev.c
2114
struct kvm_sev_info *dst_sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2117
struct kvm *source_kvm;
arch/x86/kvm/svm/sev.c
212
!WARN_ON_ONCE(!sev_snp_guest(kvm));
arch/x86/kvm/svm/sev.c
2128
ret = sev_lock_two_vms(kvm, source_kvm);
arch/x86/kvm/svm/sev.c
2133
if (kvm->arch.vm_type != source_kvm->arch.vm_type ||
arch/x86/kvm/svm/sev.c
2134
sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm)) {
arch/x86/kvm/svm/sev.c
2150
ret = kvm_lock_all_vcpus(kvm);
arch/x86/kvm/svm/sev.c
2157
ret = sev_check_source_vcpus(kvm, source_kvm);
arch/x86/kvm/svm/sev.c
2172
sev_migrate_from(kvm, source_kvm);
arch/x86/kvm/svm/sev.c
2180
kvm_unlock_all_vcpus(kvm);
arch/x86/kvm/svm/sev.c
2188
sev_unlock_two_vms(kvm, source_kvm);
arch/x86/kvm/svm/sev.c
2220
static void *snp_context_create(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
2243
static int snp_bind_asid(struct kvm *kvm, int *error)
arch/x86/kvm/svm/sev.c
2245
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2249
data.asid = sev_get_asid(kvm);
arch/x86/kvm/svm/sev.c
2250
return sev_issue_cmd(kvm, SEV_CMD_SNP_ACTIVATE, &data, error);
arch/x86/kvm/svm/sev.c
2253
static int snp_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
2255
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2260
if (!sev_snp_guest(kvm))
arch/x86/kvm/svm/sev.c
2280
if (snp_is_secure_tsc_enabled(kvm)) {
arch/x86/kvm/svm/sev.c
2281
if (WARN_ON_ONCE(!kvm->arch.default_tsc_khz))
arch/x86/kvm/svm/sev.c
2284
start.desired_tsc_khz = kvm->arch.default_tsc_khz;
arch/x86/kvm/svm/sev.c
2287
sev->snp_context = snp_context_create(kvm, argp);
arch/x86/kvm/svm/sev.c
2304
rc = snp_bind_asid(kvm, &argp->error);
arch/x86/kvm/svm/sev.c
2314
snp_decommission_context(kvm);
arch/x86/kvm/svm/sev.c
2325
static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
arch/x86/kvm/svm/sev.c
2330
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2357
sev_get_asid(kvm), true);
arch/x86/kvm/svm/sev.c
2380
if (ret && !snp_page_reclaim(kvm, pfn) &&
arch/x86/kvm/svm/sev.c
2400
static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
2402
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2409
if (!sev_snp_guest(kvm) || !sev->snp_context)
arch/x86/kvm/svm/sev.c
2452
guard(mutex)(&kvm->slots_lock);
arch/x86/kvm/svm/sev.c
2454
memslot = gfn_to_memslot(kvm, params.gfn_start);
arch/x86/kvm/svm/sev.c
2461
count = kvm_gmem_populate(kvm, params.gfn_start, src, npages,
arch/x86/kvm/svm/sev.c
2482
static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
2484
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2490
if (kvm_is_vcpu_creation_in_progress(kvm))
arch/x86/kvm/svm/sev.c
2493
ret = kvm_lock_all_vcpus(kvm);
arch/x86/kvm/svm/sev.c
2500
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/svm/sev.c
2518
snp_page_reclaim(kvm, pfn);
arch/x86/kvm/svm/sev.c
2535
kvm_unlock_all_vcpus(kvm);
arch/x86/kvm/svm/sev.c
2539
static int snp_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
2541
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2547
if (!sev_snp_guest(kvm))
arch/x86/kvm/svm/sev.c
2560
ret = snp_launch_update_vmsa(kvm, argp);
arch/x86/kvm/svm/sev.c
2594
ret = sev_issue_cmd(kvm, SEV_CMD_SNP_LAUNCH_FINISH, data, &argp->error);
arch/x86/kvm/svm/sev.c
2602
kvm->arch.pre_fault_allowed = true;
arch/x86/kvm/svm/sev.c
2615
static int snp_enable_certs(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
2617
if (kvm->created_vcpus || !sev_snp_guest(kvm))
arch/x86/kvm/svm/sev.c
2620
to_kvm_sev_info(kvm)->snp_certs_enabled = true;
arch/x86/kvm/svm/sev.c
2625
int sev_mem_enc_ioctl(struct kvm *kvm, void __user *argp)
arch/x86/kvm/svm/sev.c
2639
guard(mutex)(&kvm->lock);
arch/x86/kvm/svm/sev.c
2642
if (is_mirroring_enc_context(kvm) &&
arch/x86/kvm/svm/sev.c
2650
if (sev_snp_guest(kvm) && sev_cmd.id < KVM_SEV_SNP_LAUNCH_START)
arch/x86/kvm/svm/sev.c
2659
r = sev_guest_init(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2662
r = sev_guest_init2(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2665
r = sev_launch_start(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2668
r = sev_launch_update_data(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2671
r = sev_launch_update_vmsa(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2674
r = sev_launch_measure(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2677
r = sev_launch_finish(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2680
r = sev_guest_status(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2684
r = sev_dbg_crypt(kvm, &sev_cmd, sev_cmd.id);
arch/x86/kvm/svm/sev.c
2687
r = sev_launch_secret(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2690
r = sev_get_attestation_report(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2693
r = sev_send_start(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2696
r = sev_send_update_data(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2699
r = sev_send_finish(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2702
r = sev_send_cancel(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2705
r = sev_receive_start(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2708
r = sev_receive_update_data(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2711
r = sev_receive_finish(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2714
r = snp_launch_start(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2717
r = snp_launch_update(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2720
r = snp_launch_finish(kvm, &sev_cmd);
arch/x86/kvm/svm/sev.c
2723
r = snp_enable_certs(kvm);
arch/x86/kvm/svm/sev.c
2735
int sev_mem_enc_register_region(struct kvm *kvm,
arch/x86/kvm/svm/sev.c
2738
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2742
guard(mutex)(&kvm->lock);
arch/x86/kvm/svm/sev.c
2744
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
2748
if (is_mirroring_enc_context(kvm))
arch/x86/kvm/svm/sev.c
2755
region->pages = sev_pin_memory(kvm, range->addr, range->size, &region->npages,
arch/x86/kvm/svm/sev.c
2784
find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
arch/x86/kvm/svm/sev.c
2786
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2799
static void __unregister_enc_region_locked(struct kvm *kvm,
arch/x86/kvm/svm/sev.c
2802
sev_unpin_memory(kvm, region->pages, region->npages);
arch/x86/kvm/svm/sev.c
2807
int sev_mem_enc_unregister_region(struct kvm *kvm,
arch/x86/kvm/svm/sev.c
2813
if (is_mirroring_enc_context(kvm))
arch/x86/kvm/svm/sev.c
2816
guard(mutex)(&kvm->lock);
arch/x86/kvm/svm/sev.c
2818
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
2821
region = find_enc_region(kvm, range);
arch/x86/kvm/svm/sev.c
2825
sev_writeback_caches(kvm);
arch/x86/kvm/svm/sev.c
2827
__unregister_enc_region_locked(kvm, region);
arch/x86/kvm/svm/sev.c
2832
int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd)
arch/x86/kvm/svm/sev.c
2835
struct kvm *source_kvm;
arch/x86/kvm/svm/sev.c
2846
ret = sev_lock_two_vms(kvm, source_kvm);
arch/x86/kvm/svm/sev.c
2857
if (sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm) ||
arch/x86/kvm/svm/sev.c
2858
is_mirroring_enc_context(source_kvm) || kvm->created_vcpus) {
arch/x86/kvm/svm/sev.c
2863
mirror_sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2896
sev_unlock_two_vms(kvm, source_kvm);
arch/x86/kvm/svm/sev.c
2900
static int snp_decommission_context(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
2902
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2925
void sev_vm_init(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
2927
switch (kvm->arch.vm_type) {
arch/x86/kvm/svm/sev.c
2932
kvm->arch.has_private_mem = true;
arch/x86/kvm/svm/sev.c
2935
kvm->arch.has_protected_state = true;
arch/x86/kvm/svm/sev.c
2938
kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem;
arch/x86/kvm/svm/sev.c
2939
to_kvm_sev_info(kvm)->need_init = true;
arch/x86/kvm/svm/sev.c
2942
WARN_ONCE(1, "Unsupported VM type %u", kvm->arch.vm_type);
arch/x86/kvm/svm/sev.c
2947
void sev_vm_destroy(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
2949
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
2953
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
2965
if (is_mirroring_enc_context(kvm)) {
arch/x86/kvm/svm/sev.c
2966
struct kvm *owner_kvm = sev->enc_context_owner;
arch/x86/kvm/svm/sev.c
2982
__unregister_enc_region_locked(kvm,
arch/x86/kvm/svm/sev.c
2988
if (sev_snp_guest(kvm)) {
arch/x86/kvm/svm/sev.c
2989
snp_guest_req_cleanup(kvm);
arch/x86/kvm/svm/sev.c
2995
if (snp_decommission_context(kvm))
arch/x86/kvm/svm/sev.c
2998
sev_unbind_asid(kvm, sev->handle);
arch/x86/kvm/svm/sev.c
316
static unsigned int sev_get_asid(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
318
return to_kvm_sev_info(kvm)->asid;
arch/x86/kvm/svm/sev.c
3261
unsigned int asid = sev_get_asid(vcpu->kvm);
arch/x86/kvm/svm/sev.c
3292
sev_writeback_caches(vcpu->kvm);
arch/x86/kvm/svm/sev.c
3295
void sev_guest_memory_reclaimed(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
3308
if (!____sev_guest(kvm) || ____sev_snp_guest(kvm))
arch/x86/kvm/svm/sev.c
3311
sev_writeback_caches(kvm);
arch/x86/kvm/svm/sev.c
3495
kvm_write_guest(svm->vcpu.kvm, svm->sev_es.sw_scratch,
arch/x86/kvm/svm/sev.c
3524
if (kvm_rmp_make_shared(vcpu->kvm, pfn, PG_LEVEL_4K))
arch/x86/kvm/svm/sev.c
3541
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/svm/sev.c
3542
unsigned int asid = sev_get_asid(kvm);
arch/x86/kvm/svm/sev.c
3558
if (!cpumask_test_cpu(cpu, to_kvm_sev_info(kvm)->have_run_cpus))
arch/x86/kvm/svm/sev.c
3559
cpumask_set_cpu(cpu, to_kvm_sev_info(kvm)->have_run_cpus);
arch/x86/kvm/svm/sev.c
358
static int kvm_rmp_make_shared(struct kvm *kvm, u64 pfn, enum pg_level level)
arch/x86/kvm/svm/sev.c
360
if (KVM_BUG_ON(rmp_make_shared(pfn, level), kvm)) {
arch/x86/kvm/svm/sev.c
3632
if (to_kvm_sev_info(svm->vcpu.kvm)->ghcb_version >= 2)
arch/x86/kvm/svm/sev.c
3648
if (kvm_read_guest(svm->vcpu.kvm, scratch_gpa_beg, scratch_va, min_len)) {
arch/x86/kvm/svm/sev.c
3737
if (!user_exit_on_hypercall(vcpu->kvm, KVM_HC_MAP_GPA_RANGE)) {
arch/x86/kvm/svm/sev.c
382
static int snp_page_reclaim(struct kvm *kvm, u64 pfn)
arch/x86/kvm/svm/sev.c
389
if (KVM_BUG(rc, kvm, "Failed to reclaim PFN %llx, rc %d fw_err %d", pfn, rc, fw_err)) {
arch/x86/kvm/svm/sev.c
394
if (kvm_rmp_make_shared(kvm, pfn, PG_LEVEL_4K))
arch/x86/kvm/svm/sev.c
3951
if (!user_exit_on_hypercall(svm->vcpu.kvm, KVM_HC_MAP_GPA_RANGE)) {
arch/x86/kvm/svm/sev.c
400
static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
arch/x86/kvm/svm/sev.c
4022
slot = gfn_to_memslot(vcpu->kvm, gfn);
arch/x86/kvm/svm/sev.c
4030
if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL))
arch/x86/kvm/svm/sev.c
4060
struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm);
arch/x86/kvm/svm/sev.c
4071
target_vcpu = kvm_get_vcpu_by_id(vcpu->kvm, apic_id);
arch/x86/kvm/svm/sev.c
4137
struct kvm *kvm = svm->vcpu.kvm;
arch/x86/kvm/svm/sev.c
4138
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
4147
if (kvm_read_guest(kvm, req_gpa, sev->guest_req_buf, PAGE_SIZE))
arch/x86/kvm/svm/sev.c
4159
ret = sev_issue_cmd(kvm, SEV_CMD_SNP_GUEST_REQUEST, &data, &fw_err);
arch/x86/kvm/svm/sev.c
4163
if (kvm_write_guest(kvm, resp_gpa, sev->guest_resp_buf, PAGE_SIZE))
arch/x86/kvm/svm/sev.c
4206
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/svm/sev.c
4213
if (kvm_read_guest(kvm, req_gpa + offsetof(struct snp_guest_msg_hdr, msg_type),
arch/x86/kvm/svm/sev.c
4228
struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
arch/x86/kvm/svm/sev.c
4254
if (data_npages && kvm_clear_guest(kvm, data_gpa, 24))
arch/x86/kvm/svm/sev.c
4269
struct kvm_sev_info *sev = to_kvm_sev_info(vcpu->kvm);
arch/x86/kvm/svm/sev.c
4521
if (to_kvm_sev_info(vcpu->kvm)->ghcb_version >= 2 && len > 8) {
arch/x86/kvm/svm/sev.c
454
static int snp_guest_req_init(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
4542
struct kvm_sev_info *sev = to_kvm_sev_info(vcpu->kvm);
arch/x86/kvm/svm/sev.c
456
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
4615
KVM_BUG_ON(1, vcpu->kvm);
arch/x86/kvm/svm/sev.c
4656
!snp_is_secure_tsc_enabled(vcpu->kvm));
arch/x86/kvm/svm/sev.c
4688
struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm);
arch/x86/kvm/svm/sev.c
475
static void snp_guest_req_cleanup(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
477
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
4798
vcpu->arch.guest_tsc_protected = snp_is_secure_tsc_enabled(vcpu->kvm);
arch/x86/kvm/svm/sev.c
489
static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp,
arch/x86/kvm/svm/sev.c
493
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
4934
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/svm/sev.c
4950
if (!kvm_mem_is_private(kvm, gfn)) {
arch/x86/kvm/svm/sev.c
4956
slot = gfn_to_memslot(kvm, gfn);
arch/x86/kvm/svm/sev.c
4963
ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &page, &order);
arch/x86/kvm/svm/sev.c
500
if (kvm->created_vcpus)
arch/x86/kvm/svm/sev.c
5017
kvm_zap_gfn_range(kvm, gfn, gfn + PTRS_PER_PMD);
arch/x86/kvm/svm/sev.c
5059
static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order)
arch/x86/kvm/svm/sev.c
5075
int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order)
arch/x86/kvm/svm/sev.c
5077
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
5083
if (!sev_snp_guest(kvm))
arch/x86/kvm/svm/sev.c
5099
if (is_large_rmp_possible(kvm, pfn, max_order)) {
arch/x86/kvm/svm/sev.c
5185
int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private)
arch/x86/kvm/svm/sev.c
5190
if (!sev_snp_guest(kvm))
arch/x86/kvm/svm/sev.c
5218
sev = to_kvm_sev_info(vcpu->kvm);
arch/x86/kvm/svm/sev.c
5247
if (snp_page_reclaim(vcpu->kvm, PHYS_PFN(__pa(vmsa))))
arch/x86/kvm/svm/sev.c
553
ret = snp_guest_req_init(kvm);
arch/x86/kvm/svm/sev.c
562
kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_SEV);
arch/x86/kvm/svm/sev.c
579
static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
587
if (kvm->arch.vm_type != KVM_X86_DEFAULT_VM)
arch/x86/kvm/svm/sev.c
599
return __sev_guest_init(kvm, argp, &data, vm_type);
arch/x86/kvm/svm/sev.c
602
static int sev_guest_init2(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
606
if (!to_kvm_sev_info(kvm)->need_init)
arch/x86/kvm/svm/sev.c
609
if (kvm->arch.vm_type != KVM_X86_SEV_VM &&
arch/x86/kvm/svm/sev.c
610
kvm->arch.vm_type != KVM_X86_SEV_ES_VM &&
arch/x86/kvm/svm/sev.c
611
kvm->arch.vm_type != KVM_X86_SNP_VM)
arch/x86/kvm/svm/sev.c
617
return __sev_guest_init(kvm, argp, &data, kvm->arch.vm_type);
arch/x86/kvm/svm/sev.c
620
static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
arch/x86/kvm/svm/sev.c
622
unsigned int asid = sev_get_asid(kvm);
arch/x86/kvm/svm/sev.c
644
static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
arch/x86/kvm/svm/sev.c
646
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
651
static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
653
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/sev.c
660
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
699
ret = sev_bind_asid(kvm, start.handle, error);
arch/x86/kvm/svm/sev.c
708
sev_unbind_asid(kvm, start.handle);
arch/x86/kvm/svm/sev.c
724
static int sev_check_pin_count(struct kvm *kvm, unsigned long npages)
arch/x86/kvm/svm/sev.c
728
total_npages = to_kvm_sev_info(kvm)->pages_locked + npages;
arch/x86/kvm/svm/sev.c
742
static int sev_pin_user_pages(struct kvm *kvm, unsigned long addr, int npages,
arch/x86/kvm/svm/sev.c
747
lockdep_assert_held(&kvm->lock);
arch/x86/kvm/svm/sev.c
757
to_kvm_sev_info(kvm)->pages_locked += npages;
arch/x86/kvm/svm/sev.c
761
static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
arch/x86/kvm/svm/sev.c
769
lockdep_assert_held(&kvm->lock);
arch/x86/kvm/svm/sev.c
783
ret = sev_check_pin_count(kvm, npages);
arch/x86/kvm/svm/sev.c
797
ret = sev_pin_user_pages(kvm, uaddr, npages, flags, pages);
arch/x86/kvm/svm/sev.c
807
static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
arch/x86/kvm/svm/sev.c
812
to_kvm_sev_info(kvm)->pages_locked -= npages;
arch/x86/kvm/svm/sev.c
815
static struct page *sev_pin_page(struct kvm *kvm, unsigned long addr,
arch/x86/kvm/svm/sev.c
821
r = sev_check_pin_count(kvm, 1);
arch/x86/kvm/svm/sev.c
825
r = sev_pin_user_pages(kvm, addr, 1, flags, &page);
arch/x86/kvm/svm/sev.c
832
static void sev_unpin_page(struct kvm *kvm, struct page *page)
arch/x86/kvm/svm/sev.c
835
to_kvm_sev_info(kvm)->pages_locked -= 1;
arch/x86/kvm/svm/sev.c
855
static void sev_writeback_caches(struct kvm *kvm)
arch/x86/kvm/svm/sev.c
875
wbnoinvd_on_cpus_mask(to_kvm_sev_info(kvm)->have_run_cpus);
arch/x86/kvm/svm/sev.c
899
static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
arch/x86/kvm/svm/sev.c
907
if (!sev_guest(kvm))
arch/x86/kvm/svm/sev.c
918
inpages = sev_pin_memory(kvm, vaddr, size, &npages, FOLL_WRITE);
arch/x86/kvm/svm/sev.c
929
data.handle = to_kvm_sev_info(kvm)->handle;
arch/x86/kvm/svm/sev.c
947
ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, &data, &argp->error);
arch/x86/kvm/svm/sev.c
962
sev_unpin_memory(kvm, inpages, npages);
arch/x86/kvm/svm/sev.c
969
struct kvm_sev_info *sev = to_kvm_sev_info(vcpu->kvm);
arch/x86/kvm/svm/svm.c
1185
if (!kvm_mwait_in_guest(vcpu->kvm)) {
arch/x86/kvm/svm/svm.c
1190
if (!kvm_hlt_in_guest(vcpu->kvm)) {
arch/x86/kvm/svm/svm.c
1242
if (!kvm_pause_in_guest(vcpu->kvm)) {
arch/x86/kvm/svm/svm.c
1254
if (enable_apicv && irqchip_in_kernel(vcpu->kvm))
arch/x86/kvm/svm/svm.c
1266
if (vcpu->kvm->arch.bus_lock_detection_enabled)
arch/x86/kvm/svm/svm.c
1287
if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS))
arch/x86/kvm/svm/svm.c
1314
static int svm_vcpu_precreate(struct kvm *kvm)
arch/x86/kvm/svm/svm.c
1316
return avic_alloc_physical_id_table(kvm);
arch/x86/kvm/svm/svm.c
1485
if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm))
arch/x86/kvm/svm/svm.c
1553
KVM_BUG_ON(1, vcpu->kvm);
arch/x86/kvm/svm/svm.c
1813
if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
arch/x86/kvm/svm/svm.c
2791
KVM_BUG_ON(1, svm->vcpu.kvm);
arch/x86/kvm/svm/svm.c
2823
return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
arch/x86/kvm/svm/svm.c
2989
return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
arch/x86/kvm/svm/svm.c
3860
kvm_dec_apicv_irq_window_req(svm->vcpu.kvm);
arch/x86/kvm/svm/svm.c
4113
kvm_inc_apicv_irq_window_req(vcpu->kvm);
arch/x86/kvm/svm/svm.c
4225
hv_flush_remote_tlbs(vcpu->kvm);
arch/x86/kvm/svm/svm.c
4400
if (to_kvm_sev_info(vcpu->kvm)->need_init)
arch/x86/kvm/svm/svm.c
4681
static bool svm_tdp_has_smep(struct kvm *kvm)
arch/x86/kvm/svm/svm.c
4690
static bool svm_has_emulated_msr(struct kvm *kvm, u32 index)
arch/x86/kvm/svm/svm.c
4707
if (kvm && ____sev_es_guest(kvm))
arch/x86/kvm/svm/svm.c
5295
static void svm_vm_destroy(struct kvm *kvm)
arch/x86/kvm/svm/svm.c
5297
avic_vm_destroy(kvm);
arch/x86/kvm/svm/svm.c
5298
sev_vm_destroy(kvm);
arch/x86/kvm/svm/svm.c
5303
static int svm_vm_init(struct kvm *kvm)
arch/x86/kvm/svm/svm.c
5305
sev_vm_init(kvm);
arch/x86/kvm/svm/svm.c
5308
kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE);
arch/x86/kvm/svm/svm.c
5311
int ret = avic_vm_init(kvm);
arch/x86/kvm/svm/svm.c
840
if (kvm_aperfmperf_in_guest(vcpu->kvm)) {
arch/x86/kvm/svm/svm.c
880
KVM_BUG_ON(is_sev_es_guest(vcpu), vcpu->kvm);
arch/x86/kvm/svm/svm.c
926
WARN_ON_ONCE(kvm_pause_in_guest(vcpu->kvm));
arch/x86/kvm/svm/svm.c
955
WARN_ON_ONCE(kvm_pause_in_guest(vcpu->kvm));
arch/x86/kvm/svm/svm.h
1001
void sev_vm_init(struct kvm *kvm);
arch/x86/kvm/svm/svm.h
1002
void sev_vm_destroy(struct kvm *kvm);
arch/x86/kvm/svm/svm.h
1010
int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order);
arch/x86/kvm/svm/svm.h
1012
int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private);
arch/x86/kvm/svm/svm.h
1028
static inline void sev_vm_init(struct kvm *kvm) {}
arch/x86/kvm/svm/svm.h
1029
static inline void sev_vm_destroy(struct kvm *kvm) {}
arch/x86/kvm/svm/svm.h
1037
static inline int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order)
arch/x86/kvm/svm/svm.h
1042
static inline int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private)
arch/x86/kvm/svm/svm.h
112
struct kvm *enc_context_owner; /* Owner of copied encryption context */
arch/x86/kvm/svm/svm.h
127
struct kvm kvm;
arch/x86/kvm/svm/svm.h
388
static __always_inline struct kvm_svm *to_kvm_svm(struct kvm *kvm)
arch/x86/kvm/svm/svm.h
390
return container_of(kvm, struct kvm_svm, kvm);
arch/x86/kvm/svm/svm.h
394
static __always_inline struct kvm_sev_info *to_kvm_sev_info(struct kvm *kvm)
arch/x86/kvm/svm/svm.h
396
return &to_kvm_svm(kvm)->sev_info;
arch/x86/kvm/svm/svm.h
399
static __always_inline bool ____sev_guest(struct kvm *kvm)
arch/x86/kvm/svm/svm.h
401
return to_kvm_sev_info(kvm)->active;
arch/x86/kvm/svm/svm.h
403
static __always_inline bool ____sev_es_guest(struct kvm *kvm)
arch/x86/kvm/svm/svm.h
405
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/svm.h
410
static __always_inline bool ____sev_snp_guest(struct kvm *kvm)
arch/x86/kvm/svm/svm.h
412
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
arch/x86/kvm/svm/svm.h
415
!WARN_ON_ONCE(!____sev_es_guest(kvm));
arch/x86/kvm/svm/svm.h
420
return ____sev_guest(vcpu->kvm);
arch/x86/kvm/svm/svm.h
424
return ____sev_es_guest(vcpu->kvm);
arch/x86/kvm/svm/svm.h
429
return ____sev_snp_guest(vcpu->kvm);
arch/x86/kvm/svm/svm.h
663
!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT);
arch/x86/kvm/svm/svm.h
948
int avic_alloc_physical_id_table(struct kvm *kvm);
arch/x86/kvm/svm/svm.h
949
void avic_vm_destroy(struct kvm *kvm);
arch/x86/kvm/svm/svm.h
950
int avic_vm_init(struct kvm *kvm);
arch/x86/kvm/svm/svm.h
959
int avic_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
arch/x86/kvm/svm/svm.h
981
int sev_mem_enc_ioctl(struct kvm *kvm, void __user *argp);
arch/x86/kvm/svm/svm.h
982
int sev_mem_enc_register_region(struct kvm *kvm,
arch/x86/kvm/svm/svm.h
984
int sev_mem_enc_unregister_region(struct kvm *kvm,
arch/x86/kvm/svm/svm.h
986
int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd);
arch/x86/kvm/svm/svm.h
987
int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd);
arch/x86/kvm/svm/svm.h
988
void sev_guest_memory_reclaimed(struct kvm *kvm);
arch/x86/kvm/svm/svm_onhyperv.c
29
hve->hv_vm_id = (unsigned long)vcpu->kvm;
arch/x86/kvm/trace.h
12
#define TRACE_SYSTEM kvm
arch/x86/kvm/trace.h
1971
#define TRACE_INCLUDE_PATH ../../arch/x86/kvm
arch/x86/kvm/vmx/common.h
113
if (vt_is_tdx_private_gpa(vcpu->kvm, gpa))
arch/x86/kvm/vmx/common.h
60
static __always_inline bool is_td(struct kvm *kvm)
arch/x86/kvm/vmx/common.h
62
return kvm->arch.vm_type == KVM_X86_TDX_VM;
arch/x86/kvm/vmx/common.h
67
return is_td(vcpu->kvm);
arch/x86/kvm/vmx/common.h
72
static __always_inline bool is_td(struct kvm *kvm) { return false; }
arch/x86/kvm/vmx/common.h
77
static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa)
arch/x86/kvm/vmx/common.h
80
return !kvm_is_addr_direct(kvm, gpa);
arch/x86/kvm/vmx/main.c
180
static bool vt_has_emulated_msr(struct kvm *kvm, u32 index)
arch/x86/kvm/vmx/main.c
182
if (kvm && is_td(kvm))
arch/x86/kvm/vmx/main.c
185
return vmx_has_emulated_msr(kvm, index);
arch/x86/kvm/vmx/main.c
221
if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm))
arch/x86/kvm/vmx/main.c
229
if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm))
arch/x86/kvm/vmx/main.c
237
if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm))
arch/x86/kvm/vmx/main.c
245
if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm))
arch/x86/kvm/vmx/main.c
43
static int vt_vm_init(struct kvm *kvm)
arch/x86/kvm/vmx/main.c
45
if (is_td(kvm))
arch/x86/kvm/vmx/main.c
46
return tdx_vm_init(kvm);
arch/x86/kvm/vmx/main.c
48
return vmx_vm_init(kvm);
arch/x86/kvm/vmx/main.c
51
static void vt_vm_pre_destroy(struct kvm *kvm)
arch/x86/kvm/vmx/main.c
53
if (is_td(kvm))
arch/x86/kvm/vmx/main.c
54
return tdx_mmu_release_hkid(kvm);
arch/x86/kvm/vmx/main.c
57
static void vt_vm_destroy(struct kvm *kvm)
arch/x86/kvm/vmx/main.c
59
if (is_td(kvm))
arch/x86/kvm/vmx/main.c
60
return tdx_vm_destroy(kvm);
arch/x86/kvm/vmx/main.c
62
vmx_vm_destroy(kvm);
arch/x86/kvm/vmx/main.c
65
static int vt_vcpu_precreate(struct kvm *kvm)
arch/x86/kvm/vmx/main.c
67
if (is_td(kvm))
arch/x86/kvm/vmx/main.c
70
return vmx_vcpu_precreate(kvm);
arch/x86/kvm/vmx/main.c
727
KVM_BUG_ON(!kvm_vcpu_apicv_active(vcpu), vcpu->kvm);
arch/x86/kvm/vmx/main.c
742
static int vt_set_tss_addr(struct kvm *kvm, unsigned int addr)
arch/x86/kvm/vmx/main.c
744
if (is_td(kvm))
arch/x86/kvm/vmx/main.c
747
return vmx_set_tss_addr(kvm, addr);
arch/x86/kvm/vmx/main.c
750
static int vt_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
arch/x86/kvm/vmx/main.c
752
if (is_td(kvm))
arch/x86/kvm/vmx/main.c
755
return vmx_set_identity_map_addr(kvm, ident_addr);
arch/x86/kvm/vmx/main.c
758
static bool vt_tdp_has_smep(struct kvm *kvm)
arch/x86/kvm/vmx/main.c
760
if (is_td(kvm))
arch/x86/kvm/vmx/main.c
763
return vmx_tdp_has_smep(kvm);
arch/x86/kvm/vmx/main.c
831
static int vt_mem_enc_ioctl(struct kvm *kvm, void __user *argp)
arch/x86/kvm/vmx/main.c
833
if (!is_td(kvm))
arch/x86/kvm/vmx/main.c
836
return tdx_vm_ioctl(kvm, argp);
arch/x86/kvm/vmx/main.c
855
static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn,
arch/x86/kvm/vmx/main.c
858
if (is_td(kvm))
arch/x86/kvm/vmx/main.c
859
return tdx_gmem_max_mapping_level(kvm, pfn, is_private);
arch/x86/kvm/vmx/nested.c
1998
KVM_BUG_ON(1, vmx->vcpu.kvm);
arch/x86/kvm/vmx/nested.c
2176
KVM_BUG_ON(1, vmx->vcpu.kvm);
arch/x86/kvm/vmx/nested.c
2352
struct kvm *kvm = vmx->vcpu.kvm;
arch/x86/kvm/vmx/nested.c
2390
if (kvm_notify_vmexit_enabled(kvm))
arch/x86/kvm/vmx/nested.c
2391
vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window);
arch/x86/kvm/vmx/nested.c
3282
CC(kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc,
arch/x86/kvm/vmx/nested.c
3286
if (CC(kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr,
arch/x86/kvm/vmx/nested.c
3337
if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM))
arch/x86/kvm/vmx/nested.c
3585
if (kvm_write_guest_page(vcpu->kvm, gpa_to_gfn(dst), &gpa,
arch/x86/kvm/vmx/nested.c
374
kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
arch/x86/kvm/vmx/nested.c
5538
if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) ||
arch/x86/kvm/vmx/nested.c
5572
kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
arch/x86/kvm/vmx/nested.c
5899
if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, vmptr, VMCS12_SIZE)) {
arch/x86/kvm/vmx/nested.c
5910
if (kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr,
arch/x86/kvm/vmx/nested.c
5930
if (kvm_read_guest_cached(vcpu->kvm, ghc, vmx->nested.cached_vmcs12,
arch/x86/kvm/vmx/nested.c
6046
kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);
arch/x86/kvm/vmx/nested.c
6145
kvm_mmu_free_guest_mode_roots(vcpu->kvm, &vcpu->arch.root_mmu);
arch/x86/kvm/vmx/nested.c
626
!kvm_read_guest(vcpu->kvm, vtpr_gpa, &vtpr, sizeof(vtpr)) &&
arch/x86/kvm/vmx/nested.c
865
kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc,
arch/x86/kvm/vmx/nested.c
869
kvm_read_guest_cached(vcpu->kvm, ghc, get_shadow_vmcs12(vcpu),
arch/x86/kvm/vmx/nested.c
884
kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc,
arch/x86/kvm/vmx/nested.c
888
kvm_write_guest_cached(vcpu->kvm, ghc, get_shadow_vmcs12(vcpu),
arch/x86/kvm/vmx/nested.h
61
!refcount_read(&vcpu->kvm->users_count));
arch/x86/kvm/vmx/nested.h
69
!refcount_read(&vcpu->kvm->users_count));
arch/x86/kvm/vmx/pmu_intel.c
521
if (KVM_BUG_ON(lbr_desc->msr_passthrough, vcpu->kvm))
arch/x86/kvm/vmx/posted_intr.c
147
static bool vmx_can_use_vtd_pi(struct kvm *kvm)
arch/x86/kvm/vmx/posted_intr.c
154
return irqchip_in_kernel(kvm) && kvm_arch_has_irq_bypass() &&
arch/x86/kvm/vmx/posted_intr.c
155
READ_ONCE(kvm->arch.nr_possible_bypass_irqs);
arch/x86/kvm/vmx/posted_intr.c
220
vmx_can_use_vtd_pi(vcpu->kvm);
arch/x86/kvm/vmx/posted_intr.c
297
void vmx_pi_start_bypass(struct kvm *kvm)
arch/x86/kvm/vmx/posted_intr.c
299
if (WARN_ON_ONCE(!vmx_can_use_vtd_pi(kvm)))
arch/x86/kvm/vmx/posted_intr.c
302
kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK);
arch/x86/kvm/vmx/posted_intr.c
305
int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
arch/x86/kvm/vmx/posted_intr.h
17
int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
arch/x86/kvm/vmx/posted_intr.h
20
void vmx_pi_start_bypass(struct kvm *kvm);
arch/x86/kvm/vmx/sgx.c
167
if (!vcpu->kvm->arch.sgx_provisioning_allowed &&
arch/x86/kvm/vmx/sgx.c
446
if (!vcpu->kvm->arch.sgx_provisioning_allowed)
arch/x86/kvm/vmx/tdx.c
1032
struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
arch/x86/kvm/vmx/tdx.c
1077
if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap)))
arch/x86/kvm/vmx/tdx.c
1239
tdx->vcpu.run->hypercall.args[0] = gpa & ~gfn_to_gpa(kvm_gfn_direct_bits(tdx->vcpu.kvm));
arch/x86/kvm/vmx/tdx.c
1241
tdx->vcpu.run->hypercall.args[2] = vt_is_tdx_private_gpa(tdx->vcpu.kvm, gpa) ?
arch/x86/kvm/vmx/tdx.c
1264
if (!user_exit_on_hypercall(vcpu->kvm, KVM_HC_MAP_GPA_RANGE)) {
arch/x86/kvm/vmx/tdx.c
1271
(vt_is_tdx_private_gpa(vcpu->kvm, gpa) !=
arch/x86/kvm/vmx/tdx.c
1272
vt_is_tdx_private_gpa(vcpu->kvm, gpa + size - 1))) {
arch/x86/kvm/vmx/tdx.c
1460
if (vt_is_tdx_private_gpa(vcpu->kvm, gpa) ||
arch/x86/kvm/vmx/tdx.c
1461
vt_is_tdx_private_gpa(vcpu->kvm, gpa + size - 1))
arch/x86/kvm/vmx/tdx.c
1464
gpa = gpa & ~gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm));
arch/x86/kvm/vmx/tdx.c
1554
if (vt_is_tdx_private_gpa(vcpu->kvm, gpa)) {
arch/x86/kvm/vmx/tdx.c
1563
vcpu->run->tdx.get_quote.gpa = gpa & ~gfn_to_gpa(kvm_gfn_direct_bits(tdx->vcpu.kvm));
arch/x86/kvm/vmx/tdx.c
1618
if (KVM_BUG_ON(shared_bit != kvm_gfn_direct_bits(vcpu->kvm), vcpu->kvm))
arch/x86/kvm/vmx/tdx.c
1624
static int tdx_mem_page_add(struct kvm *kvm, gfn_t gfn, enum pg_level level,
arch/x86/kvm/vmx/tdx.c
1627
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
1631
lockdep_assert_held(&kvm->slots_lock);
arch/x86/kvm/vmx/tdx.c
1633
if (KVM_BUG_ON(kvm->arch.pre_fault_allowed, kvm) ||
arch/x86/kvm/vmx/tdx.c
1634
KVM_BUG_ON(!kvm_tdx->page_add_src, kvm))
arch/x86/kvm/vmx/tdx.c
1642
if (TDX_BUG_ON_2(err, TDH_MEM_PAGE_ADD, entry, level_state, kvm))
arch/x86/kvm/vmx/tdx.c
1648
static int tdx_mem_page_aug(struct kvm *kvm, gfn_t gfn,
arch/x86/kvm/vmx/tdx.c
1651
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
1660
if (TDX_BUG_ON_2(err, TDH_MEM_PAGE_AUG, entry, level_state, kvm))
arch/x86/kvm/vmx/tdx.c
1666
static struct page *tdx_spte_to_sept_pt(struct kvm *kvm, gfn_t gfn,
arch/x86/kvm/vmx/tdx.c
1671
if (KVM_BUG_ON(!sp->external_spt, kvm) ||
arch/x86/kvm/vmx/tdx.c
1672
KVM_BUG_ON(sp->role.level + 1 != level, kvm) ||
arch/x86/kvm/vmx/tdx.c
1673
KVM_BUG_ON(sp->gfn != gfn, kvm))
arch/x86/kvm/vmx/tdx.c
1679
static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn,
arch/x86/kvm/vmx/tdx.c
1686
sept_pt = tdx_spte_to_sept_pt(kvm, gfn, new_spte, level);
arch/x86/kvm/vmx/tdx.c
1690
err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, level, sept_pt,
arch/x86/kvm/vmx/tdx.c
1695
if (TDX_BUG_ON_2(err, TDH_MEM_SEPT_ADD, entry, level_state, kvm))
arch/x86/kvm/vmx/tdx.c
1701
static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level,
arch/x86/kvm/vmx/tdx.c
1704
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
1708
if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
arch/x86/kvm/vmx/tdx.c
1726
return tdx_mem_page_add(kvm, gfn, level, pfn);
arch/x86/kvm/vmx/tdx.c
1728
return tdx_mem_page_aug(kvm, gfn, level, pfn);
arch/x86/kvm/vmx/tdx.c
1755
static void tdx_track(struct kvm *kvm)
arch/x86/kvm/vmx/tdx.c
1757
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
1769
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/vmx/tdx.c
1771
err = tdh_do_no_vcpus(tdh_mem_track, kvm, &kvm_tdx->td);
arch/x86/kvm/vmx/tdx.c
1772
TDX_BUG_ON(err, TDH_MEM_TRACK, kvm);
arch/x86/kvm/vmx/tdx.c
1774
kvm_make_all_cpus_request(kvm, KVM_REQ_OUTSIDE_GUEST_MODE);
arch/x86/kvm/vmx/tdx.c
1777
static int tdx_sept_remove_leaf_spte(struct kvm *kvm, gfn_t gfn,
arch/x86/kvm/vmx/tdx.c
1780
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
1785
lockdep_assert_held_write(&kvm->mmu_lock);
arch/x86/kvm/vmx/tdx.c
1792
if (KVM_BUG_ON(!is_hkid_assigned(to_kvm_tdx(kvm)), kvm))
arch/x86/kvm/vmx/tdx.c
1796
if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
arch/x86/kvm/vmx/tdx.c
1799
err = tdh_do_no_vcpus(tdh_mem_range_block, kvm, &kvm_tdx->td, gpa,
arch/x86/kvm/vmx/tdx.c
1801
if (TDX_BUG_ON_2(err, TDH_MEM_RANGE_BLOCK, entry, level_state, kvm))
arch/x86/kvm/vmx/tdx.c
1808
tdx_track(kvm);
arch/x86/kvm/vmx/tdx.c
1815
err = tdh_do_no_vcpus(tdh_mem_page_remove, kvm, &kvm_tdx->td, gpa,
arch/x86/kvm/vmx/tdx.c
1817
if (TDX_BUG_ON_2(err, TDH_MEM_PAGE_REMOVE, entry, level_state, kvm))
arch/x86/kvm/vmx/tdx.c
1821
if (TDX_BUG_ON(err, TDH_PHYMEM_PAGE_WBINVD, kvm))
arch/x86/kvm/vmx/tdx.c
1845
static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte,
arch/x86/kvm/vmx/tdx.c
1848
lockdep_assert_held(&kvm->mmu_lock);
arch/x86/kvm/vmx/tdx.c
1851
return tdx_sept_remove_leaf_spte(kvm, gfn, level, old_spte);
arch/x86/kvm/vmx/tdx.c
1853
if (KVM_BUG_ON(!is_shadow_present_pte(new_spte), kvm))
arch/x86/kvm/vmx/tdx.c
1857
return tdx_sept_map_nonleaf_spte(kvm, gfn, level, new_spte);
arch/x86/kvm/vmx/tdx.c
1859
return tdx_sept_map_leaf_spte(kvm, gfn, level, new_spte);
arch/x86/kvm/vmx/tdx.c
1866
static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp)
arch/x86/kvm/vmx/tdx.c
1880
if (KVM_BUG_ON(is_hkid_assigned(to_kvm_tdx(kvm)), kvm) ||
arch/x86/kvm/vmx/tdx.c
1924
if (vt_is_tdx_private_gpa(vcpu->kvm, gpa)) {
arch/x86/kvm/vmx/tdx.c
1928
kvm_vm_dead(vcpu->kvm);
arch/x86/kvm/vmx/tdx.c
1951
if (KVM_BUG_ON(exit_qual & EPT_VIOLATION_ACC_INSTR, vcpu->kvm))
arch/x86/kvm/vmx/tdx.c
2044
KVM_BUG_ON(1, vcpu->kvm);
arch/x86/kvm/vmx/tdx.c
2053
KVM_BUG_ON(!virt_rebooting, vcpu->kvm);
arch/x86/kvm/vmx/tdx.c
2396
static int setup_tdparams(struct kvm *kvm, struct td_params *td_params,
arch/x86/kvm/vmx/tdx.c
2403
if (kvm->created_vcpus)
arch/x86/kvm/vmx/tdx.c
2412
td_params->max_vcpus = kvm->max_vcpus;
arch/x86/kvm/vmx/tdx.c
2417
td_params->tsc_frequency = TDX_TSC_KHZ_TO_25MHZ(kvm->arch.default_tsc_khz);
arch/x86/kvm/vmx/tdx.c
2440
static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
arch/x86/kvm/vmx/tdx.c
2443
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
2519
if (TDX_BUG_ON(err, TDH_MNG_CREATE, kvm)) {
arch/x86/kvm/vmx/tdx.c
2560
if (TDX_BUG_ON(err, TDH_MNG_ADDCX, kvm)) {
arch/x86/kvm/vmx/tdx.c
2577
} else if (TDX_BUG_ON_1(err, TDH_MNG_INIT, rcx, kvm)) {
arch/x86/kvm/vmx/tdx.c
2601
tdx_mmu_release_hkid(kvm);
arch/x86/kvm/vmx/tdx.c
2602
tdx_reclaim_td_control_pages(kvm);
arch/x86/kvm/vmx/tdx.c
2646
struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
arch/x86/kvm/vmx/tdx.c
2707
gpa_t gpa_bits = gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm));
arch/x86/kvm/vmx/tdx.c
2728
static tdx_vm_state_guard_t tdx_acquire_vm_state_locks(struct kvm *kvm)
arch/x86/kvm/vmx/tdx.c
2732
mutex_lock(&kvm->lock);
arch/x86/kvm/vmx/tdx.c
2734
if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus)) {
arch/x86/kvm/vmx/tdx.c
2739
r = kvm_lock_all_vcpus(kvm);
arch/x86/kvm/vmx/tdx.c
2747
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/vmx/tdx.c
2748
return kvm;
arch/x86/kvm/vmx/tdx.c
2751
mutex_unlock(&kvm->lock);
arch/x86/kvm/vmx/tdx.c
2755
static void tdx_release_vm_state_locks(struct kvm *kvm)
arch/x86/kvm/vmx/tdx.c
2757
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/vmx/tdx.c
2758
kvm_unlock_all_vcpus(kvm);
arch/x86/kvm/vmx/tdx.c
2759
mutex_unlock(&kvm->lock);
arch/x86/kvm/vmx/tdx.c
2764
tdx_acquire_vm_state_locks(kvm), struct kvm *kvm);
arch/x86/kvm/vmx/tdx.c
2766
static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd)
arch/x86/kvm/vmx/tdx.c
2769
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
2815
ret = setup_tdparams(kvm, td_params, init_vm);
arch/x86/kvm/vmx/tdx.c
2819
ret = __tdx_td_init(kvm, td_params, &cmd->hw_error);
arch/x86/kvm/vmx/tdx.c
2829
kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_5;
arch/x86/kvm/vmx/tdx.c
2831
kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_4;
arch/x86/kvm/vmx/tdx.c
2877
static int tdx_td_finalize(struct kvm *kvm, struct kvm_tdx_cmd *cmd)
arch/x86/kvm/vmx/tdx.c
2879
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
2887
if (TDX_BUG_ON(cmd->hw_error, TDH_MR_FINALIZE, kvm))
arch/x86/kvm/vmx/tdx.c
2893
kvm->arch.pre_fault_allowed = true;
arch/x86/kvm/vmx/tdx.c
2912
int tdx_vm_ioctl(struct kvm *kvm, void __user *argp)
arch/x86/kvm/vmx/tdx.c
2924
CLASS(tdx_vm_state_guard, guard)(kvm);
arch/x86/kvm/vmx/tdx.c
2930
r = tdx_td_init(kvm, &tdx_cmd);
arch/x86/kvm/vmx/tdx.c
2933
r = tdx_td_finalize(kvm, &tdx_cmd);
arch/x86/kvm/vmx/tdx.c
2948
struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
arch/x86/kvm/vmx/tdx.c
2983
if (TDX_BUG_ON(err, TDH_VP_CREATE, vcpu->kvm)) {
arch/x86/kvm/vmx/tdx.c
2990
if (TDX_BUG_ON(err, TDH_VP_ADDCX, vcpu->kvm)) {
arch/x86/kvm/vmx/tdx.c
30
struct kvm *_kvm = (__kvm); \
arch/x86/kvm/vmx/tdx.c
3012
scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
arch/x86/kvm/vmx/tdx.c
3014
if (TDX_BUG_ON(err, TDH_VP_INIT, vcpu->kvm))
arch/x86/kvm/vmx/tdx.c
303
#define tdh_do_no_vcpus(tdh_func, kvm, args...) \
arch/x86/kvm/vmx/tdx.c
305
struct kvm_tdx *__kvm_tdx = to_kvm_tdx(kvm); \
arch/x86/kvm/vmx/tdx.c
308
lockdep_assert_held_write(&kvm->mmu_lock); \
arch/x86/kvm/vmx/tdx.c
313
kvm_make_all_cpus_request(kvm, KVM_REQ_OUTSIDE_GUEST_MODE); \
arch/x86/kvm/vmx/tdx.c
3183
static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
arch/x86/kvm/vmx/tdx.c
3187
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
3192
if (KVM_BUG_ON(kvm_tdx->page_add_src, kvm))
arch/x86/kvm/vmx/tdx.c
3214
if (TDX_BUG_ON_2(err, TDH_MR_EXTEND, entry, level_state, kvm))
arch/x86/kvm/vmx/tdx.c
3224
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/vmx/tdx.c
3225
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
3247
!vt_is_tdx_private_gpa(kvm, region.gpa) ||
arch/x86/kvm/vmx/tdx.c
3248
!vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1))
arch/x86/kvm/vmx/tdx.c
3262
gmem_ret = kvm_gmem_populate(kvm, gpa_to_gfn(region.gpa),
arch/x86/kvm/vmx/tdx.c
3289
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/vmx/tdx.c
3290
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
3298
CLASS(tdx_vm_state_guard, guard)(kvm);
arch/x86/kvm/vmx/tdx.c
3326
struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
arch/x86/kvm/vmx/tdx.c
3349
int tdx_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private)
arch/x86/kvm/vmx/tdx.c
425
TDX_BUG_ON(arg.err, TDH_VP_FLUSH, vcpu->kvm);
arch/x86/kvm/vmx/tdx.c
477
void tdx_mmu_release_hkid(struct kvm *kvm)
arch/x86/kvm/vmx/tdx.c
480
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
494
kvm_for_each_vcpu(j, vcpu, kvm)
arch/x86/kvm/vmx/tdx.c
513
if (TDX_BUG_ON(err, TDH_MNG_VPFLUSHDONE, kvm)) {
arch/x86/kvm/vmx/tdx.c
536
if (TDX_BUG_ON(err, TDH_MNG_KEY_FREEID, kvm)) {
arch/x86/kvm/vmx/tdx.c
550
static void tdx_reclaim_td_control_pages(struct kvm *kvm)
arch/x86/kvm/vmx/tdx.c
552
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
587
if (TDX_BUG_ON(err, TDH_PHYMEM_PAGE_WBINVD, kvm))
arch/x86/kvm/vmx/tdx.c
596
void tdx_vm_destroy(struct kvm *kvm)
arch/x86/kvm/vmx/tdx.c
598
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
600
tdx_reclaim_td_control_pages(kvm);
arch/x86/kvm/vmx/tdx.c
612
if (TDX_BUG_ON(err, TDH_MNG_KEY_CONFIG, &kvm_tdx->kvm))
arch/x86/kvm/vmx/tdx.c
618
int tdx_vm_init(struct kvm *kvm)
arch/x86/kvm/vmx/tdx.c
620
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
arch/x86/kvm/vmx/tdx.c
622
kvm->arch.has_protected_state = true;
arch/x86/kvm/vmx/tdx.c
627
kvm->arch.has_protected_eoi = true;
arch/x86/kvm/vmx/tdx.c
628
kvm->arch.has_private_mem = true;
arch/x86/kvm/vmx/tdx.c
629
kvm->arch.disabled_quirks |= KVM_X86_QUIRK_IGNORE_GUEST_PAT;
arch/x86/kvm/vmx/tdx.c
635
kvm->arch.has_protected_pmu = true;
arch/x86/kvm/vmx/tdx.c
648
kvm_mmu_set_mmio_spte_value(kvm, 0);
arch/x86/kvm/vmx/tdx.c
66
KVM_BUG_ON(1, tdx->vcpu.kvm);
arch/x86/kvm/vmx/tdx.c
661
kvm->max_vcpus = min_t(int, kvm->max_vcpus, num_present_cpus());
arch/x86/kvm/vmx/tdx.c
670
struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
arch/x86/kvm/vmx/tdx.c
681
if (!irqchip_split(vcpu->kvm))
arch/x86/kvm/vmx/tdx.c
702
!(to_kvm_tdx(vcpu->kvm)->attributes & TDX_TD_ATTR_DEBUG);
arch/x86/kvm/vmx/tdx.c
720
if (vcpu->cpu == cpu || !is_hkid_assigned(to_kvm_tdx(vcpu->kvm)))
arch/x86/kvm/vmx/tdx.c
725
KVM_BUG_ON(cpu != raw_smp_processor_id(), vcpu->kvm);
arch/x86/kvm/vmx/tdx.c
73
KVM_BUG_ON(1, tdx->vcpu.kvm);
arch/x86/kvm/vmx/tdx.c
79
static __always_inline struct kvm_tdx *to_kvm_tdx(struct kvm *kvm)
arch/x86/kvm/vmx/tdx.c
81
return container_of(kvm, struct kvm_tdx, kvm);
arch/x86/kvm/vmx/tdx.c
865
struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
arch/x86/kvm/vmx/tdx.c
870
KVM_BUG_ON(tdx->state == VCPU_TD_STATE_INITIALIZED, vcpu->kvm);
arch/x86/kvm/vmx/tdx.c
903
to_kvm_tdx(vcpu->kvm)->state != TD_STATE_RUNNABLE))
arch/x86/kvm/vmx/tdx.h
191
struct kvm kvm;
arch/x86/kvm/vmx/tdx.h
23
struct kvm kvm;
arch/x86/kvm/vmx/vmx.c
1068
unsigned long vmcs_count_field, struct kvm *kvm)
arch/x86/kvm/vmx/vmx.c
1074
if (KVM_BUG_ON(m->nr == MAX_NR_LOADSTORE_MSRS, kvm))
arch/x86/kvm/vmx/vmx.c
1088
struct kvm *kvm = vmx->vcpu.kvm;
arch/x86/kvm/vmx/vmx.c
1122
vmx_add_auto_msr(&m->guest, msr, guest_val, VM_ENTRY_MSR_LOAD_COUNT, kvm);
arch/x86/kvm/vmx/vmx.c
1123
vmx_add_auto_msr(&m->host, msr, host_val, VM_EXIT_MSR_LOAD_COUNT, kvm);
arch/x86/kvm/vmx/vmx.c
1181
vmx->vcpu.kvm);
arch/x86/kvm/vmx/vmx.c
1559
if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm))
arch/x86/kvm/vmx/vmx.c
1904
if (kvm_hlt_in_guest(vcpu->kvm) &&
arch/x86/kvm/vmx/vmx.c
2660
KVM_BUG_ON(1, vcpu->kvm);
arch/x86/kvm/vmx/vmx.c
3192
struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm);
arch/x86/kvm/vmx/vmx.c
3251
if (KVM_BUG_ON(efer & EFER_LMA, vcpu->kvm))
arch/x86/kvm/vmx/vmx.c
3523
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/vmx/vmx.c
3535
guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr;
arch/x86/kvm/vmx/vmx.c
3988
static int init_rmode_tss(struct kvm *kvm, void __user *ua)
arch/x86/kvm/vmx/vmx.c
4010
static int init_rmode_identity_map(struct kvm *kvm)
arch/x86/kvm/vmx/vmx.c
4012
struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm);
arch/x86/kvm/vmx/vmx.c
4018
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/vmx/vmx.c
4026
uaddr = __x86_set_memory_region(kvm,
arch/x86/kvm/vmx/vmx.c
4047
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/vmx/vmx.c
4281
if (kvm_cstate_in_guest(vcpu->kvm)) {
arch/x86/kvm/vmx/vmx.c
4287
if (kvm_aperfmperf_in_guest(vcpu->kvm)) {
arch/x86/kvm/vmx/vmx.c
4658
if (kvm_mwait_in_guest(vmx->vcpu.kvm))
arch/x86/kvm/vmx/vmx.c
4661
if (kvm_hlt_in_guest(vmx->vcpu.kvm))
arch/x86/kvm/vmx/vmx.c
4705
kvm_check_has_quirk(vmx->vcpu.kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) {
arch/x86/kvm/vmx/vmx.c
4763
if (kvm_pause_in_guest(vmx->vcpu.kvm))
arch/x86/kvm/vmx/vmx.c
4795
if (!enable_pml || !atomic_read(&vcpu->kvm->nr_memslots_dirty_logging))
arch/x86/kvm/vmx/vmx.c
4826
if (!vcpu->kvm->arch.bus_lock_detection_enabled)
arch/x86/kvm/vmx/vmx.c
4829
if (!kvm_notify_vmexit_enabled(vcpu->kvm))
arch/x86/kvm/vmx/vmx.c
4835
static inline int vmx_get_pid_table_order(struct kvm *kvm)
arch/x86/kvm/vmx/vmx.c
4837
return get_order(kvm->arch.max_vcpu_ids * sizeof(*to_kvm_vmx(kvm)->pid_table));
arch/x86/kvm/vmx/vmx.c
4840
static int vmx_alloc_ipiv_pid_table(struct kvm *kvm)
arch/x86/kvm/vmx/vmx.c
4843
struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm);
arch/x86/kvm/vmx/vmx.c
4845
if (!irqchip_in_kernel(kvm) || !enable_ipiv)
arch/x86/kvm/vmx/vmx.c
4852
vmx_get_pid_table_order(kvm));
arch/x86/kvm/vmx/vmx.c
4860
int vmx_vcpu_precreate(struct kvm *kvm)
arch/x86/kvm/vmx/vmx.c
4862
return vmx_alloc_ipiv_pid_table(kvm);
arch/x86/kvm/vmx/vmx.c
4869
struct kvm *kvm = vmx->vcpu.kvm;
arch/x86/kvm/vmx/vmx.c
4870
struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm);
arch/x86/kvm/vmx/vmx.c
4909
vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1);
arch/x86/kvm/vmx/vmx.c
4912
if (!kvm_pause_in_guest(kvm)) {
arch/x86/kvm/vmx/vmx.c
4918
if (kvm_notify_vmexit_enabled(kvm))
arch/x86/kvm/vmx/vmx.c
4919
vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window);
arch/x86/kvm/vmx/vmx.c
4998
kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS))
arch/x86/kvm/vmx/vmx.c
5011
if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS))
arch/x86/kvm/vmx/vmx.c
5254
int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
arch/x86/kvm/vmx/vmx.c
5261
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/vmx/vmx.c
5262
ret = __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr,
arch/x86/kvm/vmx/vmx.c
5264
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/vmx/vmx.c
5269
to_kvm_vmx(kvm)->tss_addr = addr;
arch/x86/kvm/vmx/vmx.c
5271
return init_rmode_tss(kvm, ret);
arch/x86/kvm/vmx/vmx.c
5274
int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
arch/x86/kvm/vmx/vmx.c
5276
to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr;
arch/x86/kvm/vmx/vmx.c
5703
KVM_BUG(1, vcpu->kvm, "Guest always owns CR0.TS");
arch/x86/kvm/vmx/vmx.c
6006
if (KVM_BUG_ON(!enable_vnmi, vcpu->kvm))
arch/x86/kvm/vmx/vmx.c
6112
if (!kvm_pause_in_guest(vcpu->kvm))
arch/x86/kvm/vmx/vmx.c
6284
if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER ||
arch/x86/kvm/vmx/vmx.c
645
evmcs->hv_vm_id = (unsigned long)vcpu->kvm;
arch/x86/kvm/vmx/vmx.c
6723
if (KVM_BUG_ON(vcpu->arch.nested_run_pending, vcpu->kvm))
arch/x86/kvm/vmx/vmx.c
6949
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/vmx/vmx.c
6950
struct kvm_memslots *slots = kvm_memslots(kvm);
arch/x86/kvm/vmx/vmx.c
6980
mmu_seq = kvm->mmu_invalidate_seq;
arch/x86/kvm/vmx/vmx.c
6992
read_lock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/vmx/vmx.c
6993
if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn))
arch/x86/kvm/vmx/vmx.c
7011
read_unlock(&vcpu->kvm->mmu_lock);
arch/x86/kvm/vmx/vmx.c
7061
if (KVM_BUG_ON(!enable_apicv, vcpu->kvm))
arch/x86/kvm/vmx/vmx.c
7173
if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm,
arch/x86/kvm/vmx/vmx.c
7208
bool vmx_has_emulated_msr(struct kvm *kvm, u32 index)
arch/x86/kvm/vmx/vmx.c
7730
err = kvm_alloc_apic_access_page(vcpu->kvm);
arch/x86/kvm/vmx/vmx.c
7736
err = init_rmode_identity_map(vcpu->kvm);
arch/x86/kvm/vmx/vmx.c
7756
WRITE_ONCE(to_kvm_vmx(vcpu->kvm)->pid_table[vcpu->vcpu_id],
arch/x86/kvm/vmx/vmx.c
7773
int vmx_vm_init(struct kvm *kvm)
arch/x86/kvm/vmx/vmx.c
7776
kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE);
arch/x86/kvm/vmx/vmx.c
7804
kvm->arch.cpu_dirty_log_size = PML_LOG_NR_ENTRIES;
arch/x86/kvm/vmx/vmx.c
7808
static inline bool vmx_ignore_guest_pat(struct kvm *kvm)
arch/x86/kvm/vmx/vmx.c
7815
return !kvm_arch_has_noncoherent_dma(kvm) &&
arch/x86/kvm/vmx/vmx.c
7816
kvm_check_has_quirk(kvm, KVM_X86_QUIRK_IGNORE_GUEST_PAT);
arch/x86/kvm/vmx/vmx.c
7829
if (vmx_ignore_guest_pat(vcpu->kvm))
arch/x86/kvm/vmx/vmx.c
8374
if (atomic_read(&vcpu->kvm->nr_memslots_dirty_logging))
arch/x86/kvm/vmx/vmx.c
8474
void vmx_vm_destroy(struct kvm *kvm)
arch/x86/kvm/vmx/vmx.c
8476
struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm);
arch/x86/kvm/vmx/vmx.c
8478
free_pages((unsigned long)kvm_vmx->pid_table, vmx_get_pid_table_order(kvm));
arch/x86/kvm/vmx/vmx.h
298
struct kvm kvm;
arch/x86/kvm/vmx/vmx.h
657
static __always_inline struct kvm_vmx *to_kvm_vmx(struct kvm *kvm)
arch/x86/kvm/vmx/vmx.h
659
return container_of(kvm, struct kvm_vmx, kvm);
arch/x86/kvm/vmx/vmx_onhyperv.h
112
if (KVM_BUG_ON(!vp_ap, kvm_get_running_vcpu()->kvm))
arch/x86/kvm/vmx/x86_ops.h
104
int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
arch/x86/kvm/vmx/x86_ops.h
105
int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr);
arch/x86/kvm/vmx/x86_ops.h
108
static inline bool vmx_tdp_has_smep(struct kvm *kvm)
arch/x86/kvm/vmx/x86_ops.h
131
int tdx_vm_init(struct kvm *kvm);
arch/x86/kvm/vmx/x86_ops.h
132
void tdx_mmu_release_hkid(struct kvm *kvm);
arch/x86/kvm/vmx/x86_ops.h
133
void tdx_vm_destroy(struct kvm *kvm);
arch/x86/kvm/vmx/x86_ops.h
134
int tdx_vm_ioctl(struct kvm *kvm, void __user *argp);
arch/x86/kvm/vmx/x86_ops.h
162
int tdx_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private);
arch/x86/kvm/vmx/x86_ops.h
20
int vmx_vm_init(struct kvm *kvm);
arch/x86/kvm/vmx/x86_ops.h
21
void vmx_vm_destroy(struct kvm *kvm);
arch/x86/kvm/vmx/x86_ops.h
22
int vmx_vcpu_precreate(struct kvm *kvm);
arch/x86/kvm/vmx/x86_ops.h
55
bool vmx_has_emulated_msr(struct kvm *kvm, u32 index);
arch/x86/kvm/x86.c
10282
if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
arch/x86/kvm/x86.c
10295
static void kvm_pv_kick_cpu_op(struct kvm *kvm, int apicid)
arch/x86/kvm/x86.c
10308
kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq);
arch/x86/kvm/x86.c
10311
bool kvm_apicv_activated(struct kvm *kvm)
arch/x86/kvm/x86.c
10313
return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0);
arch/x86/kvm/x86.c
10319
ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons);
arch/x86/kvm/x86.c
10342
static void kvm_apicv_init(struct kvm *kvm)
arch/x86/kvm/x86.c
10347
set_or_clear_apicv_inhibit(&kvm->arch.apicv_inhibit_reasons, reason, true);
arch/x86/kvm/x86.c
10349
init_rwsem(&kvm->arch.apicv_update_lock);
arch/x86/kvm/x86.c
10363
map = rcu_dereference(vcpu->kvm->arch.apic_map);
arch/x86/kvm/x86.c
10438
kvm_pv_kick_cpu_op(vcpu->kvm, a1);
arch/x86/kvm/x86.c
10451
ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
arch/x86/kvm/x86.c
10464
if (!user_exit_on_hypercall(vcpu->kvm, KVM_HC_MAP_GPA_RANGE))
arch/x86/kvm/x86.c
10506
if (kvm_xen_hypercall_enabled(vcpu->kvm))
arch/x86/kvm/x86.c
10527
if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) {
arch/x86/kvm/x86.c
10543
likely(!pic_in_kernel(vcpu->kvm));
arch/x86/kvm/x86.c
10556
pic_in_kernel(vcpu->kvm) ||
arch/x86/kvm/x86.c
1065
kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);
arch/x86/kvm/x86.c
10883
void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
arch/x86/kvm/x86.c
10886
kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC, vcpu_bitmap);
arch/x86/kvm/x86.c
10889
void kvm_make_scan_ioapic_request(struct kvm *kvm)
arch/x86/kvm/x86.c
10891
kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
arch/x86/kvm/x86.c
10902
down_read(&vcpu->kvm->arch.apicv_update_lock);
arch/x86/kvm/x86.c
10927
up_read(&vcpu->kvm->arch.apicv_update_lock);
arch/x86/kvm/x86.c
10954
void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
arch/x86/kvm/x86.c
10959
lockdep_assert_held_write(&kvm->arch.apicv_update_lock);
arch/x86/kvm/x86.c
10964
old = new = kvm->arch.apicv_inhibit_reasons;
arch/x86/kvm/x86.c
10970
atomic_read(&kvm->arch.apicv_nr_irq_window_req));
arch/x86/kvm/x86.c
10985
kvm_make_all_cpus_request(kvm, KVM_REQ_APICV_UPDATE);
arch/x86/kvm/x86.c
10986
kvm->arch.apicv_inhibit_reasons = new;
arch/x86/kvm/x86.c
10989
int idx = srcu_read_lock(&kvm->srcu);
arch/x86/kvm/x86.c
10991
kvm_zap_gfn_range(kvm, gfn, gfn+1);
arch/x86/kvm/x86.c
10992
srcu_read_unlock(&kvm->srcu, idx);
arch/x86/kvm/x86.c
10995
kvm->arch.apicv_inhibit_reasons = new;
arch/x86/kvm/x86.c
10999
void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
arch/x86/kvm/x86.c
11005
down_write(&kvm->arch.apicv_update_lock);
arch/x86/kvm/x86.c
11006
__kvm_set_or_clear_apicv_inhibit(kvm, reason, set);
arch/x86/kvm/x86.c
11007
up_write(&kvm->arch.apicv_update_lock);
arch/x86/kvm/x86.c
11011
void kvm_inc_or_dec_irq_window_inhibit(struct kvm *kvm, bool inc)
arch/x86/kvm/x86.c
11030
if (READ_ONCE(kvm->arch.apicv_inhibit_reasons) & ~BIT(APICV_INHIBIT_REASON_IRQWIN)) {
arch/x86/kvm/x86.c
11031
guard(rwsem_read)(&kvm->arch.apicv_update_lock);
arch/x86/kvm/x86.c
11032
if (READ_ONCE(kvm->arch.apicv_inhibit_reasons) & ~BIT(APICV_INHIBIT_REASON_IRQWIN)) {
arch/x86/kvm/x86.c
11033
atomic_add(add, &kvm->arch.apicv_nr_irq_window_req);
arch/x86/kvm/x86.c
11044
guard(rwsem_write)(&kvm->arch.apicv_update_lock);
arch/x86/kvm/x86.c
11045
if (atomic_add_return(add, &kvm->arch.apicv_nr_irq_window_req) == inc)
arch/x86/kvm/x86.c
11046
__kvm_set_or_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_IRQWIN, inc);
arch/x86/kvm/x86.c
11060
if (irqchip_split(vcpu->kvm))
arch/x86/kvm/x86.c
11063
else if (ioapic_in_kernel(vcpu->kvm))
arch/x86/kvm/x86.c
11093
void kvm_arch_guest_memory_reclaimed(struct kvm *kvm)
arch/x86/kvm/x86.c
11095
kvm_x86_call(guest_memory_reclaimed)(kvm);
arch/x86/kvm/x86.c
11145
kvm_update_masterclock(vcpu->kvm);
arch/x86/kvm/x86.c
11837
if (KVM_BUG_ON(!vcpu->arch.pio.count, vcpu->kvm))
arch/x86/kvm/x86.c
11867
if (KVM_BUG_ON(!vcpu->mmio_needed, vcpu->kvm))
arch/x86/kvm/x86.c
11908
if (KVM_BUG_ON(vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm))
arch/x86/kvm/x86.c
11919
if (KVM_BUG_ON(!vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm))
arch/x86/kvm/x86.c
11963
r = kvm_mmu_post_init_vm(vcpu->kvm);
arch/x86/kvm/x86.c
12006
sync_valid_fields = kvm_sync_valid_fields(vcpu->kvm);
arch/x86/kvm/x86.c
12114
if (vcpu->kvm->arch.has_protected_state &&
arch/x86/kvm/x86.c
12159
if (vcpu->kvm->arch.has_protected_state &&
arch/x86/kvm/x86.c
12239
if (vcpu->kvm->arch.has_protected_state &&
arch/x86/kvm/x86.c
12431
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/x86/kvm/x86.c
12436
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/x86/kvm/x86.c
12525
if (vcpu->kvm->arch.has_protected_state &&
arch/x86/kvm/x86.c
12535
static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm)
arch/x86/kvm/x86.c
12544
down_write(&kvm->arch.apicv_update_lock);
arch/x86/kvm/x86.c
12546
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/x86.c
12552
__kvm_set_or_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_BLOCKIRQ, set);
arch/x86/kvm/x86.c
12553
up_write(&kvm->arch.apicv_update_lock);
arch/x86/kvm/x86.c
12608
kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm);
arch/x86/kvm/x86.c
12629
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/x86/kvm/x86.c
12631
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/x86/kvm/x86.c
12646
return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
arch/x86/kvm/x86.c
12669
return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
arch/x86/kvm/x86.c
12734
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
arch/x86/kvm/x86.c
12738
if (kvm_check_tsc_unstable() && kvm->created_vcpus)
arch/x86/kvm/x86.c
12742
if (!kvm->arch.max_vcpu_ids)
arch/x86/kvm/x86.c
12743
kvm->arch.max_vcpu_ids = KVM_MAX_VCPU_IDS;
arch/x86/kvm/x86.c
12745
if (id >= kvm->arch.max_vcpu_ids)
arch/x86/kvm/x86.c
12753
r = kvm_x86_call(vcpu_precreate)(kvm);
arch/x86/kvm/x86.c
12757
if (enable_mediated_pmu && kvm->arch.enable_pmu &&
arch/x86/kvm/x86.c
12758
!kvm->arch.created_mediated_pmu) {
arch/x86/kvm/x86.c
12759
if (irqchip_in_kernel(kvm)) {
arch/x86/kvm/x86.c
12765
kvm->arch.created_mediated_pmu = true;
arch/x86/kvm/x86.c
12767
kvm->arch.enable_pmu = false;
arch/x86/kvm/x86.c
12782
kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm);
arch/x86/kvm/x86.c
12784
if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
arch/x86/kvm/x86.c
12826
if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) {
arch/x86/kvm/x86.c
12847
kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz);
arch/x86/kvm/x86.c
12908
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/x86/kvm/x86.c
12910
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/x86/kvm/x86.c
13123
struct kvm *kvm;
arch/x86/kvm/x86.c
13143
list_for_each_entry(kvm, &vm_list, vm_list) {
arch/x86/kvm/x86.c
13144
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/x86.c
13195
list_for_each_entry(kvm, &vm_list, vm_list) {
arch/x86/kvm/x86.c
13196
kvm->arch.backwards_tsc_observed = true;
arch/x86/kvm/x86.c
13197
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/x86.c
13209
kvm->arch.last_tsc_nsec = 0;
arch/x86/kvm/x86.c
13210
kvm->arch.last_tsc_write = 0;
arch/x86/kvm/x86.c
13258
return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
arch/x86/kvm/x86.c
13267
void kvm_arch_free_vm(struct kvm *kvm)
arch/x86/kvm/x86.c
13270
kfree(kvm->arch.hv_pa_pg);
arch/x86/kvm/x86.c
13272
__kvm_arch_free_vm(kvm);
arch/x86/kvm/x86.c
13276
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
arch/x86/kvm/x86.c
13284
kvm->arch.vm_type = type;
arch/x86/kvm/x86.c
13285
kvm->arch.has_private_mem =
arch/x86/kvm/x86.c
13288
kvm->arch.pre_fault_allowed =
arch/x86/kvm/x86.c
13290
kvm->arch.disabled_quirks = kvm_caps.inapplicable_quirks & kvm_caps.supported_quirks;
arch/x86/kvm/x86.c
13292
ret = kvm_page_track_init(kvm);
arch/x86/kvm/x86.c
13296
ret = kvm_mmu_init_vm(kvm);
arch/x86/kvm/x86.c
13300
ret = kvm_x86_call(vm_init)(kvm);
arch/x86/kvm/x86.c
13304
atomic_set(&kvm->arch.noncoherent_dma_count, 0);
arch/x86/kvm/x86.c
13306
raw_spin_lock_init(&kvm->arch.tsc_write_lock);
arch/x86/kvm/x86.c
13307
mutex_init(&kvm->arch.apic_map_lock);
arch/x86/kvm/x86.c
13308
seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock);
arch/x86/kvm/x86.c
13309
ratelimit_state_init(&kvm->arch.kvmclock_update_rs, HZ, 10);
arch/x86/kvm/x86.c
13310
ratelimit_set_flags(&kvm->arch.kvmclock_update_rs, RATELIMIT_MSG_ON_RELEASE);
arch/x86/kvm/x86.c
13311
kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
arch/x86/kvm/x86.c
13313
raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
arch/x86/kvm/x86.c
13314
pvclock_update_vm_gtod_copy(kvm);
arch/x86/kvm/x86.c
13315
raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
arch/x86/kvm/x86.c
13317
kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz;
arch/x86/kvm/x86.c
13318
kvm->arch.apic_bus_cycle_ns = APIC_BUS_CYCLE_NS_DEFAULT;
arch/x86/kvm/x86.c
13319
kvm->arch.guest_can_read_msr_platform_info = true;
arch/x86/kvm/x86.c
13320
kvm->arch.enable_pmu = enable_pmu && !kvm->arch.has_protected_pmu;
arch/x86/kvm/x86.c
13323
spin_lock_init(&kvm->arch.hv_root_tdp_lock);
arch/x86/kvm/x86.c
13324
kvm->arch.hv_root_tdp = INVALID_PAGE;
arch/x86/kvm/x86.c
13327
kvm_apicv_init(kvm);
arch/x86/kvm/x86.c
13328
kvm_hv_init_vm(kvm);
arch/x86/kvm/x86.c
13329
kvm_xen_init_vm(kvm);
arch/x86/kvm/x86.c
13338
once_init(&kvm->arch.nx_once);
arch/x86/kvm/x86.c
13342
kvm_mmu_uninit_vm(kvm);
arch/x86/kvm/x86.c
13344
kvm_page_track_cleanup(kvm);
arch/x86/kvm/x86.c
13371
void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
arch/x86/kvm/x86.c
13376
struct kvm_memslots *slots = kvm_memslots(kvm);
arch/x86/kvm/x86.c
13379
lockdep_assert_held(&kvm->slots_lock);
arch/x86/kvm/x86.c
13405
for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
arch/x86/kvm/x86.c
13413
r = kvm_set_internal_memslot(kvm, &m);
arch/x86/kvm/x86.c
13425
void kvm_arch_pre_destroy_vm(struct kvm *kvm)
arch/x86/kvm/x86.c
13438
kvm_free_pit(kvm);
arch/x86/kvm/x86.c
13439
if (kvm->arch.vioapic)
arch/x86/kvm/x86.c
13440
cancel_delayed_work_sync(&kvm->arch.vioapic->eoi_inject);
arch/x86/kvm/x86.c
13443
kvm_mmu_pre_destroy_vm(kvm);
arch/x86/kvm/x86.c
13444
kvm_x86_call(vm_pre_destroy)(kvm);
arch/x86/kvm/x86.c
13447
void kvm_arch_destroy_vm(struct kvm *kvm)
arch/x86/kvm/x86.c
13449
if (current->mm == kvm->mm) {
arch/x86/kvm/x86.c
13455
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/x86.c
13456
__x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
arch/x86/kvm/x86.c
13458
__x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
arch/x86/kvm/x86.c
13460
__x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
arch/x86/kvm/x86.c
13461
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/x86.c
13463
if (kvm->arch.created_mediated_pmu)
arch/x86/kvm/x86.c
13465
kvm_destroy_vcpus(kvm);
arch/x86/kvm/x86.c
13466
kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1));
arch/x86/kvm/x86.c
13468
kvm_pic_destroy(kvm);
arch/x86/kvm/x86.c
13469
kvm_ioapic_destroy(kvm);
arch/x86/kvm/x86.c
13471
kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
arch/x86/kvm/x86.c
13472
kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
arch/x86/kvm/x86.c
13473
kvm_mmu_uninit_vm(kvm);
arch/x86/kvm/x86.c
13474
kvm_page_track_cleanup(kvm);
arch/x86/kvm/x86.c
13475
kvm_xen_destroy_vm(kvm);
arch/x86/kvm/x86.c
13476
kvm_hv_destroy_vm(kvm);
arch/x86/kvm/x86.c
13477
kvm_x86_call(vm_destroy)(kvm);
arch/x86/kvm/x86.c
13490
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
arch/x86/kvm/x86.c
13526
static int kvm_alloc_memslot_metadata(struct kvm *kvm,
arch/x86/kvm/x86.c
13539
if (kvm_memslots_have_rmaps(kvm)) {
arch/x86/kvm/x86.c
13577
kvm_mmu_init_memslot_memory_attributes(kvm, slot);
arch/x86/kvm/x86.c
13580
if (kvm_page_track_create_memslot(kvm, slot, npages))
arch/x86/kvm/x86.c
13595
void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
arch/x86/kvm/x86.c
13604
kvm_mmu_invalidate_mmio_sptes(kvm, gen);
arch/x86/kvm/x86.c
13607
kvm_for_each_vcpu(i, vcpu, kvm)
arch/x86/kvm/x86.c
13611
int kvm_arch_prepare_memory_region(struct kvm *kvm,
arch/x86/kvm/x86.c
13620
if (change == KVM_MR_MOVE && kvm_page_track_has_external_user(kvm))
arch/x86/kvm/x86.c
13627
if (kvm_is_gfn_alias(kvm, new->base_gfn + new->npages - 1))
arch/x86/kvm/x86.c
13630
return kvm_alloc_memslot_metadata(kvm, new);
arch/x86/kvm/x86.c
13642
static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable)
arch/x86/kvm/x86.c
13646
if (!kvm->arch.cpu_dirty_log_size)
arch/x86/kvm/x86.c
13649
nr_slots = atomic_read(&kvm->nr_memslots_dirty_logging);
arch/x86/kvm/x86.c
13651
kvm_make_all_cpus_request(kvm, KVM_REQ_UPDATE_CPU_DIRTY_LOGGING);
arch/x86/kvm/x86.c
13654
static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
arch/x86/kvm/x86.c
13668
kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages);
arch/x86/kvm/x86.c
13706
kvm_mmu_recover_huge_pages(kvm, new);
arch/x86/kvm/x86.c
13712
if (kvm_dirty_log_manual_protect_and_init_set(kvm))
arch/x86/kvm/x86.c
13716
kvm_mmu_slot_try_split_huge_pages(kvm, new, PG_LEVEL_4K);
arch/x86/kvm/x86.c
13718
if (kvm->arch.cpu_dirty_log_size) {
arch/x86/kvm/x86.c
13719
kvm_mmu_slot_leaf_clear_dirty(kvm, new);
arch/x86/kvm/x86.c
13720
kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_2M);
arch/x86/kvm/x86.c
13722
kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K);
arch/x86/kvm/x86.c
13767
kvm_flush_remote_tlbs_memslot(kvm, new);
arch/x86/kvm/x86.c
13771
void kvm_arch_commit_memory_region(struct kvm *kvm,
arch/x86/kvm/x86.c
13777
kvm_page_track_delete_slot(kvm, old);
arch/x86/kvm/x86.c
13779
if (!kvm->arch.n_requested_mmu_pages &&
arch/x86/kvm/x86.c
13783
nr_mmu_pages = kvm->nr_memslot_pages / KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO;
arch/x86/kvm/x86.c
13785
kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
arch/x86/kvm/x86.c
13788
kvm_mmu_slot_apply_flags(kvm, old, new, change);
arch/x86/kvm/x86.c
13792
kvm_arch_free_memslot(kvm, old);
arch/x86/kvm/x86.c
13941
return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,
arch/x86/kvm/x86.c
13949
return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
arch/x86/kvm/x86.c
13958
if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
arch/x86/kvm/x86.c
13998
if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
arch/x86/kvm/x86.c
14084
static void kvm_noncoherent_dma_assignment_start_or_stop(struct kvm *kvm)
arch/x86/kvm/x86.c
14095
if (kvm_check_has_quirk(kvm, KVM_X86_QUIRK_IGNORE_GUEST_PAT))
arch/x86/kvm/x86.c
14096
kvm_zap_gfn_range(kvm, gpa_to_gfn(0), gpa_to_gfn(~0ULL));
arch/x86/kvm/x86.c
14099
void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
arch/x86/kvm/x86.c
14101
if (atomic_inc_return(&kvm->arch.noncoherent_dma_count) == 1)
arch/x86/kvm/x86.c
14102
kvm_noncoherent_dma_assignment_start_or_stop(kvm);
arch/x86/kvm/x86.c
14105
void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
arch/x86/kvm/x86.c
14107
if (!atomic_dec_return(&kvm->arch.noncoherent_dma_count))
arch/x86/kvm/x86.c
14108
kvm_noncoherent_dma_assignment_start_or_stop(kvm);
arch/x86/kvm/x86.c
14111
bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
arch/x86/kvm/x86.c
14113
return atomic_read(&kvm->arch.noncoherent_dma_count);
arch/x86/kvm/x86.c
14128
bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm)
arch/x86/kvm/x86.c
14130
return !kvm_arch_has_private_mem(kvm);
arch/x86/kvm/x86.c
14134
int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order)
arch/x86/kvm/x86.c
14136
return kvm_x86_call(gmem_prepare)(kvm, pfn, gfn, max_order);
arch/x86/kvm/x86.c
1415
kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);
arch/x86/kvm/x86.c
14208
if (KVM_BUG_ON(!e, vcpu->kvm))
arch/x86/kvm/x86.c
14313
if (KVM_BUG_ON(!vcpu->mmio_needed, vcpu->kvm))
arch/x86/kvm/x86.c
1781
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/x86.c
1790
idx = srcu_read_lock(&kvm->srcu);
arch/x86/kvm/x86.c
1792
msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu);
arch/x86/kvm/x86.c
1814
srcu_read_unlock(&kvm->srcu, idx);
arch/x86/kvm/x86.c
2093
if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))
arch/x86/kvm/x86.c
2220
if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS))
arch/x86/kvm/x86.c
2223
if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT))
arch/x86/kvm/x86.c
2387
static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs)
arch/x86/kvm/x86.c
2398
r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
arch/x86/kvm/x86.c
2407
if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
arch/x86/kvm/x86.c
2410
wall_nsec = kvm_get_wall_clock_epoch(kvm);
arch/x86/kvm/x86.c
2416
kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
arch/x86/kvm/x86.c
2420
kvm_write_guest(kvm, wall_clock + sec_hi_ofs,
arch/x86/kvm/x86.c
2425
kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
arch/x86/kvm/x86.c
2431
struct kvm_arch *ka = &vcpu->kvm->arch;
arch/x86/kvm/x86.c
2592
struct kvm_arch *ka = &vcpu->kvm->arch;
arch/x86/kvm/x86.c
2601
atomic_read(&vcpu->kvm->online_vcpus)) &&
arch/x86/kvm/x86.c
2615
atomic_read(&vcpu->kvm->online_vcpus),
arch/x86/kvm/x86.c
2750
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/x86.c
2752
lockdep_assert_held(&kvm->arch.tsc_write_lock);
arch/x86/kvm/x86.c
2758
vcpu->kvm->arch.user_set_tsc = true;
arch/x86/kvm/x86.c
2764
kvm->arch.last_tsc_nsec = ns;
arch/x86/kvm/x86.c
2765
kvm->arch.last_tsc_write = tsc;
arch/x86/kvm/x86.c
2766
kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
arch/x86/kvm/x86.c
2767
kvm->arch.last_tsc_offset = offset;
arch/x86/kvm/x86.c
2783
kvm->arch.cur_tsc_generation++;
arch/x86/kvm/x86.c
2784
kvm->arch.cur_tsc_nsec = ns;
arch/x86/kvm/x86.c
2785
kvm->arch.cur_tsc_write = tsc;
arch/x86/kvm/x86.c
2786
kvm->arch.cur_tsc_offset = offset;
arch/x86/kvm/x86.c
2787
kvm->arch.nr_vcpus_matched_tsc = 0;
arch/x86/kvm/x86.c
2788
} else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {
arch/x86/kvm/x86.c
2789
kvm->arch.nr_vcpus_matched_tsc++;
arch/x86/kvm/x86.c
2793
vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
arch/x86/kvm/x86.c
2794
vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
arch/x86/kvm/x86.c
2795
vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
arch/x86/kvm/x86.c
2803
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/x86.c
2809
raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
arch/x86/kvm/x86.c
2812
elapsed = ns - kvm->arch.last_tsc_nsec;
arch/x86/kvm/x86.c
2821
} else if (kvm->arch.user_set_tsc) {
arch/x86/kvm/x86.c
2822
u64 tsc_exp = kvm->arch.last_tsc_write +
arch/x86/kvm/x86.c
2853
vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
arch/x86/kvm/x86.c
2855
offset = kvm->arch.cur_tsc_offset;
arch/x86/kvm/x86.c
2865
raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
arch/x86/kvm/x86.c
3096
static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
arch/x86/kvm/x86.c
3099
struct kvm_arch *ka = &kvm->arch;
arch/x86/kvm/x86.c
3103
lockdep_assert_held(&kvm->arch.tsc_write_lock);
arch/x86/kvm/x86.c
3105
atomic_read(&kvm->online_vcpus));
arch/x86/kvm/x86.c
3128
static void kvm_make_mclock_inprogress_request(struct kvm *kvm)
arch/x86/kvm/x86.c
3130
kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
arch/x86/kvm/x86.c
3133
static void __kvm_start_pvclock_update(struct kvm *kvm)
arch/x86/kvm/x86.c
3135
raw_spin_lock_irq(&kvm->arch.tsc_write_lock);
arch/x86/kvm/x86.c
3136
write_seqcount_begin(&kvm->arch.pvclock_sc);
arch/x86/kvm/x86.c
3139
static void kvm_start_pvclock_update(struct kvm *kvm)
arch/x86/kvm/x86.c
3141
kvm_make_mclock_inprogress_request(kvm);
arch/x86/kvm/x86.c
3144
__kvm_start_pvclock_update(kvm);
arch/x86/kvm/x86.c
3147
static void kvm_end_pvclock_update(struct kvm *kvm)
arch/x86/kvm/x86.c
3149
struct kvm_arch *ka = &kvm->arch;
arch/x86/kvm/x86.c
3155
kvm_for_each_vcpu(i, vcpu, kvm)
arch/x86/kvm/x86.c
3159
kvm_for_each_vcpu(i, vcpu, kvm)
arch/x86/kvm/x86.c
3163
static void kvm_update_masterclock(struct kvm *kvm)
arch/x86/kvm/x86.c
3165
kvm_hv_request_tsc_page_update(kvm);
arch/x86/kvm/x86.c
3166
kvm_start_pvclock_update(kvm);
arch/x86/kvm/x86.c
3167
pvclock_update_vm_gtod_copy(kvm);
arch/x86/kvm/x86.c
3168
kvm_end_pvclock_update(kvm);
arch/x86/kvm/x86.c
3188
static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
arch/x86/kvm/x86.c
3190
struct kvm_arch *ka = &kvm->arch;
arch/x86/kvm/x86.c
3223
static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
arch/x86/kvm/x86.c
3225
struct kvm_arch *ka = &kvm->arch;
arch/x86/kvm/x86.c
3230
__get_kvmclock(kvm, data);
arch/x86/kvm/x86.c
3234
u64 get_kvmclock_ns(struct kvm *kvm)
arch/x86/kvm/x86.c
3238
get_kvmclock(kvm, &data);
arch/x86/kvm/x86.c
3296
struct kvm_arch *ka = &v->kvm->arch;
arch/x86/kvm/x86.c
3370
hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
arch/x86/kvm/x86.c
3393
kvm_hv_setup_tsc_page(v->kvm, &hv_clock);
arch/x86/kvm/x86.c
3435
uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm)
arch/x86/kvm/x86.c
3439
struct kvm_arch *ka = &kvm->arch;
arch/x86/kvm/x86.c
3489
return ktime_get_real_ns() - get_kvmclock_ns(kvm);
arch/x86/kvm/x86.c
3505
struct kvm *kvm = v->kvm;
arch/x86/kvm/x86.c
3507
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/x86.c
3627
kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
arch/x86/kvm/x86.c
3729
if (kvm_xen_msr_enabled(vcpu->kvm)) {
arch/x86/kvm/x86.c
3737
if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))
arch/x86/kvm/x86.c
3740
slots = kvm_memslots(vcpu->kvm);
arch/x86/kvm/x86.c
3748
if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||
arch/x86/kvm/x86.c
3815
mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
arch/x86/kvm/x86.c
3857
KVM_BUG_ON(!is_xstate_managed_msr(vcpu, msr_info->index), vcpu->kvm);
arch/x86/kvm/x86.c
3858
KVM_BUG_ON(!vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm);
arch/x86/kvm/x86.c
3888
if (kvm_xen_is_hypercall_page_msr(vcpu->kvm, msr) &&
arch/x86/kvm/x86.c
4047
if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
arch/x86/kvm/x86.c
4095
vcpu->kvm->arch.wall_clock = data;
arch/x86/kvm/x86.c
4096
kvm_write_wall_clock(vcpu->kvm, data, 0);
arch/x86/kvm/x86.c
4102
vcpu->kvm->arch.wall_clock = data;
arch/x86/kvm/x86.c
4103
kvm_write_wall_clock(vcpu->kvm, data, 0);
arch/x86/kvm/x86.c
4475
msr_info->data = vcpu->kvm->arch.wall_clock;
arch/x86/kvm/x86.c
4481
msr_info->data = vcpu->kvm->arch.wall_clock;
arch/x86/kvm/x86.c
4598
!vcpu->kvm->arch.guest_can_read_msr_platform_info)
arch/x86/kvm/x86.c
4763
static inline u64 kvm_sync_valid_fields(struct kvm *kvm)
arch/x86/kvm/x86.c
4765
return kvm && kvm->arch.has_protected_state ? 0 : KVM_SYNC_X86_VALID_FIELDS;
arch/x86/kvm/x86.c
4768
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
arch/x86/kvm/x86.c
4857
r = kvm ? kvm->arch.apic_bus_cycle_ns : APIC_BUS_CYCLE_NS_DEFAULT;
arch/x86/kvm/x86.c
4879
r = kvm_sync_valid_fields(kvm);
arch/x86/kvm/x86.c
4899
r = kvm_x86_call(has_emulated_msr)(kvm, MSR_IA32_SMBASE);
arch/x86/kvm/x86.c
4906
if (kvm)
arch/x86/kvm/x86.c
4907
r = kvm->max_vcpus;
arch/x86/kvm/x86.c
4927
if (kvm && !irqchip_split(kvm))
arch/x86/kvm/x86.c
4974
r = kvm ? kvm_arch_has_readonly_mem(kvm) : 1;
arch/x86/kvm/x86.c
5138
return kvm_arch_has_noncoherent_dma(vcpu->kvm);
arch/x86/kvm/x86.c
5208
if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1) {
arch/x86/kvm/x86.c
5209
if (__ratelimit(&vcpu->kvm->arch.kvmclock_update_rs))
arch/x86/kvm/x86.c
5251
if (unlikely(current->mm != vcpu->kvm->mm))
arch/x86/kvm/x86.c
5254
slots = kvm_memslots(vcpu->kvm);
arch/x86/kvm/x86.c
5267
mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
arch/x86/kvm/x86.c
5287
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/x86/kvm/x86.c
5288
if (kvm_xen_msr_enabled(vcpu->kvm))
arch/x86/kvm/x86.c
5292
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/x86/kvm/x86.c
5362
if (!irqchip_in_kernel(vcpu->kvm)) {
arch/x86/kvm/x86.c
5372
if (pic_in_kernel(vcpu->kvm))
arch/x86/kvm/x86.c
5552
if (!vcpu->kvm->arch.exception_payload_enabled &&
arch/x86/kvm/x86.c
5587
if (!vcpu->kvm->arch.exception_payload_enabled)
arch/x86/kvm/x86.c
5618
if (vcpu->kvm->arch.exception_payload_enabled)
arch/x86/kvm/x86.c
5620
if (vcpu->kvm->arch.triple_fault_event) {
arch/x86/kvm/x86.c
5638
if (!vcpu->kvm->arch.exception_payload_enabled)
arch/x86/kvm/x86.c
5726
if (!vcpu->kvm->arch.triple_fault_event)
arch/x86/kvm/x86.c
5744
if (vcpu->kvm->arch.has_protected_state &&
arch/x86/kvm/x86.c
5766
if (vcpu->kvm->arch.has_protected_state &&
arch/x86/kvm/x86.c
5809
return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
arch/x86/kvm/x86.c
5829
return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
arch/x86/kvm/x86.c
5847
if (vcpu->kvm->arch.has_protected_state &&
arch/x86/kvm/x86.c
5868
if (vcpu->kvm->arch.has_protected_state &&
arch/x86/kvm/x86.c
5945
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/x86.c
5958
raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
arch/x86/kvm/x86.c
5961
kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz &&
arch/x86/kvm/x86.c
5962
kvm->arch.last_tsc_offset == offset);
arch/x86/kvm/x86.c
5968
raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
arch/x86/kvm/x86.c
6022
if (!irqchip_in_kernel(vcpu->kvm))
arch/x86/kvm/x86.c
6150
guard(srcu)(&vcpu->kvm->srcu);
arch/x86/kvm/x86.c
6294
int idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/x86/kvm/x86.c
6296
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/x86/kvm/x86.c
6300
int idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/x86/kvm/x86.c
6302
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/x86/kvm/x86.c
6337
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/x86/kvm/x86.c
6339
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/x86/kvm/x86.c
6584
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/x86/kvm/x86.c
6586
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/x86/kvm/x86.c
6618
if (vcpu->kvm->arch.has_protected_state &&
arch/x86/kvm/x86.c
6635
if (vcpu->kvm->arch.has_protected_state &&
arch/x86/kvm/x86.c
6674
static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
arch/x86/kvm/x86.c
6680
ret = kvm_x86_call(set_tss_addr)(kvm, addr);
arch/x86/kvm/x86.c
6684
static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
arch/x86/kvm/x86.c
6687
return kvm_x86_call(set_identity_map_addr)(kvm, ident_addr);
arch/x86/kvm/x86.c
6690
static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
arch/x86/kvm/x86.c
6696
mutex_lock(&kvm->slots_lock);
arch/x86/kvm/x86.c
6698
kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
arch/x86/kvm/x86.c
6699
kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
arch/x86/kvm/x86.c
6701
mutex_unlock(&kvm->slots_lock);
arch/x86/kvm/x86.c
6705
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
arch/x86/kvm/x86.c
6717
if (!kvm->arch.cpu_dirty_log_size)
arch/x86/kvm/x86.c
6720
kvm_for_each_vcpu(i, vcpu, kvm)
arch/x86/kvm/x86.c
6724
int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
arch/x86/kvm/x86.c
6739
kvm->arch.disabled_quirks |= cap->args[0] & kvm_caps.supported_quirks;
arch/x86/kvm/x86.c
6743
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
6748
if (irqchip_in_kernel(kvm))
arch/x86/kvm/x86.c
6750
if (kvm->created_vcpus)
arch/x86/kvm/x86.c
6754
kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
arch/x86/kvm/x86.c
6755
kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
arch/x86/kvm/x86.c
6756
kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT);
arch/x86/kvm/x86.c
6759
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
6772
!irqchip_split(kvm))
arch/x86/kvm/x86.c
6776
kvm->arch.x2apic_format = true;
arch/x86/kvm/x86.c
6778
kvm->arch.x2apic_broadcast_quirk_disabled = true;
arch/x86/kvm/x86.c
6781
kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_ENABLED;
arch/x86/kvm/x86.c
6783
kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_DISABLED;
arch/x86/kvm/x86.c
6792
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
6793
if (kvm->created_vcpus)
arch/x86/kvm/x86.c
6805
kvm_disable_exits(kvm, cap->args[0]);
arch/x86/kvm/x86.c
6808
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
6811
kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
arch/x86/kvm/x86.c
6815
kvm->arch.exception_payload_enabled = cap->args[0];
arch/x86/kvm/x86.c
6819
kvm->arch.triple_fault_event = cap->args[0];
arch/x86/kvm/x86.c
6826
kvm->arch.user_space_msr_mask = cap->args[0];
arch/x86/kvm/x86.c
6840
kvm->arch.bus_lock_detection_enabled = true;
arch/x86/kvm/x86.c
6854
kvm->arch.sgx_provisioning_allowed = true;
arch/x86/kvm/x86.c
6865
r = kvm_x86_call(vm_copy_enc_context_from)(kvm, cap->args[0]);
arch/x86/kvm/x86.c
6872
r = kvm_x86_call(vm_move_enc_context_from)(kvm, cap->args[0]);
arch/x86/kvm/x86.c
6879
kvm->arch.hypercall_exit_enabled = cap->args[0];
arch/x86/kvm/x86.c
6886
kvm->arch.exit_on_emulation_error = cap->args[0];
arch/x86/kvm/x86.c
6894
if (kvm->arch.has_protected_pmu &&
arch/x86/kvm/x86.c
6898
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
6899
if (!kvm->created_vcpus && !kvm->arch.created_mediated_pmu) {
arch/x86/kvm/x86.c
6900
kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE);
arch/x86/kvm/x86.c
6903
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
6910
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
6911
if (kvm->arch.bsp_vcpu_id > cap->args[0]) {
arch/x86/kvm/x86.c
6913
} else if (kvm->arch.max_vcpu_ids == cap->args[0]) {
arch/x86/kvm/x86.c
6915
} else if (!kvm->arch.max_vcpu_ids) {
arch/x86/kvm/x86.c
6916
kvm->arch.max_vcpu_ids = cap->args[0];
arch/x86/kvm/x86.c
6919
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
6929
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
6930
if (!kvm->created_vcpus) {
arch/x86/kvm/x86.c
6931
kvm->arch.notify_window = cap->args[0] >> 32;
arch/x86/kvm/x86.c
6932
kvm->arch.notify_vmexit_flags = (u32)cap->args[0];
arch/x86/kvm/x86.c
6935
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
6959
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
6960
if (!kvm->created_vcpus) {
arch/x86/kvm/x86.c
6961
kvm->arch.disable_nx_huge_pages = true;
arch/x86/kvm/x86.c
6964
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
6980
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
6981
if (!irqchip_in_kernel(kvm))
arch/x86/kvm/x86.c
6983
else if (kvm->created_vcpus)
arch/x86/kvm/x86.c
6986
kvm->arch.apic_bus_cycle_ns = bus_cycle_ns;
arch/x86/kvm/x86.c
6987
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
7056
static int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm,
arch/x86/kvm/x86.c
7087
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
7088
old_filter = rcu_replace_pointer(kvm->arch.msr_filter, new_filter,
arch/x86/kvm/x86.c
7089
mutex_is_locked(&kvm->lock));
arch/x86/kvm/x86.c
7090
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
7091
synchronize_srcu(&kvm->srcu);
arch/x86/kvm/x86.c
7099
kvm_make_all_cpus_request(kvm, KVM_REQ_RECALC_INTERCEPTS);
arch/x86/kvm/x86.c
7124
struct kvm *kvm = filp->private_data;
arch/x86/kvm/x86.c
7151
r = kvm_vm_ioctl_set_msr_filter(kvm, &filter);
arch/x86/kvm/x86.c
7161
static int kvm_arch_suspend_notifier(struct kvm *kvm)
arch/x86/kvm/x86.c
7170
kvm_for_each_vcpu(i, vcpu, kvm)
arch/x86/kvm/x86.c
7176
int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state)
arch/x86/kvm/x86.c
7181
return kvm_arch_suspend_notifier(kvm);
arch/x86/kvm/x86.c
7188
static int kvm_vm_ioctl_get_clock(struct kvm *kvm, void __user *argp)
arch/x86/kvm/x86.c
7192
get_kvmclock(kvm, &data);
arch/x86/kvm/x86.c
7199
static int kvm_vm_ioctl_set_clock(struct kvm *kvm, void __user *argp)
arch/x86/kvm/x86.c
7201
struct kvm_arch *ka = &kvm->arch;
arch/x86/kvm/x86.c
7215
kvm_hv_request_tsc_page_update(kvm);
arch/x86/kvm/x86.c
7216
kvm_start_pvclock_update(kvm);
arch/x86/kvm/x86.c
7217
pvclock_update_vm_gtod_copy(kvm);
arch/x86/kvm/x86.c
7241
kvm_end_pvclock_update(kvm);
arch/x86/kvm/x86.c
7260
struct kvm *kvm = filp->private_data;
arch/x86/kvm/x86.c
7279
r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
arch/x86/kvm/x86.c
7284
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
7286
if (kvm->created_vcpus)
arch/x86/kvm/x86.c
7291
r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
arch/x86/kvm/x86.c
7293
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
7297
r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
arch/x86/kvm/x86.c
7301
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
7304
if (irqchip_in_kernel(kvm))
arch/x86/kvm/x86.c
7313
if (kvm->arch.has_protected_eoi)
arch/x86/kvm/x86.c
7317
if (kvm->created_vcpus)
arch/x86/kvm/x86.c
7320
r = kvm_pic_init(kvm);
arch/x86/kvm/x86.c
7324
r = kvm_ioapic_init(kvm);
arch/x86/kvm/x86.c
7326
kvm_pic_destroy(kvm);
arch/x86/kvm/x86.c
7330
r = kvm_setup_default_ioapic_and_pic_routing(kvm);
arch/x86/kvm/x86.c
7332
kvm_ioapic_destroy(kvm);
arch/x86/kvm/x86.c
7333
kvm_pic_destroy(kvm);
arch/x86/kvm/x86.c
7338
kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
arch/x86/kvm/x86.c
7339
kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT);
arch/x86/kvm/x86.c
7341
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
7353
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
7355
if (kvm->arch.vpit)
arch/x86/kvm/x86.c
7358
if (!pic_in_kernel(kvm))
arch/x86/kvm/x86.c
7361
kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
arch/x86/kvm/x86.c
7362
if (kvm->arch.vpit)
arch/x86/kvm/x86.c
7365
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
7378
if (!irqchip_full(kvm))
arch/x86/kvm/x86.c
7380
r = kvm_vm_ioctl_get_irqchip(kvm, chip);
arch/x86/kvm/x86.c
7402
if (!irqchip_full(kvm))
arch/x86/kvm/x86.c
7404
r = kvm_vm_ioctl_set_irqchip(kvm, chip);
arch/x86/kvm/x86.c
7414
if (!kvm->arch.vpit)
arch/x86/kvm/x86.c
7416
r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
arch/x86/kvm/x86.c
7429
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
7431
if (!kvm->arch.vpit)
arch/x86/kvm/x86.c
7433
r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
arch/x86/kvm/x86.c
7435
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
7440
if (!kvm->arch.vpit)
arch/x86/kvm/x86.c
7442
r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
arch/x86/kvm/x86.c
7455
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
7457
if (!kvm->arch.vpit)
arch/x86/kvm/x86.c
7459
r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
arch/x86/kvm/x86.c
7461
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
7470
if (!kvm->arch.vpit)
arch/x86/kvm/x86.c
7472
r = kvm_vm_ioctl_reinject(kvm, &control);
arch/x86/kvm/x86.c
7478
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
7479
if (kvm->created_vcpus)
arch/x86/kvm/x86.c
7482
(kvm->arch.max_vcpu_ids && arg > kvm->arch.max_vcpu_ids))
arch/x86/kvm/x86.c
7485
kvm->arch.bsp_vcpu_id = arg;
arch/x86/kvm/x86.c
7486
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
7494
r = kvm_xen_hvm_config(kvm, &xhc);
arch/x86/kvm/x86.c
7503
r = kvm_xen_hvm_get_attr(kvm, &xha);
arch/x86/kvm/x86.c
7514
r = kvm_xen_hvm_set_attr(kvm, &xha);
arch/x86/kvm/x86.c
7523
r = kvm_xen_hvm_evtchn_send(kvm, &uxe);
arch/x86/kvm/x86.c
7528
r = kvm_vm_ioctl_set_clock(kvm, argp);
arch/x86/kvm/x86.c
7531
r = kvm_vm_ioctl_get_clock(kvm, argp);
arch/x86/kvm/x86.c
7546
mutex_lock(&kvm->lock);
arch/x86/kvm/x86.c
7547
if (!kvm->created_vcpus) {
arch/x86/kvm/x86.c
7548
WRITE_ONCE(kvm->arch.default_tsc_khz, user_tsc_khz);
arch/x86/kvm/x86.c
7551
mutex_unlock(&kvm->lock);
arch/x86/kvm/x86.c
7555
r = READ_ONCE(kvm->arch.default_tsc_khz);
arch/x86/kvm/x86.c
7563
r = kvm_x86_call(mem_enc_ioctl)(kvm, argp);
arch/x86/kvm/x86.c
7576
r = kvm_x86_call(mem_enc_register_region)(kvm, &region);
arch/x86/kvm/x86.c
7590
r = kvm_x86_call(mem_enc_unregister_region)(kvm, &region);
arch/x86/kvm/x86.c
7600
r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd);
arch/x86/kvm/x86.c
7605
r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp);
arch/x86/kvm/x86.c
7614
r = kvm_vm_ioctl_set_msr_filter(kvm, &filter);
arch/x86/kvm/x86.c
8878
struct kvm *kvm = emul_to_vcpu(ctxt)->kvm;
arch/x86/kvm/x86.c
8880
if (!kvm->vm_bugged)
arch/x86/kvm/x86.c
8881
kvm_vm_bugged(kvm);
arch/x86/kvm/x86.c
9181
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/x86.c
9191
if (kvm->arch.exit_on_emulation_error ||
arch/x86/kvm/x86.c
9684
kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
arch/x86/kvm/x86.c
9700
if (KVM_BUG_ON(vcpu->arch.pio.count != 1, vcpu->kvm))
arch/x86/kvm/x86.c
9775
struct kvm *kvm;
arch/x86/kvm/x86.c
9779
list_for_each_entry(kvm, &vm_list, vm_list)
arch/x86/kvm/x86.c
9780
kvm_make_mclock_inprogress_request(kvm);
arch/x86/kvm/x86.c
9792
list_for_each_entry(kvm, &vm_list, vm_list) {
arch/x86/kvm/x86.c
9793
__kvm_start_pvclock_update(kvm);
arch/x86/kvm/x86.c
9794
pvclock_update_vm_gtod_copy(kvm);
arch/x86/kvm/x86.c
9795
kvm_end_pvclock_update(kvm);
arch/x86/kvm/x86.c
9804
struct kvm *kvm;
arch/x86/kvm/x86.c
9851
list_for_each_entry(kvm, &vm_list, vm_list) {
arch/x86/kvm/x86.c
9852
kvm_for_each_vcpu(i, vcpu, kvm) {
arch/x86/kvm/x86.c
9935
struct kvm *kvm;
arch/x86/kvm/x86.c
9940
list_for_each_entry(kvm, &vm_list, vm_list)
arch/x86/kvm/x86.c
9941
kvm_for_each_vcpu(i, vcpu, kvm)
arch/x86/kvm/x86.h
316
u64 gen = kvm_memslots(vcpu->kvm)->generation;
arch/x86/kvm/x86.h
333
return vcpu->arch.mmio_gen == kvm_memslots(vcpu->kvm)->generation;
arch/x86/kvm/x86.h
368
static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk)
arch/x86/kvm/x86.h
370
return !(kvm->arch.disabled_quirks & quirk);
arch/x86/kvm/x86.h
389
u64 get_kvmclock_ns(struct kvm *kvm);
arch/x86/kvm/x86.h
390
uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm);
arch/x86/kvm/x86.h
500
static inline void kvm_disable_exits(struct kvm *kvm, u64 mask)
arch/x86/kvm/x86.h
502
kvm->arch.disabled_exits |= mask;
arch/x86/kvm/x86.h
505
static inline bool kvm_mwait_in_guest(struct kvm *kvm)
arch/x86/kvm/x86.h
507
return kvm->arch.disabled_exits & KVM_X86_DISABLE_EXITS_MWAIT;
arch/x86/kvm/x86.h
510
static inline bool kvm_hlt_in_guest(struct kvm *kvm)
arch/x86/kvm/x86.h
512
return kvm->arch.disabled_exits & KVM_X86_DISABLE_EXITS_HLT;
arch/x86/kvm/x86.h
515
static inline bool kvm_pause_in_guest(struct kvm *kvm)
arch/x86/kvm/x86.h
517
return kvm->arch.disabled_exits & KVM_X86_DISABLE_EXITS_PAUSE;
arch/x86/kvm/x86.h
520
static inline bool kvm_cstate_in_guest(struct kvm *kvm)
arch/x86/kvm/x86.h
522
return kvm->arch.disabled_exits & KVM_X86_DISABLE_EXITS_CSTATE;
arch/x86/kvm/x86.h
525
static inline bool kvm_aperfmperf_in_guest(struct kvm *kvm)
arch/x86/kvm/x86.h
527
return kvm->arch.disabled_exits & KVM_X86_DISABLE_EXITS_APERFMPERF;
arch/x86/kvm/x86.h
530
static inline bool kvm_notify_vmexit_enabled(struct kvm *kvm)
arch/x86/kvm/x86.h
532
return kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_ENABLED;
arch/x86/kvm/x86.h
637
KVM_BUG_ON(len > 8, vcpu->kvm);
arch/x86/kvm/x86.h
662
static inline bool user_exit_on_hypercall(struct kvm *kvm, unsigned long hc_nr)
arch/x86/kvm/x86.h
664
return kvm->arch.hypercall_exit_enabled & BIT(hc_nr);
arch/x86/kvm/xen.c
101
kvm_make_all_cpus_request(kvm, KVM_REQ_MASTERCLOCK_UPDATE);
arch/x86/kvm/xen.c
1039
if (get_kvmclock_ns(vcpu->kvm) <
arch/x86/kvm/xen.c
104
srcu_read_unlock(&kvm->srcu, idx);
arch/x86/kvm/xen.c
1080
if (get_kvmclock_ns(vcpu->kvm) <
arch/x86/kvm/xen.c
1145
srcu_read_unlock(&vcpu->kvm->srcu, idx);
arch/x86/kvm/xen.c
1146
mutex_unlock(&vcpu->kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
1154
mutex_lock(&vcpu->kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
118
kvm_xen_set_evtchn(&e, vcpu->kvm);
arch/x86/kvm/xen.c
1272
mutex_unlock(&vcpu->kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
1278
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/xen.c
1284
mutex_lock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
1285
if (kvm->arch.xen.long_mode != lm) {
arch/x86/kvm/xen.c
1286
kvm->arch.xen.long_mode = lm;
arch/x86/kvm/xen.c
1292
if (kvm->arch.xen.shinfo_cache.active &&
arch/x86/kvm/xen.c
1293
kvm_xen_shared_info_init(kvm))
arch/x86/kvm/xen.c
1296
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
1307
if (kvm_xen_hypercall_enabled(kvm)) {
arch/x86/kvm/xen.c
1338
hva_t blob_addr = lm ? kvm->arch.xen.hvm_config.blob_addr_64
arch/x86/kvm/xen.c
1339
: kvm->arch.xen.hvm_config.blob_addr_32;
arch/x86/kvm/xen.c
1340
u8 blob_size = lm ? kvm->arch.xen.hvm_config.blob_size_64
arch/x86/kvm/xen.c
1341
: kvm->arch.xen.hvm_config.blob_size_32;
arch/x86/kvm/xen.c
1362
int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc)
arch/x86/kvm/xen.c
1391
mutex_lock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
1393
if (xhc->msr && !kvm->arch.xen.hvm_config.msr)
arch/x86/kvm/xen.c
1395
else if (!xhc->msr && kvm->arch.xen.hvm_config.msr)
arch/x86/kvm/xen.c
1398
old_flags = kvm->arch.xen.hvm_config.flags;
arch/x86/kvm/xen.c
1399
memcpy(&kvm->arch.xen.hvm_config, xhc, sizeof(*xhc));
arch/x86/kvm/xen.c
140
rc = kvm_xen_set_evtchn_fast(&e, vcpu->kvm);
arch/x86/kvm/xen.c
1401
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
1404
kvm_make_all_cpus_request(kvm, KVM_REQ_CLOCK_UPDATE);
arch/x86/kvm/xen.c
1425
static inline int max_evtchn_port(struct kvm *kvm)
arch/x86/kvm/xen.c
1427
if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode)
arch/x86/kvm/xen.c
1436
struct kvm *kvm = vcpu->kvm;
arch/x86/kvm/xen.c
1437
struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
arch/x86/kvm/xen.c
1443
idx = srcu_read_lock(&kvm->srcu);
arch/x86/kvm/xen.c
1449
if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
arch/x86/kvm/xen.c
1466
srcu_read_unlock(&kvm->srcu, idx);
arch/x86/kvm/xen.c
1480
!(vcpu->kvm->arch.xen.hvm_config.flags & KVM_XEN_HVM_CONFIG_EVTCHN_SEND))
arch/x86/kvm/xen.c
1532
if (ports[i] >= max_evtchn_port(vcpu->kvm)) {
arch/x86/kvm/xen.c
1543
set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask);
arch/x86/kvm/xen.c
1564
clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask);
arch/x86/kvm/xen.c
1706
KVM_BUG_ON(1, vcpu->kvm);
arch/x86/kvm/xen.c
1723
if (params[0] == XENVER_version && vcpu->kvm->arch.xen.xen_version) {
arch/x86/kvm/xen.c
1724
r = vcpu->kvm->arch.xen.xen_version;
arch/x86/kvm/xen.c
1779
test_and_clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask)) {
arch/x86/kvm/xen.c
1795
int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm)
arch/x86/kvm/xen.c
1797
struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
arch/x86/kvm/xen.c
1807
vcpu = kvm_get_vcpu(kvm, vcpu_idx);
arch/x86/kvm/xen.c
1809
vcpu = kvm_get_vcpu_by_id(kvm, xe->vcpu_id);
arch/x86/kvm/xen.c
1815
if (xe->port >= max_evtchn_port(kvm))
arch/x86/kvm/xen.c
1820
idx = srcu_read_lock(&kvm->srcu);
arch/x86/kvm/xen.c
1826
if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
arch/x86/kvm/xen.c
1867
if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
arch/x86/kvm/xen.c
1891
srcu_read_unlock(&kvm->srcu, idx);
arch/x86/kvm/xen.c
1901
static int kvm_xen_set_evtchn(struct kvm_xen_evtchn *xe, struct kvm *kvm)
arch/x86/kvm/xen.c
1906
rc = kvm_xen_set_evtchn_fast(xe, kvm);
arch/x86/kvm/xen.c
1910
if (current->mm != kvm->mm) {
arch/x86/kvm/xen.c
1918
kthread_use_mm(kvm->mm);
arch/x86/kvm/xen.c
1937
struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
arch/x86/kvm/xen.c
1940
rc = kvm_xen_set_evtchn_fast(xe, kvm);
arch/x86/kvm/xen.c
1944
idx = srcu_read_lock(&kvm->srcu);
arch/x86/kvm/xen.c
1946
srcu_read_unlock(&kvm->srcu, idx);
arch/x86/kvm/xen.c
1950
kthread_unuse_mm(kvm->mm);
arch/x86/kvm/xen.c
1956
static int evtchn_set_fn(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
arch/x86/kvm/xen.c
1962
return kvm_xen_set_evtchn(&e->xen_evtchn, kvm);
arch/x86/kvm/xen.c
1969
int kvm_xen_setup_evtchn(struct kvm *kvm,
arch/x86/kvm/xen.c
2002
vcpu = kvm_get_vcpu_by_id(kvm, ue->u.xen_evtchn.vcpu);
arch/x86/kvm/xen.c
2019
int kvm_xen_hvm_evtchn_send(struct kvm *kvm, struct kvm_irq_routing_xen_evtchn *uxe)
arch/x86/kvm/xen.c
2024
if (!uxe->port || uxe->port >= max_evtchn_port(kvm))
arch/x86/kvm/xen.c
2036
ret = kvm_xen_set_evtchn(&e, kvm);
arch/x86/kvm/xen.c
2066
static int kvm_xen_eventfd_update(struct kvm *kvm,
arch/x86/kvm/xen.c
2074
mutex_lock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
2075
evtchnfd = idr_find(&kvm->arch.xen.evtchn_ports, port);
arch/x86/kvm/xen.c
2105
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
2113
static int kvm_xen_eventfd_assign(struct kvm *kvm,
arch/x86/kvm/xen.c
2134
if (data->u.evtchn.deliver.port.port >= max_evtchn_port(kvm))
arch/x86/kvm/xen.c
216
!vcpu->kvm->arch.use_master_clock)
arch/x86/kvm/xen.c
2168
mutex_lock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
2169
ret = idr_alloc(&kvm->arch.xen.evtchn_ports, evtchnfd, port, port + 1,
arch/x86/kvm/xen.c
2171
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
2185
static int kvm_xen_eventfd_deassign(struct kvm *kvm, u32 port)
arch/x86/kvm/xen.c
2189
mutex_lock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
2190
evtchnfd = idr_remove(&kvm->arch.xen.evtchn_ports, port);
arch/x86/kvm/xen.c
2191
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
2196
synchronize_srcu(&kvm->srcu);
arch/x86/kvm/xen.c
2203
static int kvm_xen_eventfd_reset(struct kvm *kvm)
arch/x86/kvm/xen.c
2209
mutex_lock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
2216
idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i)
arch/x86/kvm/xen.c
2221
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
2226
idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) {
arch/x86/kvm/xen.c
2228
idr_remove(&kvm->arch.xen.evtchn_ports, evtchnfd->send_port);
arch/x86/kvm/xen.c
2230
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
2232
synchronize_srcu(&kvm->srcu);
arch/x86/kvm/xen.c
2245
static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data)
arch/x86/kvm/xen.c
2250
return kvm_xen_eventfd_reset(kvm);
arch/x86/kvm/xen.c
2252
if (!port || port >= max_evtchn_port(kvm))
arch/x86/kvm/xen.c
2256
return kvm_xen_eventfd_deassign(kvm, port);
arch/x86/kvm/xen.c
2258
return kvm_xen_eventfd_update(kvm, data);
arch/x86/kvm/xen.c
2262
return kvm_xen_eventfd_assign(kvm, data);
arch/x86/kvm/xen.c
2283
evtchnfd = idr_find(&vcpu->kvm->arch.xen.evtchn_ports, send.port);
arch/x86/kvm/xen.c
2289
int ret = kvm_xen_set_evtchn(&evtchnfd->deliver.port, vcpu->kvm);
arch/x86/kvm/xen.c
2309
kvm_gpc_init(&vcpu->arch.xen.runstate_cache, vcpu->kvm);
arch/x86/kvm/xen.c
2310
kvm_gpc_init(&vcpu->arch.xen.runstate2_cache, vcpu->kvm);
arch/x86/kvm/xen.c
2311
kvm_gpc_init(&vcpu->arch.xen.vcpu_info_cache, vcpu->kvm);
arch/x86/kvm/xen.c
2312
kvm_gpc_init(&vcpu->arch.xen.vcpu_time_info_cache, vcpu->kvm);
arch/x86/kvm/xen.c
2328
void kvm_xen_init_vm(struct kvm *kvm)
arch/x86/kvm/xen.c
2330
mutex_init(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
2331
idr_init(&kvm->arch.xen.evtchn_ports);
arch/x86/kvm/xen.c
2332
kvm_gpc_init(&kvm->arch.xen.shinfo_cache, kvm);
arch/x86/kvm/xen.c
2335
void kvm_xen_destroy_vm(struct kvm *kvm)
arch/x86/kvm/xen.c
2340
kvm_gpc_deactivate(&kvm->arch.xen.shinfo_cache);
arch/x86/kvm/xen.c
2342
idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) {
arch/x86/kvm/xen.c
2347
idr_destroy(&kvm->arch.xen.evtchn_ports);
arch/x86/kvm/xen.c
2349
if (kvm->arch.xen.hvm_config.msr)
arch/x86/kvm/xen.c
272
guest_now = get_kvmclock_ns(vcpu->kvm);
arch/x86/kvm/xen.c
32
static int kvm_xen_set_evtchn(struct kvm_xen_evtchn *xe, struct kvm *kvm);
arch/x86/kvm/xen.c
33
static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data);
arch/x86/kvm/xen.c
38
static int kvm_xen_shared_info_init(struct kvm *kvm)
arch/x86/kvm/xen.c
393
if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) {
arch/x86/kvm/xen.c
40
struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
arch/x86/kvm/xen.c
457
if (v->kvm->arch.xen.runstate_update_flag)
arch/x86/kvm/xen.c
46
int idx = srcu_read_lock(&kvm->srcu);
arch/x86/kvm/xen.c
521
if (v->kvm->arch.xen.runstate_update_flag) {
arch/x86/kvm/xen.c
590
u64 now = get_kvmclock_ns(v->kvm);
arch/x86/kvm/xen.c
629
kvm_irq_delivery_to_apic(v->kvm, NULL, &irq);
arch/x86/kvm/xen.c
63
wall_nsec = kvm_get_wall_clock_epoch(kvm);
arch/x86/kvm/xen.c
664
if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) {
arch/x86/kvm/xen.c
746
int kvm_xen_hvm_set_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data)
arch/x86/kvm/xen.c
75
if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
arch/x86/kvm/xen.c
756
mutex_lock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
757
kvm->arch.xen.long_mode = !!data->u.long_mode;
arch/x86/kvm/xen.c
765
r = kvm->arch.xen.shinfo_cache.active ?
arch/x86/kvm/xen.c
766
kvm_xen_shared_info_init(kvm) : 0;
arch/x86/kvm/xen.c
767
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
775
mutex_lock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
777
idx = srcu_read_lock(&kvm->srcu);
arch/x86/kvm/xen.c
783
kvm_gpc_deactivate(&kvm->arch.xen.shinfo_cache);
arch/x86/kvm/xen.c
786
r = kvm_gpc_activate(&kvm->arch.xen.shinfo_cache,
arch/x86/kvm/xen.c
795
kvm_gpc_deactivate(&kvm->arch.xen.shinfo_cache);
arch/x86/kvm/xen.c
798
r = kvm_gpc_activate_hva(&kvm->arch.xen.shinfo_cache,
arch/x86/kvm/xen.c
803
srcu_read_unlock(&kvm->srcu, idx);
arch/x86/kvm/xen.c
805
if (!r && kvm->arch.xen.shinfo_cache.active)
arch/x86/kvm/xen.c
806
r = kvm_xen_shared_info_init(kvm);
arch/x86/kvm/xen.c
808
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
815
mutex_lock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
816
kvm->arch.xen.upcall_vector = data->u.vector;
arch/x86/kvm/xen.c
817
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
823
r = kvm_xen_setattr_evtchn(kvm, data);
arch/x86/kvm/xen.c
827
mutex_lock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
828
kvm->arch.xen.xen_version = data->u.xen_version;
arch/x86/kvm/xen.c
829
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
838
mutex_lock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
839
kvm->arch.xen.runstate_update_flag = !!data->u.runstate_update_flag;
arch/x86/kvm/xen.c
840
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
851
int kvm_xen_hvm_get_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data)
arch/x86/kvm/xen.c
855
mutex_lock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
859
data->u.long_mode = kvm->arch.xen.long_mode;
arch/x86/kvm/xen.c
864
if (kvm_gpc_is_gpa_active(&kvm->arch.xen.shinfo_cache))
arch/x86/kvm/xen.c
865
data->u.shared_info.gfn = gpa_to_gfn(kvm->arch.xen.shinfo_cache.gpa);
arch/x86/kvm/xen.c
872
if (kvm_gpc_is_hva_active(&kvm->arch.xen.shinfo_cache))
arch/x86/kvm/xen.c
873
data->u.shared_info.hva = kvm->arch.xen.shinfo_cache.uhva;
arch/x86/kvm/xen.c
880
data->u.vector = kvm->arch.xen.upcall_vector;
arch/x86/kvm/xen.c
885
data->u.xen_version = kvm->arch.xen.xen_version;
arch/x86/kvm/xen.c
894
data->u.runstate_update_flag = kvm->arch.xen.runstate_update_flag;
arch/x86/kvm/xen.c
902
mutex_unlock(&kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
910
mutex_lock(&vcpu->kvm->arch.xen.xen_lock);
arch/x86/kvm/xen.c
911
idx = srcu_read_lock(&vcpu->kvm->srcu);
arch/x86/kvm/xen.c
981
if (IS_ENABLED(CONFIG_64BIT) && vcpu->kvm->arch.xen.long_mode)
arch/x86/kvm/xen.h
105
if (kvm_xen_hypercall_enabled(vcpu->kvm) && kvm_xen_timer_enabled(vcpu))
arch/x86/kvm/xen.h
118
static inline void kvm_xen_init_vm(struct kvm *kvm)
arch/x86/kvm/xen.h
122
static inline void kvm_xen_destroy_vm(struct kvm *kvm)
arch/x86/kvm/xen.h
138
static inline bool kvm_xen_msr_enabled(struct kvm *kvm)
arch/x86/kvm/xen.h
143
static inline bool kvm_xen_is_hypercall_page_msr(struct kvm *kvm, u32 msr)
arch/x86/kvm/xen.h
148
static inline bool kvm_xen_hypercall_enabled(struct kvm *kvm)
arch/x86/kvm/xen.h
25
int kvm_xen_hvm_set_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data);
arch/x86/kvm/xen.h
26
int kvm_xen_hvm_get_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data);
arch/x86/kvm/xen.h
27
int kvm_xen_hvm_evtchn_send(struct kvm *kvm, struct kvm_irq_routing_xen_evtchn *evt);
arch/x86/kvm/xen.h
29
int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc);
arch/x86/kvm/xen.h
30
void kvm_xen_init_vm(struct kvm *kvm);
arch/x86/kvm/xen.h
31
void kvm_xen_destroy_vm(struct kvm *kvm);
arch/x86/kvm/xen.h
35
struct kvm *kvm);
arch/x86/kvm/xen.h
36
int kvm_xen_setup_evtchn(struct kvm *kvm,
arch/x86/kvm/xen.h
61
static inline bool kvm_xen_msr_enabled(struct kvm *kvm)
arch/x86/kvm/xen.h
64
kvm->arch.xen.hvm_config.msr;
arch/x86/kvm/xen.h
67
static inline bool kvm_xen_is_hypercall_page_msr(struct kvm *kvm, u32 msr)
arch/x86/kvm/xen.h
72
return msr && msr == kvm->arch.xen.hvm_config.msr;
arch/x86/kvm/xen.h
75
static inline bool kvm_xen_hypercall_enabled(struct kvm *kvm)
arch/x86/kvm/xen.h
78
(kvm->arch.xen.hvm_config.flags &
arch/x86/kvm/xen.h
86
vcpu->kvm->arch.xen.upcall_vector)
drivers/gpu/drm/i915/gvt/kvmgt.c
1535
r = kvm_write_track_add_gfn(info->vfio_device.kvm, gfn);
drivers/gpu/drm/i915/gvt/kvmgt.c
1553
r = kvm_write_track_remove_gfn(info->vfio_device.kvm, gfn);
drivers/gpu/drm/i915/gvt/kvmgt.c
653
if (vgpu->vfio_device.kvm == itr->vfio_device.kvm) {
drivers/gpu/drm/i915/gvt/kvmgt.c
673
ret = kvm_page_track_register_notifier(vgpu->vfio_device.kvm,
drivers/gpu/drm/i915/gvt/kvmgt.c
711
kvm_page_track_unregister_notifier(vgpu->vfio_device.kvm,
drivers/pci/tsm.c
342
int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u32 tdi_id)
drivers/pci/tsm.c
347
if (!kvm)
drivers/pci/tsm.c
363
if (pdev->tsm->tdi->kvm != kvm)
drivers/pci/tsm.c
368
tdi = to_pci_tsm_ops(pdev->tsm)->bind(pdev, kvm, tdi_id);
drivers/pci/tsm.c
705
struct kvm *kvm, u32 tdi_id)
drivers/pci/tsm.c
708
tdi->kvm = kvm;
drivers/s390/crypto/vfio_ap_ops.c
103
if (matrix_mdev && matrix_mdev->kvm)
drivers/s390/crypto/vfio_ap_ops.c
104
mutex_lock(&matrix_mdev->kvm->lock);
drivers/s390/crypto/vfio_ap_ops.c
126
if (matrix_mdev && matrix_mdev->kvm)
drivers/s390/crypto/vfio_ap_ops.c
127
mutex_unlock(&matrix_mdev->kvm->lock);
drivers/s390/crypto/vfio_ap_ops.c
159
if (matrix_mdev->kvm)
drivers/s390/crypto/vfio_ap_ops.c
160
mutex_lock(&matrix_mdev->kvm->lock);
drivers/s390/crypto/vfio_ap_ops.c
1822
struct kvm *kvm)
drivers/s390/crypto/vfio_ap_ops.c
1826
if (kvm->arch.crypto.crycbd) {
drivers/s390/crypto/vfio_ap_ops.c
1827
down_write(&kvm->arch.crypto.pqap_hook_rwsem);
drivers/s390/crypto/vfio_ap_ops.c
1828
kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
drivers/s390/crypto/vfio_ap_ops.c
1829
up_write(&kvm->arch.crypto.pqap_hook_rwsem);
drivers/s390/crypto/vfio_ap_ops.c
1831
get_update_locks_for_kvm(kvm);
drivers/s390/crypto/vfio_ap_ops.c
1834
if (m != matrix_mdev && m->kvm == kvm) {
drivers/s390/crypto/vfio_ap_ops.c
1835
release_update_locks_for_kvm(kvm);
drivers/s390/crypto/vfio_ap_ops.c
1840
kvm_get_kvm(kvm);
drivers/s390/crypto/vfio_ap_ops.c
1841
matrix_mdev->kvm = kvm;
drivers/s390/crypto/vfio_ap_ops.c
1844
release_update_locks_for_kvm(kvm);
drivers/s390/crypto/vfio_ap_ops.c
1883
struct kvm *kvm = matrix_mdev->kvm;
drivers/s390/crypto/vfio_ap_ops.c
1885
if (kvm && kvm->arch.crypto.crycbd) {
drivers/s390/crypto/vfio_ap_ops.c
1886
down_write(&kvm->arch.crypto.pqap_hook_rwsem);
drivers/s390/crypto/vfio_ap_ops.c
1887
kvm->arch.crypto.pqap_hook = NULL;
drivers/s390/crypto/vfio_ap_ops.c
1888
up_write(&kvm->arch.crypto.pqap_hook_rwsem);
drivers/s390/crypto/vfio_ap_ops.c
1890
get_update_locks_for_kvm(kvm);
drivers/s390/crypto/vfio_ap_ops.c
1892
kvm_arch_crypto_clear_masks(kvm);
drivers/s390/crypto/vfio_ap_ops.c
1894
kvm_put_kvm(kvm);
drivers/s390/crypto/vfio_ap_ops.c
1895
matrix_mdev->kvm = NULL;
drivers/s390/crypto/vfio_ap_ops.c
1897
release_update_locks_for_kvm(kvm);
drivers/s390/crypto/vfio_ap_ops.c
192
if (q->matrix_mdev && q->matrix_mdev->kvm)
drivers/s390/crypto/vfio_ap_ops.c
193
mutex_lock(&q->matrix_mdev->kvm->lock);
drivers/s390/crypto/vfio_ap_ops.c
2058
if (!vdev->kvm)
drivers/s390/crypto/vfio_ap_ops.c
2061
return vfio_ap_mdev_set_kvm(matrix_mdev, vdev->kvm);
drivers/s390/crypto/vfio_ap_ops.c
2081
if (matrix_mdev->kvm) {
drivers/s390/crypto/vfio_ap_ops.c
2082
kvm_arch_crypto_clear_masks(matrix_mdev->kvm);
drivers/s390/crypto/vfio_ap_ops.c
2335
if (matrix_mdev->kvm &&
drivers/s390/crypto/vfio_ap_ops.c
2609
mutex_lock(&matrix_mdev->kvm->lock);
drivers/s390/crypto/vfio_ap_ops.c
2627
mutex_unlock(&matrix_mdev->kvm->lock);
drivers/s390/crypto/vfio_ap_ops.c
274
!WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) {
drivers/s390/crypto/vfio_ap_ops.c
275
kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc);
drivers/s390/crypto/vfio_ap_ops.c
2824
mutex_lock(&matrix_mdev->kvm->lock);
drivers/s390/crypto/vfio_ap_ops.c
2846
mutex_unlock(&matrix_mdev->kvm->lock);
drivers/s390/crypto/vfio_ap_ops.c
357
if (!kvm_s390_is_gpa_in_memslot(vcpu->kvm, *nib))
drivers/s390/crypto/vfio_ap_ops.c
425
struct kvm *kvm;
drivers/s390/crypto/vfio_ap_ops.c
454
kvm = q->matrix_mdev->kvm;
drivers/s390/crypto/vfio_ap_ops.c
455
gisa = kvm->arch.gisa_int.origin;
drivers/s390/crypto/vfio_ap_ops.c
468
nisc = kvm_s390_gisc_register(kvm, isc);
drivers/s390/crypto/vfio_ap_ops.c
492
ret = kvm_s390_gisc_unregister(kvm, isc);
drivers/s390/crypto/vfio_ap_ops.c
55
static inline void get_update_locks_for_kvm(struct kvm *kvm)
drivers/s390/crypto/vfio_ap_ops.c
58
if (kvm)
drivers/s390/crypto/vfio_ap_ops.c
59
mutex_lock(&kvm->lock);
drivers/s390/crypto/vfio_ap_ops.c
603
if (!vcpu->kvm->arch.crypto.pqap_hook) {
drivers/s390/crypto/vfio_ap_ops.c
610
matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,
drivers/s390/crypto/vfio_ap_ops.c
614
if (!matrix_mdev->kvm) {
drivers/s390/crypto/vfio_ap_ops.c
661
if (matrix_mdev->kvm) {
drivers/s390/crypto/vfio_ap_ops.c
662
kvm_arch_crypto_set_masks(matrix_mdev->kvm,
drivers/s390/crypto/vfio_ap_ops.c
76
static inline void release_update_locks_for_kvm(struct kvm *kvm)
drivers/s390/crypto/vfio_ap_ops.c
79
if (kvm)
drivers/s390/crypto/vfio_ap_ops.c
80
mutex_unlock(&kvm->lock);
drivers/s390/crypto/vfio_ap_private.h
119
struct kvm *kvm;
drivers/vfio/device_cdev.c
59
vfio_device_get_kvm_safe(df->device, df->kvm);
drivers/vfio/group.c
166
vfio_device_get_kvm_safe(device, device->group->kvm);
drivers/vfio/group.c
863
void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm)
drivers/vfio/group.c
866
group->kvm = kvm;
drivers/vfio/pci/vfio_pci_zdev.c
151
if (!vdev->vdev.kvm)
drivers/vfio/pci/vfio_pci_zdev.c
155
return zpci_kvm_hook.kvm_register(zdev, vdev->vdev.kvm);
drivers/vfio/pci/vfio_pci_zdev.c
164
if (!zdev || !vdev->vdev.kvm)
drivers/vfio/vfio.h
110
void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm);
drivers/vfio/vfio.h
173
static inline void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm)
drivers/vfio/vfio.h
26
struct kvm *kvm;
drivers/vfio/vfio.h
437
void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm);
drivers/vfio/vfio.h
441
struct kvm *kvm)
drivers/vfio/vfio.h
91
struct kvm *kvm;
drivers/vfio/vfio_main.c
1523
static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm)
drivers/vfio/vfio_main.c
1533
df->kvm = kvm;
drivers/vfio/vfio_main.c
1545
void vfio_file_set_kvm(struct file *file, struct kvm *kvm)
drivers/vfio/vfio_main.c
1551
vfio_group_set_kvm(group, kvm);
drivers/vfio/vfio_main.c
1554
vfio_device_file_set_kvm(file, kvm);
drivers/vfio/vfio_main.c
451
void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm)
drivers/vfio/vfio_main.c
453
void (*pfn)(struct kvm *kvm);
drivers/vfio/vfio_main.c
454
bool (*fn)(struct kvm *kvm);
drivers/vfio/vfio_main.c
459
if (!kvm)
drivers/vfio/vfio_main.c
472
ret = fn(kvm);
drivers/vfio/vfio_main.c
480
device->kvm = kvm;
drivers/vfio/vfio_main.c
487
if (!device->kvm)
drivers/vfio/vfio_main.c
493
device->put_kvm(device->kvm);
drivers/vfio/vfio_main.c
498
device->kvm = NULL;
include/kvm/arm_arch_timer.h
105
void kvm_timer_init_vm(struct kvm *kvm);
include/kvm/arm_arch_timer.h
127
#define timer_vm_data(ctx) (&(timer_context_to_vcpu(ctx)->kvm->arch.timer_data))
include/kvm/arm_hypercalls.h
45
void kvm_arm_init_hypercalls(struct kvm *kvm);
include/kvm/arm_hypercalls.h
46
void kvm_arm_teardown_hypercalls(struct kvm *kvm);
include/kvm/arm_hypercalls.h
52
int kvm_vm_smccc_has_attr(struct kvm *kvm, struct kvm_device_attr *attr);
include/kvm/arm_hypercalls.h
53
int kvm_vm_smccc_set_attr(struct kvm *kvm, struct kvm_device_attr *attr);
include/kvm/arm_pmu.h
171
static inline u64 kvm_pmu_evtyper_mask(struct kvm *kvm)
include/kvm/arm_pmu.h
177
static inline int kvm_arm_set_default_pmu(struct kvm *kvm)
include/kvm/arm_pmu.h
182
static inline u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm)
include/kvm/arm_pmu.h
90
u64 kvm_pmu_evtyper_mask(struct kvm *kvm);
include/kvm/arm_pmu.h
91
int kvm_arm_set_default_pmu(struct kvm *kvm);
include/kvm/arm_pmu.h
92
u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm);
include/kvm/arm_psci.h
32
if (vcpu->kvm->arch.psci_version)
include/kvm/arm_psci.h
33
return vcpu->kvm->arch.psci_version;
include/kvm/arm_vgic.h
222
bool (*queue_irq_unlock)(struct kvm *kvm, struct vgic_irq *irq,
include/kvm/arm_vgic.h
543
int kvm_set_legacy_vgic_v2_addr(struct kvm *kvm, struct kvm_arm_device_addr *dev_addr);
include/kvm/arm_vgic.h
544
void kvm_vgic_early_init(struct kvm *kvm);
include/kvm/arm_vgic.h
547
int kvm_vgic_create(struct kvm *kvm, u32 type);
include/kvm/arm_vgic.h
548
void kvm_vgic_destroy(struct kvm *kvm);
include/kvm/arm_vgic.h
550
int kvm_vgic_map_resources(struct kvm *kvm);
include/kvm/arm_vgic.h
551
void kvm_vgic_finalize_idregs(struct kvm *kvm);
include/kvm/arm_vgic.h
555
int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
include/kvm/arm_vgic.h
615
int kvm_vgic_setup_default_irq_routing(struct kvm *kvm);
include/kvm/arm_vgic.h
621
int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int irq,
include/kvm/arm_vgic.h
624
void kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int host_irq);
include/kvm/arm_vgic.h
630
int vgic_v5_finalize_ppi_state(struct kvm *kvm);
include/kvm/arm_vgic.h
631
bool vgic_v5_ppi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
include/linux/kvm_dirty_ring.h
35
static inline u32 kvm_dirty_ring_get_rsvd_entries(struct kvm *kvm)
include/linux/kvm_dirty_ring.h
40
static inline bool kvm_use_dirty_bitmap(struct kvm *kvm)
include/linux/kvm_dirty_ring.h
45
static inline int kvm_dirty_ring_alloc(struct kvm *kvm, struct kvm_dirty_ring *ring,
include/linux/kvm_dirty_ring.h
51
static inline int kvm_dirty_ring_reset(struct kvm *kvm,
include/linux/kvm_dirty_ring.h
75
int kvm_cpu_dirty_log_size(struct kvm *kvm);
include/linux/kvm_dirty_ring.h
76
bool kvm_use_dirty_bitmap(struct kvm *kvm);
include/linux/kvm_dirty_ring.h
77
bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm);
include/linux/kvm_dirty_ring.h
78
u32 kvm_dirty_ring_get_rsvd_entries(struct kvm *kvm);
include/linux/kvm_dirty_ring.h
79
int kvm_dirty_ring_alloc(struct kvm *kvm, struct kvm_dirty_ring *ring,
include/linux/kvm_dirty_ring.h
81
int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring,
include/linux/kvm_host.h
1008
return xa_load(&kvm->vcpu_array, i);
include/linux/kvm_host.h
1011
#define kvm_for_each_vcpu(idx, vcpup, kvm) \
include/linux/kvm_host.h
1012
if (atomic_read(&kvm->online_vcpus)) \
include/linux/kvm_host.h
1013
xa_for_each_range(&kvm->vcpu_array, idx, vcpup, 0, \
include/linux/kvm_host.h
1014
(atomic_read(&kvm->online_vcpus) - 1))
include/linux/kvm_host.h
1016
static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id)
include/linux/kvm_host.h
1024
vcpu = kvm_get_vcpu(kvm, id);
include/linux/kvm_host.h
1027
kvm_for_each_vcpu(i, vcpu, kvm)
include/linux/kvm_host.h
1033
static inline bool kvm_is_vcpu_creation_in_progress(struct kvm *kvm)
include/linux/kvm_host.h
1035
lockdep_assert_held(&kvm->lock);
include/linux/kvm_host.h
1037
return kvm->created_vcpus != atomic_read(&kvm->online_vcpus);
include/linux/kvm_host.h
1040
void kvm_destroy_vcpus(struct kvm *kvm);
include/linux/kvm_host.h
1042
int kvm_trylock_all_vcpus(struct kvm *kvm);
include/linux/kvm_host.h
1043
int kvm_lock_all_vcpus(struct kvm *kvm);
include/linux/kvm_host.h
1044
void kvm_unlock_all_vcpus(struct kvm *kvm);
include/linux/kvm_host.h
1050
void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm);
include/linux/kvm_host.h
1052
static inline void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm)
include/linux/kvm_host.h
1073
void kvm_get_kvm(struct kvm *kvm);
include/linux/kvm_host.h
1074
bool kvm_get_kvm_safe(struct kvm *kvm);
include/linux/kvm_host.h
1075
void kvm_put_kvm(struct kvm *kvm);
include/linux/kvm_host.h
1077
void kvm_put_kvm_no_destroy(struct kvm *kvm);
include/linux/kvm_host.h
1079
static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id)
include/linux/kvm_host.h
1082
return srcu_dereference_check(kvm->memslots[as_id], &kvm->srcu,
include/linux/kvm_host.h
1083
lockdep_is_held(&kvm->slots_lock) ||
include/linux/kvm_host.h
1084
!refcount_read(&kvm->users_count));
include/linux/kvm_host.h
1087
static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm)
include/linux/kvm_host.h
1089
return __kvm_memslots(kvm, 0);
include/linux/kvm_host.h
1096
return __kvm_memslots(vcpu->kvm, as_id);
include/linux/kvm_host.h
1104
bool kvm_are_all_memslots_empty(struct kvm *kvm);
include/linux/kvm_host.h
1219
struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn);
include/linux/kvm_host.h
1241
int kvm_set_internal_memslot(struct kvm *kvm,
include/linux/kvm_host.h
1243
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot);
include/linux/kvm_host.h
1244
void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen);
include/linux/kvm_host.h
1245
int kvm_arch_prepare_memory_region(struct kvm *kvm,
include/linux/kvm_host.h
1249
void kvm_arch_commit_memory_region(struct kvm *kvm,
include/linux/kvm_host.h
1254
void kvm_arch_flush_shadow_all(struct kvm *kvm);
include/linux/kvm_host.h
1256
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
include/linux/kvm_host.h
1262
struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn, bool write);
include/linux/kvm_host.h
1263
static inline struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
include/linux/kvm_host.h
1265
return __gfn_to_page(kvm, gfn, true);
include/linux/kvm_host.h
1268
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
include/linux/kvm_host.h
1269
unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable);
include/linux/kvm_host.h
1285
static inline void kvm_release_faultin_page(struct kvm *kvm, struct page *page,
include/linux/kvm_host.h
1288
lockdep_assert_once(lockdep_is_held(&kvm->mmu_lock) || unused);
include/linux/kvm_host.h
1323
int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
include/linux/kvm_host.h
1325
int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len);
include/linux/kvm_host.h
1326
int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
include/linux/kvm_host.h
1328
int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
include/linux/kvm_host.h
1331
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
include/linux/kvm_host.h
1333
int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
include/linux/kvm_host.h
1335
int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
include/linux/kvm_host.h
1337
int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
include/linux/kvm_host.h
1340
int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
include/linux/kvm_host.h
1343
#define __kvm_get_guest(kvm, gfn, offset, v) \
include/linux/kvm_host.h
1345
unsigned long __addr = gfn_to_hva(kvm, gfn); \
include/linux/kvm_host.h
1354
#define kvm_get_guest(kvm, gpa, v) \
include/linux/kvm_host.h
1357
struct kvm *__kvm = kvm; \
include/linux/kvm_host.h
1363
#define __kvm_put_guest(kvm, gfn, offset, v) \
include/linux/kvm_host.h
1365
unsigned long __addr = gfn_to_hva(kvm, gfn); \
include/linux/kvm_host.h
1372
mark_page_dirty(kvm, gfn); \
include/linux/kvm_host.h
1376
#define kvm_put_guest(kvm, gpa, v) \
include/linux/kvm_host.h
1379
struct kvm *__kvm = kvm; \
include/linux/kvm_host.h
1385
int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len);
include/linux/kvm_host.h
1386
bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn);
include/linux/kvm_host.h
1389
void mark_page_dirty_in_slot(struct kvm *kvm, const struct kvm_memory_slot *memslot, gfn_t gfn);
include/linux/kvm_host.h
1390
void mark_page_dirty(struct kvm *kvm, gfn_t gfn);
include/linux/kvm_host.h
1439
void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm);
include/linux/kvm_host.h
1552
void kvm_flush_remote_tlbs(struct kvm *kvm);
include/linux/kvm_host.h
1553
void kvm_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages);
include/linux/kvm_host.h
1554
void kvm_flush_remote_tlbs_memslot(struct kvm *kvm,
include/linux/kvm_host.h
1565
void kvm_mmu_invalidate_start(struct kvm *kvm);
include/linux/kvm_host.h
1566
void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end);
include/linux/kvm_host.h
1567
void kvm_mmu_invalidate_end(struct kvm *kvm);
include/linux/kvm_host.h
1568
bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range);
include/linux/kvm_host.h
1578
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext);
include/linux/kvm_host.h
1580
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
include/linux/kvm_host.h
1584
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot);
include/linux/kvm_host.h
1587
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log);
include/linux/kvm_host.h
1588
int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log,
include/linux/kvm_host.h
1592
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
include/linux/kvm_host.h
1594
int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
include/linux/kvm_host.h
1622
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id);
include/linux/kvm_host.h
1628
int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state);
include/linux/kvm_host.h
1669
void kvm_arch_pre_destroy_vm(struct kvm *kvm);
include/linux/kvm_host.h
1670
void kvm_arch_create_vm_debugfs(struct kvm *kvm);
include/linux/kvm_host.h
1677
static inline struct kvm *kvm_arch_alloc_vm(void)
include/linux/kvm_host.h
1679
return kzalloc_obj(struct kvm, GFP_KERNEL_ACCOUNT);
include/linux/kvm_host.h
1683
static inline void __kvm_arch_free_vm(struct kvm *kvm)
include/linux/kvm_host.h
1685
kvfree(kvm);
include/linux/kvm_host.h
1689
static inline void kvm_arch_free_vm(struct kvm *kvm)
include/linux/kvm_host.h
1691
__kvm_arch_free_vm(kvm);
include/linux/kvm_host.h
1696
static inline int kvm_arch_flush_remote_tlbs(struct kvm *kvm)
include/linux/kvm_host.h
1701
int kvm_arch_flush_remote_tlbs(struct kvm *kvm);
include/linux/kvm_host.h
1705
static inline int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm,
include/linux/kvm_host.h
1711
int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages);
include/linux/kvm_host.h
1715
void kvm_arch_register_noncoherent_dma(struct kvm *kvm);
include/linux/kvm_host.h
1716
void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm);
include/linux/kvm_host.h
1717
bool kvm_arch_has_noncoherent_dma(struct kvm *kvm);
include/linux/kvm_host.h
1719
static inline void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
include/linux/kvm_host.h
1723
static inline void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
include/linux/kvm_host.h
1727
static inline bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
include/linux/kvm_host.h
1762
bool kvm_arch_intc_initialized(struct kvm *kvm);
include/linux/kvm_host.h
1764
static inline bool kvm_arch_intc_initialized(struct kvm *kvm)
include/linux/kvm_host.h
1787
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type);
include/linux/kvm_host.h
1788
void kvm_arch_destroy_vm(struct kvm *kvm);
include/linux/kvm_host.h
1798
int kvm_irq_map_gsi(struct kvm *kvm,
include/linux/kvm_host.h
1800
int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin);
include/linux/kvm_host.h
1802
int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
include/linux/kvm_host.h
1804
int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm,
include/linux/kvm_host.h
1807
struct kvm *kvm, int irq_source_id,
include/linux/kvm_host.h
1809
bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin);
include/linux/kvm_host.h
1810
void kvm_notify_acked_gsi(struct kvm *kvm, int gsi);
include/linux/kvm_host.h
1811
void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin);
include/linux/kvm_host.h
1812
void kvm_register_irq_ack_notifier(struct kvm *kvm,
include/linux/kvm_host.h
1814
void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
include/linux/kvm_host.h
1816
bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args);
include/linux/kvm_host.h
188
bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
include/linux/kvm_host.h
190
bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req);
include/linux/kvm_host.h
1911
static inline int memslot_id(struct kvm *kvm, gfn_t gfn)
include/linux/kvm_host.h
1913
return gfn_to_memslot(kvm, gfn)->id;
include/linux/kvm_host.h
1939
static inline bool kvm_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa)
include/linux/kvm_host.h
1941
unsigned long hva = gfn_to_hva(kvm, gpa_to_gfn(gpa));
include/linux/kvm_host.h
1953
mark_page_dirty_in_slot(gpc->kvm, gpc->memslot, gpa_to_gfn(gpc->gpa));
include/linux/kvm_host.h
1962
struct kvm *kvm;
include/linux/kvm_host.h
2125
static inline int mmu_invalidate_retry(struct kvm *kvm, unsigned long mmu_seq)
include/linux/kvm_host.h
2127
if (unlikely(kvm->mmu_invalidate_in_progress))
include/linux/kvm_host.h
2142
if (kvm->mmu_invalidate_seq != mmu_seq)
include/linux/kvm_host.h
2147
static inline int mmu_invalidate_retry_gfn(struct kvm *kvm,
include/linux/kvm_host.h
2151
lockdep_assert_held(&kvm->mmu_lock);
include/linux/kvm_host.h
2158
if (unlikely(kvm->mmu_invalidate_in_progress)) {
include/linux/kvm_host.h
2163
if (WARN_ON_ONCE(kvm->mmu_invalidate_range_start == INVALID_GPA ||
include/linux/kvm_host.h
2164
kvm->mmu_invalidate_range_end == INVALID_GPA))
include/linux/kvm_host.h
2167
if (gfn >= kvm->mmu_invalidate_range_start &&
include/linux/kvm_host.h
2168
gfn < kvm->mmu_invalidate_range_end)
include/linux/kvm_host.h
2172
if (kvm->mmu_invalidate_seq != mmu_seq)
include/linux/kvm_host.h
2183
static inline bool mmu_invalidate_retry_gfn_unsafe(struct kvm *kvm,
include/linux/kvm_host.h
2195
if (unlikely(READ_ONCE(kvm->mmu_invalidate_in_progress)) &&
include/linux/kvm_host.h
2196
gfn >= kvm->mmu_invalidate_range_start &&
include/linux/kvm_host.h
2197
gfn < kvm->mmu_invalidate_range_end)
include/linux/kvm_host.h
2200
return READ_ONCE(kvm->mmu_invalidate_seq) != mmu_seq;
include/linux/kvm_host.h
2207
bool kvm_arch_can_set_irq_routing(struct kvm *kvm);
include/linux/kvm_host.h
2208
int kvm_set_irq_routing(struct kvm *kvm,
include/linux/kvm_host.h
2212
int kvm_init_irq_routing(struct kvm *kvm);
include/linux/kvm_host.h
2213
int kvm_set_routing_entry(struct kvm *kvm,
include/linux/kvm_host.h
2216
void kvm_free_irq_routing(struct kvm *kvm);
include/linux/kvm_host.h
2220
static inline void kvm_free_irq_routing(struct kvm *kvm) {}
include/linux/kvm_host.h
2222
static inline int kvm_init_irq_routing(struct kvm *kvm)
include/linux/kvm_host.h
2229
int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi);
include/linux/kvm_host.h
2231
void kvm_eventfd_init(struct kvm *kvm);
include/linux/kvm_host.h
2232
int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args);
include/linux/kvm_host.h
2235
int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args);
include/linux/kvm_host.h
2236
void kvm_irqfd_release(struct kvm *kvm);
include/linux/kvm_host.h
2237
bool kvm_notify_irqfd_resampler(struct kvm *kvm,
include/linux/kvm_host.h
2240
void kvm_irq_routing_update(struct kvm *);
include/linux/kvm_host.h
2242
static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
include/linux/kvm_host.h
2247
static inline void kvm_irqfd_release(struct kvm *kvm) {}
include/linux/kvm_host.h
2249
static inline bool kvm_notify_irqfd_resampler(struct kvm *kvm,
include/linux/kvm_host.h
2257
void kvm_arch_irq_routing_update(struct kvm *kvm);
include/linux/kvm_host.h
229
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
include/linux/kvm_host.h
231
int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
include/linux/kvm_host.h
233
struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
include/linux/kvm_host.h
2332
struct kvm *kvm;
include/linux/kvm_host.h
2460
void kvm_arch_guest_memory_reclaimed(struct kvm *kvm);
include/linux/kvm_host.h
2537
static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn)
include/linux/kvm_host.h
2539
return xa_to_value(xa_load(&kvm->mem_attr_array, gfn));
include/linux/kvm_host.h
2542
bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
include/linux/kvm_host.h
2544
bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,
include/linux/kvm_host.h
2546
bool kvm_arch_post_set_memory_attributes(struct kvm *kvm,
include/linux/kvm_host.h
2549
static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn)
include/linux/kvm_host.h
2551
return kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE;
include/linux/kvm_host.h
2554
static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn)
include/linux/kvm_host.h
2561
int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
include/linux/kvm_host.h
2565
static inline int kvm_gmem_get_pfn(struct kvm *kvm,
include/linux/kvm_host.h
2570
KVM_BUG_ON(1, kvm);
include/linux/kvm_host.h
2576
int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order);
include/linux/kvm_host.h
2601
typedef int (*kvm_gmem_populate_cb)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
include/linux/kvm_host.h
2604
long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src,
include/linux/kvm_host.h
274
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range);
include/linux/kvm_host.h
275
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range);
include/linux/kvm_host.h
276
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range);
include/linux/kvm_host.h
326
struct kvm *kvm;
include/linux/kvm_host.h
671
struct kvm *kvm, int irq_source_id, int level,
include/linux/kvm_host.h
704
bool kvm_arch_irqchip_in_kernel(struct kvm *kvm);
include/linux/kvm_host.h
714
static inline int kvm_arch_nr_memslot_as_ids(struct kvm *kvm)
include/linux/kvm_host.h
726
static inline bool kvm_arch_has_private_mem(struct kvm *kvm)
include/linux/kvm_host.h
733
bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm);
include/linux/kvm_host.h
735
static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm)
include/linux/kvm_host.h
739
if (!kvm || kvm_arch_supports_gmem_init_shared(kvm))
include/linux/kvm_host.h
747
static inline bool kvm_arch_has_readonly_mem(struct kvm *kvm)
include/linux/kvm_host.h
907
static inline void kvm_vm_dead(struct kvm *kvm)
include/linux/kvm_host.h
909
kvm->vm_dead = true;
include/linux/kvm_host.h
910
kvm_make_all_cpus_request(kvm, KVM_REQ_VM_DEAD);
include/linux/kvm_host.h
913
static inline void kvm_vm_bugged(struct kvm *kvm)
include/linux/kvm_host.h
915
kvm->vm_bugged = true;
include/linux/kvm_host.h
916
kvm_vm_dead(kvm);
include/linux/kvm_host.h
920
#define KVM_BUG(cond, kvm, fmt...) \
include/linux/kvm_host.h
924
if (WARN_ONCE(__ret && !(kvm)->vm_bugged, fmt)) \
include/linux/kvm_host.h
925
kvm_vm_bugged(kvm); \
include/linux/kvm_host.h
929
#define KVM_BUG_ON(cond, kvm) \
include/linux/kvm_host.h
933
if (WARN_ON_ONCE(__ret && !(kvm)->vm_bugged)) \
include/linux/kvm_host.h
934
kvm_vm_bugged(kvm); \
include/linux/kvm_host.h
946
#define KVM_BUG_ON_DATA_CORRUPTION(cond, kvm) \
include/linux/kvm_host.h
952
else if (WARN_ON_ONCE(__ret && !(kvm)->vm_bugged)) \
include/linux/kvm_host.h
953
kvm_vm_bugged(kvm); \
include/linux/kvm_host.h
963
vcpu->____srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
include/linux/kvm_host.h
968
srcu_read_unlock(&vcpu->kvm->srcu, vcpu->____srcu_idx);
include/linux/kvm_host.h
976
static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm)
include/linux/kvm_host.h
978
return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET);
include/linux/kvm_host.h
986
static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx)
include/linux/kvm_host.h
988
return rcu_dereference_protected(kvm->buses[idx],
include/linux/kvm_host.h
989
lockdep_is_held(&kvm->slots_lock));
include/linux/kvm_host.h
992
static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
include/linux/kvm_host.h
994
int num_vcpus = atomic_read(&kvm->online_vcpus);
include/linux/kvm_irqfd.h
24
struct kvm *kvm;
include/linux/kvm_irqfd.h
41
struct kvm *kvm;
include/linux/kvm_types.h
37
struct kvm;
include/linux/kvm_types.h
89
struct kvm *kvm;
include/linux/pci-tsm.h
10
struct kvm;
include/linux/pci-tsm.h
212
int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u32 tdi_id);
include/linux/pci-tsm.h
215
struct kvm *kvm, u32 tdi_id);
include/linux/pci-tsm.h
227
static inline int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u64 tdi_id)
include/linux/pci-tsm.h
54
struct kvm *kvm, u32 tdi_id);
include/linux/pci-tsm.h
88
struct kvm *kvm;
include/linux/vfio.h
21
struct kvm;
include/linux/vfio.h
380
void vfio_file_set_kvm(struct file *file, struct kvm *kvm);
include/linux/vfio.h
57
struct kvm *kvm;
include/linux/vfio.h
69
void (*put_kvm)(struct kvm *kvm);
include/trace/events/kvm.h
8
#define TRACE_SYSTEM kvm
tools/perf/builtin-kvm.c
1004
static void sort_result(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
1010
kvm_hists__reinit(output_columns, kvm->sort_key);
tools/perf/builtin-kvm.c
1017
static void print_vcpu_info(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
1019
int vcpu = kvm->trace_vcpu;
tools/perf/builtin-kvm.c
1023
if (kvm->opts.target.system_wide)
tools/perf/builtin-kvm.c
1025
else if (kvm->opts.target.pid)
tools/perf/builtin-kvm.c
1026
pr_info("pid(s) %s, ", kvm->opts.target.pid);
tools/perf/builtin-kvm.c
1052
static void print_result(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
1056
int vcpu = kvm->trace_vcpu;
tools/perf/builtin-kvm.c
1059
if (kvm->live) {
tools/perf/builtin-kvm.c
1065
print_vcpu_info(kvm);
tools/perf/builtin-kvm.c
1066
pr_info("%*s ", KVM_EVENT_NAME_LEN, kvm->events_ops->name);
tools/perf/builtin-kvm.c
1090
kvm->events_ops->decode_key(kvm, &event->key, decode);
tools/perf/builtin-kvm.c
1093
pr_info("%8.2f%% ", (double)ecount / kvm->total_count * 100);
tools/perf/builtin-kvm.c
1094
pr_info("%8.2f%% ", (double)etime / kvm->total_time * 100);
tools/perf/builtin-kvm.c
1103
kvm->total_count, kvm->total_time / (double)NSEC_PER_USEC);
tools/perf/builtin-kvm.c
1105
if (kvm->lost_events)
tools/perf/builtin-kvm.c
1106
pr_info("\nLost events: %" PRIu64 "\n\n", kvm->lost_events);
tools/perf/builtin-kvm.c
1115
struct perf_kvm_stat *kvm = container_of(tool, struct perf_kvm_stat, tool);
tools/perf/builtin-kvm.c
1117
kvm->lost_events++;
tools/perf/builtin-kvm.c
1122
static bool skip_sample(struct perf_kvm_stat *kvm,
tools/perf/builtin-kvm.c
1125
if (kvm->pid_list && intlist__find(kvm->pid_list, sample->pid) == NULL)
tools/perf/builtin-kvm.c
1138
struct perf_kvm_stat *kvm = container_of(tool, struct perf_kvm_stat,
tools/perf/builtin-kvm.c
1141
if (skip_sample(kvm, sample))
tools/perf/builtin-kvm.c
1144
if (machine__resolve(machine, &kvm->al, sample) < 0) {
tools/perf/builtin-kvm.c
1158
if (!handle_kvm_event(kvm, thread, sample))
tools/perf/builtin-kvm.c
1165
static int cpu_isa_config(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
1171
if (kvm->live) {
tools/perf/builtin-kvm.c
1182
cpuid = perf_session__env(kvm->session)->cpuid;
tools/perf/builtin-kvm.c
1189
e_machine = perf_session__e_machine(kvm->session, /*e_flags=*/NULL);
tools/perf/builtin-kvm.c
1190
err = cpu_isa_init(kvm, e_machine, cpuid);
tools/perf/builtin-kvm.c
1213
static s64 perf_kvm__mmap_read_idx(struct perf_kvm_stat *kvm, int idx,
tools/perf/builtin-kvm.c
1216
struct evlist *evlist = kvm->evlist;
tools/perf/builtin-kvm.c
1237
err = perf_session__queue_event(kvm->session, event, timestamp, 0, NULL);
tools/perf/builtin-kvm.c
1263
static int perf_kvm__mmap_read(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
1269
for (i = 0; i < kvm->evlist->core.nr_mmaps; i++) {
tools/perf/builtin-kvm.c
1270
n = perf_kvm__mmap_read_idx(kvm, i, &mmap_time);
tools/perf/builtin-kvm.c
1291
struct ordered_events *oe = &kvm->session->ordered_events;
tools/perf/builtin-kvm.c
1296
if (kvm->lost_events)
tools/perf/builtin-kvm.c
1298
kvm->lost_events);
tools/perf/builtin-kvm.c
1313
static int perf_kvm__timerfd_create(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
1318
kvm->timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK);
tools/perf/builtin-kvm.c
1319
if (kvm->timerfd < 0) {
tools/perf/builtin-kvm.c
1324
new_value.it_value.tv_sec = kvm->display_time;
tools/perf/builtin-kvm.c
1326
new_value.it_interval.tv_sec = kvm->display_time;
tools/perf/builtin-kvm.c
1329
if (timerfd_settime(kvm->timerfd, 0, &new_value, NULL) != 0) {
tools/perf/builtin-kvm.c
1331
close(kvm->timerfd);
tools/perf/builtin-kvm.c
1340
static int perf_kvm__handle_timerfd(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
1345
rc = read(kvm->timerfd, &c, sizeof(uint64_t));
tools/perf/builtin-kvm.c
1363
sort_result(kvm);
tools/perf/builtin-kvm.c
1364
print_result(kvm);
tools/perf/builtin-kvm.c
1371
kvm->total_count = 0;
tools/perf/builtin-kvm.c
1372
kvm->total_time = 0;
tools/perf/builtin-kvm.c
1373
kvm->lost_events = 0;
tools/perf/builtin-kvm.c
1407
static int kvm_events_live_report(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
1413
kvm->live = true;
tools/perf/builtin-kvm.c
1415
ret = cpu_isa_config(kvm);
tools/perf/builtin-kvm.c
1419
if (!verify_vcpu(kvm->trace_vcpu) ||
tools/perf/builtin-kvm.c
1420
!is_valid_key(kvm) ||
tools/perf/builtin-kvm.c
1421
!register_kvm_events_ops(kvm, EM_HOST)) {
tools/perf/builtin-kvm.c
1433
if (perf_kvm__timerfd_create(kvm) < 0) {
tools/perf/builtin-kvm.c
1438
if (evlist__add_pollfd(kvm->evlist, kvm->timerfd) < 0)
tools/perf/builtin-kvm.c
1441
nr_stdin = evlist__add_pollfd(kvm->evlist, fileno(stdin));
tools/perf/builtin-kvm.c
1449
evlist__enable(kvm->evlist);
tools/perf/builtin-kvm.c
1452
struct fdarray *fda = &kvm->evlist->core.pollfd;
tools/perf/builtin-kvm.c
1455
rc = perf_kvm__mmap_read(kvm);
tools/perf/builtin-kvm.c
1459
err = perf_kvm__handle_timerfd(kvm);
tools/perf/builtin-kvm.c
1467
err = evlist__poll(kvm->evlist, 100);
tools/perf/builtin-kvm.c
1470
evlist__disable(kvm->evlist);
tools/perf/builtin-kvm.c
1473
sort_result(kvm);
tools/perf/builtin-kvm.c
1474
print_result(kvm);
tools/perf/builtin-kvm.c
1480
if (kvm->timerfd >= 0)
tools/perf/builtin-kvm.c
1481
close(kvm->timerfd);
tools/perf/builtin-kvm.c
1487
static int kvm_live_open_events(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
1491
struct evlist *evlist = kvm->evlist;
tools/perf/builtin-kvm.c
1494
evlist__config(evlist, &kvm->opts, NULL);
tools/perf/builtin-kvm.c
1534
if (evlist__mmap(evlist, kvm->opts.mmap_pages) < 0) {
tools/perf/builtin-kvm.c
1548
static int read_events(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
1553
.path = kvm->file_name,
tools/perf/builtin-kvm.c
1555
.force = kvm->force,
tools/perf/builtin-kvm.c
1558
perf_tool__init(&kvm->tool, /*ordered_events=*/true);
tools/perf/builtin-kvm.c
1559
kvm->tool.sample = process_sample_event;
tools/perf/builtin-kvm.c
1560
kvm->tool.comm = perf_event__process_comm;
tools/perf/builtin-kvm.c
1561
kvm->tool.namespaces = perf_event__process_namespaces;
tools/perf/builtin-kvm.c
1563
kvm->session = perf_session__new(&file, &kvm->tool);
tools/perf/builtin-kvm.c
1564
if (IS_ERR(kvm->session)) {
tools/perf/builtin-kvm.c
1566
return PTR_ERR(kvm->session);
tools/perf/builtin-kvm.c
1569
symbol__init(perf_session__env(kvm->session));
tools/perf/builtin-kvm.c
1571
if (!perf_session__has_traces(kvm->session, "kvm record")) {
tools/perf/builtin-kvm.c
1576
e_machine = perf_session__e_machine(kvm->session, /*e_flags=*/NULL);
tools/perf/builtin-kvm.c
1577
if (!register_kvm_events_ops(kvm, e_machine)) {
tools/perf/builtin-kvm.c
1586
ret = cpu_isa_config(kvm);
tools/perf/builtin-kvm.c
1590
ret = perf_session__process_events(kvm->session);
tools/perf/builtin-kvm.c
1593
perf_session__delete(kvm->session);
tools/perf/builtin-kvm.c
1597
static int parse_target_str(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
1599
if (kvm->opts.target.pid) {
tools/perf/builtin-kvm.c
1600
kvm->pid_list = intlist__new(kvm->opts.target.pid);
tools/perf/builtin-kvm.c
1601
if (kvm->pid_list == NULL) {
tools/perf/builtin-kvm.c
1610
static int kvm_events_report_vcpu(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
1613
int vcpu = kvm->trace_vcpu;
tools/perf/builtin-kvm.c
1615
if (parse_target_str(kvm) != 0)
tools/perf/builtin-kvm.c
1621
if (!is_valid_key(kvm))
tools/perf/builtin-kvm.c
1624
if (kvm->use_stdio) {
tools/perf/builtin-kvm.c
1635
ret = read_events(kvm);
tools/perf/builtin-kvm.c
1639
sort_result(kvm);
tools/perf/builtin-kvm.c
1640
kvm_display(kvm);
tools/perf/builtin-kvm.c
1648
kvm_events_record(struct perf_kvm_stat *kvm, int argc, const char **argv)
tools/perf/builtin-kvm.c
1667
ret = setup_kvm_events_tp(kvm, e_machine);
tools/perf/builtin-kvm.c
1692
rec_argv[i++] = STRDUP_FAIL_EXIT(kvm->file_name);
tools/perf/builtin-kvm.c
1728
kvm_events_report(struct perf_kvm_stat *kvm, int argc, const char **argv)
tools/perf/builtin-kvm.c
1731
OPT_STRING(0, "event", &kvm->report_event, "report event",
tools/perf/builtin-kvm.c
1734
OPT_INTEGER(0, "vcpu", &kvm->trace_vcpu,
tools/perf/builtin-kvm.c
1736
OPT_STRING('k', "key", &kvm->sort_key, "sort-key",
tools/perf/builtin-kvm.c
1739
OPT_STRING('p', "pid", &kvm->opts.target.pid, "pid",
tools/perf/builtin-kvm.c
1741
OPT_BOOLEAN('f', "force", &kvm->force, "don't complain, do it"),
tools/perf/builtin-kvm.c
1742
OPT_BOOLEAN(0, "stdio", &kvm->use_stdio, "use the stdio interface"),
tools/perf/builtin-kvm.c
1761
kvm->use_stdio = true;
tools/perf/builtin-kvm.c
1764
if (!kvm->opts.target.pid)
tools/perf/builtin-kvm.c
1765
kvm->opts.target.system_wide = true;
tools/perf/builtin-kvm.c
1767
return kvm_events_report_vcpu(kvm);
tools/perf/builtin-kvm.c
1820
static int kvm_events_live(struct perf_kvm_stat *kvm,
tools/perf/builtin-kvm.c
1827
OPT_STRING('p', "pid", &kvm->opts.target.pid, "pid",
tools/perf/builtin-kvm.c
1829
OPT_CALLBACK('m', "mmap-pages", &kvm->opts.mmap_pages, "pages",
tools/perf/builtin-kvm.c
1833
OPT_BOOLEAN('a', "all-cpus", &kvm->opts.target.system_wide,
tools/perf/builtin-kvm.c
1835
OPT_UINTEGER('d', "display", &kvm->display_time,
tools/perf/builtin-kvm.c
1837
OPT_STRING(0, "event", &kvm->report_event, "report event",
tools/perf/builtin-kvm.c
1840
OPT_INTEGER(0, "vcpu", &kvm->trace_vcpu,
tools/perf/builtin-kvm.c
1842
OPT_STRING('k', "key", &kvm->sort_key, "sort-key",
tools/perf/builtin-kvm.c
1845
OPT_U64(0, "duration", &kvm->duration,
tools/perf/builtin-kvm.c
1863
perf_tool__init(&kvm->tool, /*ordered_events=*/true);
tools/perf/builtin-kvm.c
1864
kvm->tool.sample = process_sample_event;
tools/perf/builtin-kvm.c
1865
kvm->tool.comm = perf_event__process_comm;
tools/perf/builtin-kvm.c
1866
kvm->tool.exit = perf_event__process_exit;
tools/perf/builtin-kvm.c
1867
kvm->tool.fork = perf_event__process_fork;
tools/perf/builtin-kvm.c
1868
kvm->tool.lost = process_lost_event;
tools/perf/builtin-kvm.c
1869
kvm->tool.namespaces = perf_event__process_namespaces;
tools/perf/builtin-kvm.c
1872
kvm->display_time = 1;
tools/perf/builtin-kvm.c
1873
kvm->opts.user_interval = 1;
tools/perf/builtin-kvm.c
1874
kvm->opts.mmap_pages = 512;
tools/perf/builtin-kvm.c
1875
kvm->opts.target.uses_mmap = false;
tools/perf/builtin-kvm.c
1889
kvm->duration *= NSEC_PER_USEC; /* convert usec to nsec */
tools/perf/builtin-kvm.c
1894
err = target__validate(&kvm->opts.target);
tools/perf/builtin-kvm.c
1896
target__strerror(&kvm->opts.target, err, errbuf, BUFSIZ);
tools/perf/builtin-kvm.c
1900
if (target__none(&kvm->opts.target))
tools/perf/builtin-kvm.c
1901
kvm->opts.target.system_wide = true;
tools/perf/builtin-kvm.c
1907
err = setup_kvm_events_tp(kvm, EM_HOST);
tools/perf/builtin-kvm.c
1913
kvm->evlist = kvm_live_event_list();
tools/perf/builtin-kvm.c
1914
if (kvm->evlist == NULL) {
tools/perf/builtin-kvm.c
1919
if (evlist__create_maps(kvm->evlist, &kvm->opts.target) < 0)
tools/perf/builtin-kvm.c
1925
kvm->session = perf_session__new(&data, &kvm->tool);
tools/perf/builtin-kvm.c
1926
if (IS_ERR(kvm->session)) {
tools/perf/builtin-kvm.c
1927
err = PTR_ERR(kvm->session);
tools/perf/builtin-kvm.c
1930
kvm->session->evlist = kvm->evlist;
tools/perf/builtin-kvm.c
1931
perf_session__set_id_hdr_size(kvm->session);
tools/perf/builtin-kvm.c
1932
ordered_events__set_copy_on_queue(&kvm->session->ordered_events, true);
tools/perf/builtin-kvm.c
1933
machine__synthesize_threads(&kvm->session->machines.host, &kvm->opts.target,
tools/perf/builtin-kvm.c
1934
kvm->evlist->core.threads, true, false, 1);
tools/perf/builtin-kvm.c
1935
err = kvm_live_open_events(kvm);
tools/perf/builtin-kvm.c
1939
err = kvm_events_live_report(kvm);
tools/perf/builtin-kvm.c
1942
perf_session__delete(kvm->session);
tools/perf/builtin-kvm.c
1943
kvm->session = NULL;
tools/perf/builtin-kvm.c
1944
evlist__delete(kvm->evlist);
tools/perf/builtin-kvm.c
1964
struct perf_kvm_stat kvm = {
tools/perf/builtin-kvm.c
1979
return kvm_events_record(&kvm, argc - 1, argv + 1);
tools/perf/builtin-kvm.c
1982
return kvm_events_report(&kvm, argc - 1 , argv + 1);
tools/perf/builtin-kvm.c
1986
return kvm_events_live(&kvm, argc - 1 , argv + 1);
tools/perf/builtin-kvm.c
515
static void print_result(struct perf_kvm_stat *kvm);
tools/perf/builtin-kvm.c
584
static void kvm_display(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
587
print_result(kvm);
tools/perf/builtin-kvm.c
594
static void kvm_display(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
597
print_result(kvm);
tools/perf/builtin-kvm.c
620
static bool register_kvm_events_ops(struct perf_kvm_stat *kvm, uint16_t e_machine)
tools/perf/builtin-kvm.c
625
if (!strcmp(events_ops->name, kvm->report_event)) {
tools/perf/builtin-kvm.c
626
kvm->events_ops = events_ops->ops;
tools/perf/builtin-kvm.c
722
static struct kvm_event *find_create_kvm_event(struct perf_kvm_stat *kvm,
tools/perf/builtin-kvm.c
738
kvm->events_ops->decode_key(kvm, key, ki->name);
tools/perf/builtin-kvm.c
740
&kvm->al, NULL, NULL, NULL, ki, sample, true);
tools/perf/builtin-kvm.c
749
event->perf_kvm = kvm;
tools/perf/builtin-kvm.c
756
static bool handle_begin_event(struct perf_kvm_stat *kvm,
tools/perf/builtin-kvm.c
764
event = find_create_kvm_event(kvm, key, sample);
tools/perf/builtin-kvm.c
789
static bool update_kvm_event(struct perf_kvm_stat *kvm,
tools/perf/builtin-kvm.c
794
kvm->total_count++;
tools/perf/builtin-kvm.c
795
kvm->total_time += time_diff;
tools/perf/builtin-kvm.c
809
static bool is_child_event(struct perf_kvm_stat *kvm,
tools/perf/builtin-kvm.c
815
child_ops = kvm->events_ops->child_ops;
tools/perf/builtin-kvm.c
830
static bool handle_child_event(struct perf_kvm_stat *kvm,
tools/perf/builtin-kvm.c
838
event = find_create_kvm_event(kvm, key, sample);
tools/perf/builtin-kvm.c
856
static bool handle_end_event(struct perf_kvm_stat *kvm,
tools/perf/builtin-kvm.c
865
if (kvm->trace_vcpu == -1)
tools/perf/builtin-kvm.c
887
event = find_create_kvm_event(kvm, key, sample);
tools/perf/builtin-kvm.c
903
if (kvm->duration && time_diff > kvm->duration) {
tools/perf/builtin-kvm.c
905
uint16_t e_machine = perf_session__e_machine(kvm->session, /*e_flags=*/NULL);
tools/perf/builtin-kvm.c
907
kvm->events_ops->decode_key(kvm, &event->key, decode);
tools/perf/builtin-kvm.c
915
return update_kvm_event(kvm, event, vcpu, time_diff);
tools/perf/builtin-kvm.c
941
static bool handle_kvm_event(struct perf_kvm_stat *kvm,
tools/perf/builtin-kvm.c
947
.exit_reasons = kvm->exit_reasons };
tools/perf/builtin-kvm.c
954
if ((kvm->trace_vcpu != -1) &&
tools/perf/builtin-kvm.c
955
(kvm->trace_vcpu != vcpu_record->vcpu_id))
tools/perf/builtin-kvm.c
958
if (kvm->events_ops->is_begin_event(sample, &key))
tools/perf/builtin-kvm.c
959
return handle_begin_event(kvm, vcpu_record, &key, sample);
tools/perf/builtin-kvm.c
961
if (is_child_event(kvm, sample, &key))
tools/perf/builtin-kvm.c
962
return handle_child_event(kvm, vcpu_record, &key, sample);
tools/perf/builtin-kvm.c
964
if (kvm->events_ops->is_end_event(sample, &key))
tools/perf/builtin-kvm.c
965
return handle_end_event(kvm, vcpu_record, &key, sample);
tools/perf/builtin-kvm.c
970
static bool is_valid_key(struct perf_kvm_stat *kvm)
tools/perf/builtin-kvm.c
978
if (!strcmp(key_array[i], kvm->sort_key))
tools/perf/builtin-kvm.c
981
pr_err("Unsupported sort key: %s\n", kvm->sort_key);
tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c
73
int __cpu_isa_init_arm64(struct perf_kvm_stat *kvm)
tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c
75
kvm->exit_reasons_isa = "arm64";
tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c
129
int __cpu_isa_init_loongarch(struct perf_kvm_stat *kvm)
tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c
131
kvm->exit_reasons_isa = "loongarch64";
tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c
132
kvm->exit_reasons = loongarch_exit_reasons;
tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
115
static int ppc__setup_book3s_hv(struct perf_kvm_stat *kvm,
tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
133
kvm->exit_reasons = hv_exit_reasons;
tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
134
kvm->exit_reasons_isa = "HV";
tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
140
static int ppc__setup_kvm_tp(struct perf_kvm_stat *kvm)
tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
148
return ppc__setup_book3s_hv(kvm, evlist);
tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
151
int __setup_kvm_events_tp_powerpc(struct perf_kvm_stat *kvm)
tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
153
return ppc__setup_kvm_tp(kvm);
tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
156
int __cpu_isa_init_powerpc(struct perf_kvm_stat *kvm)
tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
160
ret = ppc__setup_kvm_tp(kvm);
tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
162
kvm->exit_reasons = NULL;
tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
163
kvm->exit_reasons_isa = NULL;
tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
68
static void hcall_event_decode_key(struct perf_kvm_stat *kvm __maybe_unused,
tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c
68
int __cpu_isa_init_riscv(struct perf_kvm_stat *kvm)
tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c
70
kvm->exit_reasons_isa = "riscv64";
tools/perf/util/kvm-stat-arch/kvm-stat-s390.c
92
int __cpu_isa_init_s390(struct perf_kvm_stat *kvm, const char *cpuid)
tools/perf/util/kvm-stat-arch/kvm-stat-s390.c
95
kvm->exit_reasons = sie_exit_reasons;
tools/perf/util/kvm-stat-arch/kvm-stat-s390.c
96
kvm->exit_reasons_isa = "SIE";
tools/perf/util/kvm-stat-arch/kvm-stat-x86.c
110
static void ioport_event_decode_key(struct perf_kvm_stat *kvm __maybe_unused,
tools/perf/util/kvm-stat-arch/kvm-stat-x86.c
148
static void msr_event_decode_key(struct perf_kvm_stat *kvm __maybe_unused,
tools/perf/util/kvm-stat-arch/kvm-stat-x86.c
186
int __cpu_isa_init_x86(struct perf_kvm_stat *kvm, const char *cpuid)
tools/perf/util/kvm-stat-arch/kvm-stat-x86.c
189
kvm->exit_reasons = vmx_exit_reasons;
tools/perf/util/kvm-stat-arch/kvm-stat-x86.c
190
kvm->exit_reasons_isa = "VMX";
tools/perf/util/kvm-stat-arch/kvm-stat-x86.c
192
kvm->exit_reasons = svm_exit_reasons;
tools/perf/util/kvm-stat-arch/kvm-stat-x86.c
193
kvm->exit_reasons_isa = "SVM";
tools/perf/util/kvm-stat-arch/kvm-stat-x86.c
70
static void mmio_event_decode_key(struct perf_kvm_stat *kvm __maybe_unused,
tools/perf/util/kvm-stat.c
47
static const char *get_exit_reason(struct perf_kvm_stat *kvm,
tools/perf/util/kvm-stat.c
58
(unsigned long long)exit_code, kvm->exit_reasons_isa);
tools/perf/util/kvm-stat.c
62
void exit_event_decode_key(struct perf_kvm_stat *kvm,
tools/perf/util/kvm-stat.c
66
const char *exit_reason = get_exit_reason(kvm, key->exit_reasons,
tools/perf/util/kvm-stat.c
72
int setup_kvm_events_tp(struct perf_kvm_stat *kvm, uint16_t e_machine)
tools/perf/util/kvm-stat.c
77
return __setup_kvm_events_tp_powerpc(kvm);
tools/perf/util/kvm-stat.c
83
int cpu_isa_init(struct perf_kvm_stat *kvm, uint16_t e_machine, const char *cpuid)
tools/perf/util/kvm-stat.c
87
return __cpu_isa_init_arm64(kvm);
tools/perf/util/kvm-stat.c
89
return __cpu_isa_init_loongarch(kvm);
tools/perf/util/kvm-stat.c
92
return __cpu_isa_init_powerpc(kvm);
tools/perf/util/kvm-stat.c
94
return __cpu_isa_init_riscv(kvm);
tools/perf/util/kvm-stat.c
96
return __cpu_isa_init_s390(kvm, cpuid);
tools/perf/util/kvm-stat.c
99
return __cpu_isa_init_x86(kvm, cpuid);
tools/perf/util/kvm-stat.h
122
void exit_event_decode_key(struct perf_kvm_stat *kvm,
tools/perf/util/kvm-stat.h
137
int setup_kvm_events_tp(struct perf_kvm_stat *kvm, uint16_t e_machine);
tools/perf/util/kvm-stat.h
138
int __setup_kvm_events_tp_powerpc(struct perf_kvm_stat *kvm);
tools/perf/util/kvm-stat.h
140
int cpu_isa_init(struct perf_kvm_stat *kvm, uint16_t e_machine, const char *cpuid);
tools/perf/util/kvm-stat.h
141
int __cpu_isa_init_arm64(struct perf_kvm_stat *kvm);
tools/perf/util/kvm-stat.h
142
int __cpu_isa_init_loongarch(struct perf_kvm_stat *kvm);
tools/perf/util/kvm-stat.h
143
int __cpu_isa_init_powerpc(struct perf_kvm_stat *kvm);
tools/perf/util/kvm-stat.h
144
int __cpu_isa_init_riscv(struct perf_kvm_stat *kvm);
tools/perf/util/kvm-stat.h
145
int __cpu_isa_init_s390(struct perf_kvm_stat *kvm, const char *cpuid);
tools/perf/util/kvm-stat.h
146
int __cpu_isa_init_x86(struct perf_kvm_stat *kvm, const char *cpuid);
tools/perf/util/kvm-stat.h
66
void (*decode_key)(struct perf_kvm_stat *kvm, struct event_key *key,
virt/kvm/async_pf.c
50
struct mm_struct *mm = vcpu->kvm->mm;
virt/kvm/coalesced_mmio.c
103
kvm->coalesced_mmio_ring = page_address(page);
virt/kvm/coalesced_mmio.c
110
spin_lock_init(&kvm->ring_lock);
virt/kvm/coalesced_mmio.c
111
INIT_LIST_HEAD(&kvm->coalesced_zones);
virt/kvm/coalesced_mmio.c
116
void kvm_coalesced_mmio_free(struct kvm *kvm)
virt/kvm/coalesced_mmio.c
118
if (kvm->coalesced_mmio_ring)
virt/kvm/coalesced_mmio.c
119
free_page((unsigned long)kvm->coalesced_mmio_ring);
virt/kvm/coalesced_mmio.c
122
int kvm_vm_ioctl_register_coalesced_mmio(struct kvm *kvm,
virt/kvm/coalesced_mmio.c
136
dev->kvm = kvm;
virt/kvm/coalesced_mmio.c
139
mutex_lock(&kvm->slots_lock);
virt/kvm/coalesced_mmio.c
140
ret = kvm_io_bus_register_dev(kvm,
virt/kvm/coalesced_mmio.c
145
list_add_tail(&dev->list, &kvm->coalesced_zones);
virt/kvm/coalesced_mmio.c
146
mutex_unlock(&kvm->slots_lock);
virt/kvm/coalesced_mmio.c
151
mutex_unlock(&kvm->slots_lock);
virt/kvm/coalesced_mmio.c
157
int kvm_vm_ioctl_unregister_coalesced_mmio(struct kvm *kvm,
virt/kvm/coalesced_mmio.c
166
mutex_lock(&kvm->slots_lock);
virt/kvm/coalesced_mmio.c
168
list_for_each_entry_safe(dev, tmp, &kvm->coalesced_zones, list) {
virt/kvm/coalesced_mmio.c
171
r = kvm_io_bus_unregister_dev(kvm,
virt/kvm/coalesced_mmio.c
183
mutex_unlock(&kvm->slots_lock);
virt/kvm/coalesced_mmio.c
48
struct kvm_coalesced_mmio_ring *ring = dev->kvm->coalesced_mmio_ring;
virt/kvm/coalesced_mmio.c
54
spin_lock(&dev->kvm->ring_lock);
virt/kvm/coalesced_mmio.c
65
spin_unlock(&dev->kvm->ring_lock);
virt/kvm/coalesced_mmio.c
77
spin_unlock(&dev->kvm->ring_lock);
virt/kvm/coalesced_mmio.c
95
int kvm_coalesced_mmio_init(struct kvm *kvm)
virt/kvm/coalesced_mmio.h
21
struct kvm *kvm;
virt/kvm/coalesced_mmio.h
25
int kvm_coalesced_mmio_init(struct kvm *kvm);
virt/kvm/coalesced_mmio.h
26
void kvm_coalesced_mmio_free(struct kvm *kvm);
virt/kvm/coalesced_mmio.h
27
int kvm_vm_ioctl_register_coalesced_mmio(struct kvm *kvm,
virt/kvm/coalesced_mmio.h
29
int kvm_vm_ioctl_unregister_coalesced_mmio(struct kvm *kvm,
virt/kvm/coalesced_mmio.h
34
static inline int kvm_coalesced_mmio_init(struct kvm *kvm) { return 0; }
virt/kvm/coalesced_mmio.h
35
static inline void kvm_coalesced_mmio_free(struct kvm *kvm) { }
virt/kvm/dirty_ring.c
106
int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring,
virt/kvm/dirty_ring.c
132
lockdep_assert_held(&kvm->slots_lock);
virt/kvm/dirty_ring.c
14
int __weak kvm_cpu_dirty_log_size(struct kvm *kvm)
virt/kvm/dirty_ring.c
186
kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
virt/kvm/dirty_ring.c
19
u32 kvm_dirty_ring_get_rsvd_entries(struct kvm *kvm)
virt/kvm/dirty_ring.c
207
kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
virt/kvm/dirty_ring.c
21
return KVM_DIRTY_RING_RSVD_ENTRIES + kvm_cpu_dirty_log_size(kvm);
virt/kvm/dirty_ring.c
24
bool kvm_use_dirty_bitmap(struct kvm *kvm)
virt/kvm/dirty_ring.c
26
lockdep_assert_held(&kvm->slots_lock);
virt/kvm/dirty_ring.c
28
return !kvm->dirty_ring_size || kvm->dirty_ring_with_bitmap;
virt/kvm/dirty_ring.c
32
bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm)
virt/kvm/dirty_ring.c
53
static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask)
virt/kvm/dirty_ring.c
61
if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
virt/kvm/dirty_ring.c
64
memslot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
virt/kvm/dirty_ring.c
70
KVM_MMU_LOCK(kvm);
virt/kvm/dirty_ring.c
71
kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, mask);
virt/kvm/dirty_ring.c
72
KVM_MMU_UNLOCK(kvm);
virt/kvm/dirty_ring.c
75
int kvm_dirty_ring_alloc(struct kvm *kvm, struct kvm_dirty_ring *ring,
virt/kvm/dirty_ring.c
83
ring->soft_limit = ring->size - kvm_dirty_ring_get_rsvd_entries(kvm);
virt/kvm/eventfd.c
1010
ret = kvm_assign_ioeventfd_idx(kvm, bus_idx, args);
virt/kvm/eventfd.c
1018
ret = kvm_assign_ioeventfd_idx(kvm, KVM_FAST_MMIO_BUS, args);
virt/kvm/eventfd.c
1026
kvm_deassign_ioeventfd_idx(kvm, bus_idx, args);
virt/kvm/eventfd.c
1032
kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
virt/kvm/eventfd.c
1035
return kvm_deassign_ioeventfd(kvm, args);
virt/kvm/eventfd.c
1037
return kvm_assign_ioeventfd(kvm, args);
virt/kvm/eventfd.c
104
kvm_unregister_irq_ack_notifier(kvm, &resampler->notifier);
virt/kvm/eventfd.c
1041
kvm_eventfd_init(struct kvm *kvm)
virt/kvm/eventfd.c
1044
spin_lock_init(&kvm->irqfds.lock);
virt/kvm/eventfd.c
1045
INIT_LIST_HEAD(&kvm->irqfds.items);
virt/kvm/eventfd.c
1046
INIT_LIST_HEAD(&kvm->irqfds.resampler_list);
virt/kvm/eventfd.c
1047
mutex_init(&kvm->irqfds.resampler_lock);
virt/kvm/eventfd.c
1049
INIT_LIST_HEAD(&kvm->ioeventfds);
virt/kvm/eventfd.c
109
kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
virt/kvm/eventfd.c
113
synchronize_srcu_expedited(&kvm->irq_srcu);
virt/kvm/eventfd.c
116
mutex_unlock(&kvm->irqfds.resampler_lock);
virt/kvm/eventfd.c
127
struct kvm *kvm = irqfd->kvm;
virt/kvm/eventfd.c
131
synchronize_srcu_expedited(&kvm->irq_srcu);
virt/kvm/eventfd.c
170
lockdep_assert_once(lockdep_is_held(&irqfd->kvm->irqfds.lock) ||
virt/kvm/eventfd.c
171
srcu_read_lock_held(&irqfd->kvm->irq_srcu));
virt/kvm/eventfd.c
181
lockdep_assert_held(&irqfd->kvm->irqfds.lock);
virt/kvm/eventfd.c
192
struct kvm *kvm, int irq_source_id,
virt/kvm/eventfd.c
209
struct kvm *kvm = irqfd->kvm;
virt/kvm/eventfd.c
223
idx = srcu_read_lock(&kvm->irq_srcu);
virt/kvm/eventfd.c
236
kvm_arch_set_irq_inatomic(&irq, kvm,
virt/kvm/eventfd.c
240
srcu_read_unlock(&kvm->irq_srcu, idx);
virt/kvm/eventfd.c
253
spin_lock_irqsave(&kvm->irqfds.lock, iflags);
virt/kvm/eventfd.c
267
spin_unlock_irqrestore(&kvm->irqfds.lock, iflags);
virt/kvm/eventfd.c
273
static void irqfd_update(struct kvm *kvm, struct kvm_kernel_irqfd *irqfd)
virt/kvm/eventfd.c
279
lockdep_assert_held(&kvm->irqfds.lock);
virt/kvm/eventfd.c
281
n_entries = kvm_irq_map_gsi(kvm, entries, irqfd->gsi);
virt/kvm/eventfd.c
296
struct kvm *kvm;
virt/kvm/eventfd.c
306
struct kvm *kvm = p->kvm;
virt/kvm/eventfd.c
312
spin_lock_irq(&kvm->irqfds.lock);
virt/kvm/eventfd.c
319
irqfd_update(kvm, irqfd);
virt/kvm/eventfd.c
339
spin_release(&kvm->irqfds.lock.dep_map, _RET_IP_);
virt/kvm/eventfd.c
341
spin_acquire(&kvm->irqfds.lock.dep_map, 0, 0, _RET_IP_);
virt/kvm/eventfd.c
345
list_add_tail(&irqfd->list, &kvm->irqfds.items);
virt/kvm/eventfd.c
348
spin_unlock_irq(&kvm->irqfds.lock);
virt/kvm/eventfd.c
37
kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args)
virt/kvm/eventfd.c
371
kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args)
virt/kvm/eventfd.c
380
if (!kvm_arch_intc_initialized(kvm))
virt/kvm/eventfd.c
383
if (!kvm_arch_irqfd_allowed(kvm, args))
virt/kvm/eventfd.c
390
irqfd->kvm = kvm;
virt/kvm/eventfd.c
395
seqcount_spinlock_init(&irqfd->irq_entry_sc, &kvm->irqfds.lock);
virt/kvm/eventfd.c
423
mutex_lock(&kvm->irqfds.resampler_lock);
virt/kvm/eventfd.c
426
&kvm->irqfds.resampler_list, link) {
virt/kvm/eventfd.c
437
mutex_unlock(&kvm->irqfds.resampler_lock);
virt/kvm/eventfd.c
441
resampler->kvm = kvm;
virt/kvm/eventfd.c
447
list_add_rcu(&resampler->link, &kvm->irqfds.resampler_list);
virt/kvm/eventfd.c
448
kvm_register_irq_ack_notifier(kvm,
virt/kvm/eventfd.c
454
synchronize_srcu_expedited(&kvm->irq_srcu);
virt/kvm/eventfd.c
456
mutex_unlock(&kvm->irqfds.resampler_lock);
virt/kvm/eventfd.c
469
idx = srcu_read_lock(&kvm->irq_srcu);
virt/kvm/eventfd.c
47
struct kvm *kvm = irqfd->kvm;
virt/kvm/eventfd.c
478
irqfd_pt.kvm = kvm;
virt/kvm/eventfd.c
50
kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1,
virt/kvm/eventfd.c
503
srcu_read_unlock(&kvm->irq_srcu, idx);
virt/kvm/eventfd.c
507
srcu_read_unlock(&kvm->irq_srcu, idx);
virt/kvm/eventfd.c
52
kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0,
virt/kvm/eventfd.c
523
bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin)
virt/kvm/eventfd.c
528
idx = srcu_read_lock(&kvm->irq_srcu);
virt/kvm/eventfd.c
529
gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
virt/kvm/eventfd.c
531
hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,
virt/kvm/eventfd.c
532
link, srcu_read_lock_held(&kvm->irq_srcu))
virt/kvm/eventfd.c
534
srcu_read_unlock(&kvm->irq_srcu, idx);
virt/kvm/eventfd.c
538
srcu_read_unlock(&kvm->irq_srcu, idx);
virt/kvm/eventfd.c
544
void kvm_notify_acked_gsi(struct kvm *kvm, int gsi)
virt/kvm/eventfd.c
548
hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,
virt/kvm/eventfd.c
549
link, srcu_read_lock_held(&kvm->irq_srcu))
virt/kvm/eventfd.c
55
kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
virt/kvm/eventfd.c
554
void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin)
virt/kvm/eventfd.c
560
idx = srcu_read_lock(&kvm->irq_srcu);
virt/kvm/eventfd.c
561
gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
virt/kvm/eventfd.c
563
kvm_notify_acked_gsi(kvm, gsi);
virt/kvm/eventfd.c
564
srcu_read_unlock(&kvm->irq_srcu, idx);
virt/kvm/eventfd.c
567
void kvm_register_irq_ack_notifier(struct kvm *kvm,
virt/kvm/eventfd.c
570
mutex_lock(&kvm->irq_lock);
virt/kvm/eventfd.c
571
hlist_add_head_rcu(&kian->link, &kvm->irq_ack_notifier_list);
virt/kvm/eventfd.c
572
mutex_unlock(&kvm->irq_lock);
virt/kvm/eventfd.c
573
kvm_arch_post_irq_ack_notifier_list_update(kvm);
virt/kvm/eventfd.c
576
void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
virt/kvm/eventfd.c
579
mutex_lock(&kvm->irq_lock);
virt/kvm/eventfd.c
581
mutex_unlock(&kvm->irq_lock);
virt/kvm/eventfd.c
582
synchronize_srcu_expedited(&kvm->irq_srcu);
virt/kvm/eventfd.c
583
kvm_arch_post_irq_ack_notifier_list_update(kvm);
virt/kvm/eventfd.c
590
kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args)
virt/kvm/eventfd.c
599
spin_lock_irq(&kvm->irqfds.lock);
virt/kvm/eventfd.c
601
list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) {
virt/kvm/eventfd.c
606
spin_unlock_irq(&kvm->irqfds.lock);
virt/kvm/eventfd.c
620
kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
virt/kvm/eventfd.c
626
return kvm_irqfd_deassign(kvm, args);
virt/kvm/eventfd.c
628
return kvm_irqfd_assign(kvm, args);
virt/kvm/eventfd.c
636
kvm_irqfd_release(struct kvm *kvm)
virt/kvm/eventfd.c
64
srcu_read_lock_held(&resampler->kvm->irq_srcu))
virt/kvm/eventfd.c
640
spin_lock_irq(&kvm->irqfds.lock);
virt/kvm/eventfd.c
642
list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list)
virt/kvm/eventfd.c
645
spin_unlock_irq(&kvm->irqfds.lock);
virt/kvm/eventfd.c
659
void kvm_irq_routing_update(struct kvm *kvm)
virt/kvm/eventfd.c
663
spin_lock_irq(&kvm->irqfds.lock);
virt/kvm/eventfd.c
665
list_for_each_entry(irqfd, &kvm->irqfds.items, list) {
virt/kvm/eventfd.c
671
irqfd_update(kvm, irqfd);
virt/kvm/eventfd.c
679
spin_unlock_irq(&kvm->irqfds.lock);
virt/kvm/eventfd.c
682
bool kvm_notify_irqfd_resampler(struct kvm *kvm,
virt/kvm/eventfd.c
689
idx = srcu_read_lock(&kvm->irq_srcu);
virt/kvm/eventfd.c
690
gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
virt/kvm/eventfd.c
693
&kvm->irqfds.resampler_list, link,
virt/kvm/eventfd.c
694
srcu_read_lock_held(&kvm->irq_srcu)) {
virt/kvm/eventfd.c
697
srcu_read_unlock(&kvm->irq_srcu, idx);
virt/kvm/eventfd.c
702
srcu_read_unlock(&kvm->irq_srcu, idx);
virt/kvm/eventfd.c
77
struct kvm *kvm;
virt/kvm/eventfd.c
82
kvm = resampler->kvm;
virt/kvm/eventfd.c
836
ioeventfd_check_collision(struct kvm *kvm, struct _ioeventfd *p)
virt/kvm/eventfd.c
84
kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
virt/kvm/eventfd.c
840
list_for_each_entry(_p, &kvm->ioeventfds, list)
virt/kvm/eventfd.c
861
static int kvm_assign_ioeventfd_idx(struct kvm *kvm,
virt/kvm/eventfd.c
87
idx = srcu_read_lock(&kvm->irq_srcu);
virt/kvm/eventfd.c
89
srcu_read_unlock(&kvm->irq_srcu, idx);
virt/kvm/eventfd.c
892
mutex_lock(&kvm->slots_lock);
virt/kvm/eventfd.c
895
if (ioeventfd_check_collision(kvm, p)) {
virt/kvm/eventfd.c
902
ret = kvm_io_bus_register_dev(kvm, bus_idx, p->addr, p->length,
virt/kvm/eventfd.c
907
kvm_get_bus(kvm, bus_idx)->ioeventfd_count++;
virt/kvm/eventfd.c
908
list_add_tail(&p->list, &kvm->ioeventfds);
virt/kvm/eventfd.c
910
mutex_unlock(&kvm->slots_lock);
virt/kvm/eventfd.c
915
mutex_unlock(&kvm->slots_lock);
virt/kvm/eventfd.c
925
kvm_deassign_ioeventfd_idx(struct kvm *kvm, enum kvm_bus bus_idx,
virt/kvm/eventfd.c
940
mutex_lock(&kvm->slots_lock);
virt/kvm/eventfd.c
942
list_for_each_entry(p, &kvm->ioeventfds, list) {
virt/kvm/eventfd.c
953
kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);
virt/kvm/eventfd.c
954
bus = kvm_get_bus(kvm, bus_idx);
virt/kvm/eventfd.c
96
struct kvm *kvm = resampler->kvm;
virt/kvm/eventfd.c
961
mutex_unlock(&kvm->slots_lock);
virt/kvm/eventfd.c
968
static int kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
virt/kvm/eventfd.c
971
int ret = kvm_deassign_ioeventfd_idx(kvm, bus_idx, args);
virt/kvm/eventfd.c
974
kvm_deassign_ioeventfd_idx(kvm, KVM_FAST_MMIO_BUS, args);
virt/kvm/eventfd.c
98
mutex_lock(&kvm->irqfds.resampler_lock);
virt/kvm/eventfd.c
980
kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
virt/kvm/guest_memfd.c
108
return __kvm_gmem_prepare_folio(kvm, slot, index, folio);
virt/kvm/guest_memfd.c
166
struct kvm *kvm = f->kvm;
virt/kvm/guest_memfd.c
183
KVM_MMU_LOCK(kvm);
virt/kvm/guest_memfd.c
184
kvm_mmu_invalidate_start(kvm);
virt/kvm/guest_memfd.c
187
flush |= kvm_mmu_unmap_gfn_range(kvm, &gfn_range);
virt/kvm/guest_memfd.c
191
kvm_flush_remote_tlbs(kvm);
virt/kvm/guest_memfd.c
194
KVM_MMU_UNLOCK(kvm);
virt/kvm/guest_memfd.c
212
struct kvm *kvm = f->kvm;
virt/kvm/guest_memfd.c
215
KVM_MMU_LOCK(kvm);
virt/kvm/guest_memfd.c
216
kvm_mmu_invalidate_end(kvm);
virt/kvm/guest_memfd.c
217
KVM_MMU_UNLOCK(kvm);
virt/kvm/guest_memfd.c
25
struct kvm *kvm;
virt/kvm/guest_memfd.c
327
struct kvm *kvm = f->kvm;
virt/kvm/guest_memfd.c
343
mutex_lock(&kvm->slots_lock);
virt/kvm/guest_memfd.c
363
mutex_unlock(&kvm->slots_lock);
virt/kvm/guest_memfd.c
368
kvm_put_kvm(kvm);
virt/kvm/guest_memfd.c
555
bool __weak kvm_arch_supports_gmem_init_shared(struct kvm *kvm)
virt/kvm/guest_memfd.c
560
static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
virt/kvm/guest_memfd.c
610
kvm_get_kvm(kvm);
virt/kvm/guest_memfd.c
611
f->kvm = kvm;
virt/kvm/guest_memfd.c
629
int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args)
virt/kvm/guest_memfd.c
63
static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot,
virt/kvm/guest_memfd.c
634
if (flags & ~kvm_gmem_get_supported_flags(kvm))
virt/kvm/guest_memfd.c
640
return __kvm_gmem_create(kvm, size, flags);
virt/kvm/guest_memfd.c
643
int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
virt/kvm/guest_memfd.c
664
if (f->kvm != kvm)
virt/kvm/guest_memfd.c
69
int rc = kvm_arch_gmem_prepare(kvm, gfn, pfn, folio_order(folio));
virt/kvm/guest_memfd.c
790
int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
virt/kvm/guest_memfd.c
811
r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio);
virt/kvm/guest_memfd.c
826
static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot,
virt/kvm/guest_memfd.c
845
if (!kvm_range_has_memory_attributes(kvm, gfn, gfn + 1,
virt/kvm/guest_memfd.c
852
ret = post_populate(kvm, gfn, pfn, src_page, opaque);
virt/kvm/guest_memfd.c
86
static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot,
virt/kvm/guest_memfd.c
863
long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src,
virt/kvm/guest_memfd.c
871
lockdep_assert_held(&kvm->slots_lock);
virt/kvm/guest_memfd.c
879
slot = gfn_to_memslot(kvm, start_gfn);
virt/kvm/guest_memfd.c
909
ret = __kvm_gmem_populate(kvm, slot, file, start_gfn + i, src_page,
virt/kvm/irqchip.c
119
void kvm_free_irq_routing(struct kvm *kvm)
virt/kvm/irqchip.c
123
struct kvm_irq_routing_table *rt = rcu_access_pointer(kvm->irq_routing);
virt/kvm/irqchip.c
127
static int setup_routing_entry(struct kvm *kvm,
virt/kvm/irqchip.c
148
r = kvm_set_routing_entry(kvm, e, ue);
virt/kvm/irqchip.c
159
void __attribute__((weak)) kvm_arch_irq_routing_update(struct kvm *kvm)
virt/kvm/irqchip.c
163
bool __weak kvm_arch_can_set_irq_routing(struct kvm *kvm)
virt/kvm/irqchip.c
168
int kvm_set_irq_routing(struct kvm *kvm,
virt/kvm/irqchip.c
21
int kvm_irq_map_gsi(struct kvm *kvm,
virt/kvm/irqchip.c
212
r = setup_routing_entry(kvm, new, e, ue);
virt/kvm/irqchip.c
218
mutex_lock(&kvm->irq_lock);
virt/kvm/irqchip.c
219
old = rcu_dereference_protected(kvm->irq_routing, 1);
virt/kvm/irqchip.c
220
rcu_assign_pointer(kvm->irq_routing, new);
virt/kvm/irqchip.c
221
kvm_irq_routing_update(kvm);
virt/kvm/irqchip.c
222
kvm_arch_irq_routing_update(kvm);
virt/kvm/irqchip.c
223
mutex_unlock(&kvm->irq_lock);
virt/kvm/irqchip.c
225
synchronize_srcu_expedited(&kvm->irq_srcu);
virt/kvm/irqchip.c
244
int kvm_init_irq_routing(struct kvm *kvm)
virt/kvm/irqchip.c
258
RCU_INIT_POINTER(kvm->irq_routing, new);
virt/kvm/irqchip.c
28
irq_rt = srcu_dereference_check(kvm->irq_routing, &kvm->irq_srcu,
virt/kvm/irqchip.c
29
lockdep_is_held(&kvm->irq_lock));
virt/kvm/irqchip.c
40
int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin)
virt/kvm/irqchip.c
44
irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
virt/kvm/irqchip.c
48
int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi)
virt/kvm/irqchip.c
52
if (!kvm_arch_irqchip_in_kernel(kvm) || (msi->flags & ~KVM_MSI_VALID_DEVID))
virt/kvm/irqchip.c
61
return kvm_set_msi(&route, kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 1, false);
virt/kvm/irqchip.c
70
int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
virt/kvm/irqchip.c
82
idx = srcu_read_lock(&kvm->irq_srcu);
virt/kvm/irqchip.c
83
i = kvm_irq_map_gsi(kvm, irq_set, irq);
virt/kvm/irqchip.c
84
srcu_read_unlock(&kvm->irq_srcu, idx);
virt/kvm/irqchip.c
88
r = irq_set[i].set(&irq_set[i], kvm, irq_source_id, level,
virt/kvm/kvm_main.c
1025
kvm->debugfs_dentry = dent;
virt/kvm/kvm_main.c
1026
kvm->debugfs_stat_data = kzalloc_objs(*kvm->debugfs_stat_data,
virt/kvm/kvm_main.c
1029
if (!kvm->debugfs_stat_data)
virt/kvm/kvm_main.c
1038
stat_data->kvm = kvm;
virt/kvm/kvm_main.c
1041
kvm->debugfs_stat_data[i] = stat_data;
virt/kvm/kvm_main.c
1043
kvm->debugfs_dentry, stat_data,
virt/kvm/kvm_main.c
1053
stat_data->kvm = kvm;
virt/kvm/kvm_main.c
1056
kvm->debugfs_stat_data[i + kvm_vm_stats_header.num_desc] = stat_data;
virt/kvm/kvm_main.c
1058
kvm->debugfs_dentry, stat_data,
virt/kvm/kvm_main.c
1062
kvm_arch_create_vm_debugfs(kvm);
virt/kvm/kvm_main.c
1065
kvm_destroy_vm_debugfs(kvm);
virt/kvm/kvm_main.c
1073
void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm)
virt/kvm/kvm_main.c
1083
void __weak kvm_arch_create_vm_debugfs(struct kvm *kvm)
virt/kvm/kvm_main.c
1088
static inline struct kvm_io_bus *kvm_get_bus_for_destruction(struct kvm *kvm,
virt/kvm/kvm_main.c
1091
return rcu_dereference_protected(kvm->buses[idx],
virt/kvm/kvm_main.c
1092
!refcount_read(&kvm->users_count));
virt/kvm/kvm_main.c
1098
static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
virt/kvm/kvm_main.c
1100
struct kvm *kvm = kvm_arch_alloc_vm();
virt/kvm/kvm_main.c
1104
if (!kvm)
virt/kvm/kvm_main.c
1107
KVM_MMU_LOCK_INIT(kvm);
virt/kvm/kvm_main.c
1109
kvm->mm = current->mm;
virt/kvm/kvm_main.c
1110
kvm_eventfd_init(kvm);
virt/kvm/kvm_main.c
1111
mutex_init(&kvm->lock);
virt/kvm/kvm_main.c
1112
mutex_init(&kvm->irq_lock);
virt/kvm/kvm_main.c
1113
mutex_init(&kvm->slots_lock);
virt/kvm/kvm_main.c
1114
mutex_init(&kvm->slots_arch_lock);
virt/kvm/kvm_main.c
1115
spin_lock_init(&kvm->mn_invalidate_lock);
virt/kvm/kvm_main.c
1116
rcuwait_init(&kvm->mn_memslots_update_rcuwait);
virt/kvm/kvm_main.c
1117
xa_init(&kvm->vcpu_array);
virt/kvm/kvm_main.c
1119
xa_init(&kvm->mem_attr_array);
virt/kvm/kvm_main.c
1122
INIT_LIST_HEAD(&kvm->gpc_list);
virt/kvm/kvm_main.c
1123
spin_lock_init(&kvm->gpc_lock);
virt/kvm/kvm_main.c
1125
INIT_LIST_HEAD(&kvm->devices);
virt/kvm/kvm_main.c
1126
kvm->max_vcpus = KVM_MAX_VCPUS;
virt/kvm/kvm_main.c
1134
kvm->debugfs_dentry = ERR_PTR(-ENOENT);
virt/kvm/kvm_main.c
1136
snprintf(kvm->stats_id, sizeof(kvm->stats_id), "kvm-%d",
virt/kvm/kvm_main.c
1140
if (init_srcu_struct(&kvm->srcu))
virt/kvm/kvm_main.c
1142
if (init_srcu_struct(&kvm->irq_srcu))
virt/kvm/kvm_main.c
1145
r = kvm_init_irq_routing(kvm);
virt/kvm/kvm_main.c
1149
refcount_set(&kvm->users_count, 1);
virt/kvm/kvm_main.c
1151
for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
virt/kvm/kvm_main.c
1153
slots = &kvm->__memslots[i][j];
virt/kvm/kvm_main.c
1165
rcu_assign_pointer(kvm->memslots[i], &kvm->__memslots[i][0]);
virt/kvm/kvm_main.c
1170
rcu_assign_pointer(kvm->buses[i],
virt/kvm/kvm_main.c
1172
if (!kvm->buses[i])
virt/kvm/kvm_main.c
1176
r = kvm_arch_init_vm(kvm, type);
virt/kvm/kvm_main.c
1185
INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
virt/kvm/kvm_main.c
1188
r = kvm_init_mmu_notifier(kvm);
virt/kvm/kvm_main.c
1192
r = kvm_coalesced_mmio_init(kvm);
virt/kvm/kvm_main.c
1196
r = kvm_create_vm_debugfs(kvm, fdname);
virt/kvm/kvm_main.c
1201
list_add(&kvm->vm_list, &vm_list);
virt/kvm/kvm_main.c
1205
kvm_init_pm_notifier(kvm);
virt/kvm/kvm_main.c
1207
return kvm;
virt/kvm/kvm_main.c
1210
kvm_coalesced_mmio_free(kvm);
virt/kvm/kvm_main.c
1212
if (kvm->mmu_notifier.ops)
virt/kvm/kvm_main.c
1213
mmu_notifier_unregister(&kvm->mmu_notifier, current->mm);
virt/kvm/kvm_main.c
1217
kvm_arch_destroy_vm(kvm);
virt/kvm/kvm_main.c
1219
WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
virt/kvm/kvm_main.c
1221
kfree(kvm_get_bus_for_destruction(kvm, i));
virt/kvm/kvm_main.c
1222
kvm_free_irq_routing(kvm);
virt/kvm/kvm_main.c
1224
cleanup_srcu_struct(&kvm->irq_srcu);
virt/kvm/kvm_main.c
1226
cleanup_srcu_struct(&kvm->srcu);
virt/kvm/kvm_main.c
1228
kvm_arch_free_vm(kvm);
virt/kvm/kvm_main.c
1233
static void kvm_destroy_devices(struct kvm *kvm)
virt/kvm/kvm_main.c
1248
list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
virt/kvm/kvm_main.c
1254
static void kvm_destroy_vm(struct kvm *kvm)
virt/kvm/kvm_main.c
1257
struct mm_struct *mm = kvm->mm;
virt/kvm/kvm_main.c
1259
kvm_destroy_pm_notifier(kvm);
virt/kvm/kvm_main.c
1260
kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
virt/kvm/kvm_main.c
1261
kvm_destroy_vm_debugfs(kvm);
virt/kvm/kvm_main.c
1263
list_del(&kvm->vm_list);
virt/kvm/kvm_main.c
1265
kvm_arch_pre_destroy_vm(kvm);
virt/kvm/kvm_main.c
1267
kvm_free_irq_routing(kvm);
virt/kvm/kvm_main.c
1269
struct kvm_io_bus *bus = kvm_get_bus_for_destruction(kvm, i);
virt/kvm/kvm_main.c
1273
kvm->buses[i] = NULL;
virt/kvm/kvm_main.c
1275
kvm_coalesced_mmio_free(kvm);
virt/kvm/kvm_main.c
1276
mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
virt/kvm/kvm_main.c
1289
WARN_ON(rcuwait_active(&kvm->mn_memslots_update_rcuwait));
virt/kvm/kvm_main.c
1290
if (kvm->mn_active_invalidate_count)
virt/kvm/kvm_main.c
1291
kvm->mn_active_invalidate_count = 0;
virt/kvm/kvm_main.c
1293
WARN_ON(kvm->mmu_invalidate_in_progress);
virt/kvm/kvm_main.c
1294
kvm_arch_destroy_vm(kvm);
virt/kvm/kvm_main.c
1295
kvm_destroy_devices(kvm);
virt/kvm/kvm_main.c
1296
for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
virt/kvm/kvm_main.c
1297
kvm_free_memslots(kvm, &kvm->__memslots[i][0]);
virt/kvm/kvm_main.c
1298
kvm_free_memslots(kvm, &kvm->__memslots[i][1]);
virt/kvm/kvm_main.c
1300
cleanup_srcu_struct(&kvm->irq_srcu);
virt/kvm/kvm_main.c
1301
srcu_barrier(&kvm->srcu);
virt/kvm/kvm_main.c
1302
cleanup_srcu_struct(&kvm->srcu);
virt/kvm/kvm_main.c
1304
xa_destroy(&kvm->mem_attr_array);
virt/kvm/kvm_main.c
1306
kvm_arch_free_vm(kvm);
virt/kvm/kvm_main.c
1312
void kvm_get_kvm(struct kvm *kvm)
virt/kvm/kvm_main.c
1314
refcount_inc(&kvm->users_count);
virt/kvm/kvm_main.c
1322
bool kvm_get_kvm_safe(struct kvm *kvm)
virt/kvm/kvm_main.c
1324
return refcount_inc_not_zero(&kvm->users_count);
virt/kvm/kvm_main.c
1328
void kvm_put_kvm(struct kvm *kvm)
virt/kvm/kvm_main.c
1330
if (refcount_dec_and_test(&kvm->users_count))
virt/kvm/kvm_main.c
1331
kvm_destroy_vm(kvm);
virt/kvm/kvm_main.c
1342
void kvm_put_kvm_no_destroy(struct kvm *kvm)
virt/kvm/kvm_main.c
1344
WARN_ON(refcount_dec_and_test(&kvm->users_count));
virt/kvm/kvm_main.c
1350
struct kvm *kvm = filp->private_data;
virt/kvm/kvm_main.c
1352
kvm_irqfd_release(kvm);
virt/kvm/kvm_main.c
1354
kvm_put_kvm(kvm);
virt/kvm/kvm_main.c
1358
int kvm_trylock_all_vcpus(struct kvm *kvm)
virt/kvm/kvm_main.c
1363
lockdep_assert_held(&kvm->lock);
virt/kvm/kvm_main.c
1365
kvm_for_each_vcpu(i, vcpu, kvm)
virt/kvm/kvm_main.c
1366
if (!mutex_trylock_nest_lock(&vcpu->mutex, &kvm->lock))
virt/kvm/kvm_main.c
1371
kvm_for_each_vcpu(j, vcpu, kvm) {
virt/kvm/kvm_main.c
1380
int kvm_lock_all_vcpus(struct kvm *kvm)
virt/kvm/kvm_main.c
1386
lockdep_assert_held(&kvm->lock);
virt/kvm/kvm_main.c
1388
kvm_for_each_vcpu(i, vcpu, kvm) {
virt/kvm/kvm_main.c
1389
r = mutex_lock_killable_nest_lock(&vcpu->mutex, &kvm->lock);
virt/kvm/kvm_main.c
1396
kvm_for_each_vcpu(j, vcpu, kvm) {
virt/kvm/kvm_main.c
1405
void kvm_unlock_all_vcpus(struct kvm *kvm)
virt/kvm/kvm_main.c
1410
lockdep_assert_held(&kvm->lock);
virt/kvm/kvm_main.c
1412
kvm_for_each_vcpu(i, vcpu, kvm)
virt/kvm/kvm_main.c
1432
static struct kvm_memslots *kvm_get_inactive_memslots(struct kvm *kvm, int as_id)
virt/kvm/kvm_main.c
1434
struct kvm_memslots *active = __kvm_memslots(kvm, as_id);
virt/kvm/kvm_main.c
1437
return &kvm->__memslots[as_id][node_idx_inactive];
virt/kvm/kvm_main.c
151
static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
virt/kvm/kvm_main.c
1512
static void kvm_replace_memslot(struct kvm *kvm,
virt/kvm/kvm_main.c
1517
struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);
virt/kvm/kvm_main.c
157
__weak void kvm_arch_guest_memory_reclaimed(struct kvm *kvm)
virt/kvm/kvm_main.c
1573
static int check_memory_region_flags(struct kvm *kvm,
virt/kvm/kvm_main.c
1590
if (kvm_arch_has_readonly_mem(kvm) &&
virt/kvm/kvm_main.c
1600
static void kvm_swap_active_memslots(struct kvm *kvm, int as_id)
virt/kvm/kvm_main.c
1602
struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);
virt/kvm/kvm_main.c
1605
u64 gen = __kvm_memslots(kvm, as_id)->generation;
virt/kvm/kvm_main.c
1615
spin_lock(&kvm->mn_invalidate_lock);
virt/kvm/kvm_main.c
1616
prepare_to_rcuwait(&kvm->mn_memslots_update_rcuwait);
virt/kvm/kvm_main.c
1617
while (kvm->mn_active_invalidate_count) {
virt/kvm/kvm_main.c
1619
spin_unlock(&kvm->mn_invalidate_lock);
virt/kvm/kvm_main.c
1621
spin_lock(&kvm->mn_invalidate_lock);
virt/kvm/kvm_main.c
1623
finish_rcuwait(&kvm->mn_memslots_update_rcuwait);
virt/kvm/kvm_main.c
1624
rcu_assign_pointer(kvm->memslots[as_id], slots);
virt/kvm/kvm_main.c
1625
spin_unlock(&kvm->mn_invalidate_lock);
virt/kvm/kvm_main.c
1632
mutex_unlock(&kvm->slots_arch_lock);
virt/kvm/kvm_main.c
1634
synchronize_srcu_expedited(&kvm->srcu);
virt/kvm/kvm_main.c
1651
gen += kvm_arch_nr_memslot_as_ids(kvm);
virt/kvm/kvm_main.c
1653
kvm_arch_memslots_updated(kvm, gen);
virt/kvm/kvm_main.c
1658
static int kvm_prepare_memory_region(struct kvm *kvm,
virt/kvm/kvm_main.c
1677
else if (kvm_use_dirty_bitmap(kvm)) {
virt/kvm/kvm_main.c
1682
if (kvm_dirty_log_manual_protect_and_init_set(kvm))
virt/kvm/kvm_main.c
1687
r = kvm_arch_prepare_memory_region(kvm, old, new, change);
virt/kvm/kvm_main.c
1696
static void kvm_commit_memory_region(struct kvm *kvm,
virt/kvm/kvm_main.c
1708
kvm->nr_memslot_pages -= old->npages;
virt/kvm/kvm_main.c
1710
kvm->nr_memslot_pages += new->npages;
virt/kvm/kvm_main.c
1714
atomic_set(&kvm->nr_memslots_dirty_logging,
virt/kvm/kvm_main.c
1715
atomic_read(&kvm->nr_memslots_dirty_logging) + change);
virt/kvm/kvm_main.c
1718
kvm_arch_commit_memory_region(kvm, old, new, change);
virt/kvm/kvm_main.c
1726
kvm_free_memslot(kvm, old);
virt/kvm/kvm_main.c
1773
static void kvm_activate_memslot(struct kvm *kvm,
virt/kvm/kvm_main.c
1779
kvm_swap_active_memslots(kvm, as_id);
virt/kvm/kvm_main.c
1782
kvm_replace_memslot(kvm, old, new);
virt/kvm/kvm_main.c
1798
static void kvm_invalidate_memslot(struct kvm *kvm,
virt/kvm/kvm_main.c
1809
kvm_replace_memslot(kvm, old, invalid_slot);
virt/kvm/kvm_main.c
1816
kvm_swap_active_memslots(kvm, old->as_id);
virt/kvm/kvm_main.c
1824
kvm_arch_flush_shadow_memslot(kvm, old);
virt/kvm/kvm_main.c
1825
kvm_arch_guest_memory_reclaimed(kvm);
virt/kvm/kvm_main.c
1828
mutex_lock(&kvm->slots_arch_lock);
virt/kvm/kvm_main.c
1840
static void kvm_create_memslot(struct kvm *kvm,
virt/kvm/kvm_main.c
1844
kvm_replace_memslot(kvm, NULL, new);
virt/kvm/kvm_main.c
1845
kvm_activate_memslot(kvm, NULL, new);
virt/kvm/kvm_main.c
1848
static void kvm_delete_memslot(struct kvm *kvm,
virt/kvm/kvm_main.c
1856
kvm_replace_memslot(kvm, old, NULL);
virt/kvm/kvm_main.c
1857
kvm_activate_memslot(kvm, invalid_slot, NULL);
virt/kvm/kvm_main.c
1860
static void kvm_move_memslot(struct kvm *kvm,
virt/kvm/kvm_main.c
1869
kvm_replace_memslot(kvm, old, new);
virt/kvm/kvm_main.c
1870
kvm_activate_memslot(kvm, invalid_slot, new);
virt/kvm/kvm_main.c
1873
static void kvm_update_flags_memslot(struct kvm *kvm,
virt/kvm/kvm_main.c
1882
kvm_replace_memslot(kvm, old, new);
virt/kvm/kvm_main.c
1883
kvm_activate_memslot(kvm, old, new);
virt/kvm/kvm_main.c
1886
static int kvm_set_memslot(struct kvm *kvm,
virt/kvm/kvm_main.c
1908
mutex_lock(&kvm->slots_arch_lock);
virt/kvm/kvm_main.c
1926
mutex_unlock(&kvm->slots_arch_lock);
virt/kvm/kvm_main.c
1929
kvm_invalidate_memslot(kvm, old, invalid_slot);
virt/kvm/kvm_main.c
1932
r = kvm_prepare_memory_region(kvm, old, new, change);
virt/kvm/kvm_main.c
1941
kvm_activate_memslot(kvm, invalid_slot, old);
virt/kvm/kvm_main.c
1944
mutex_unlock(&kvm->slots_arch_lock);
virt/kvm/kvm_main.c
1957
kvm_create_memslot(kvm, new);
virt/kvm/kvm_main.c
1959
kvm_delete_memslot(kvm, old, invalid_slot);
virt/kvm/kvm_main.c
1961
kvm_move_memslot(kvm, old, new, invalid_slot);
virt/kvm/kvm_main.c
1963
kvm_update_flags_memslot(kvm, old, new);
virt/kvm/kvm_main.c
1976
kvm_commit_memory_region(kvm, old, new, change);
virt/kvm/kvm_main.c
1994
static int kvm_set_memory_region(struct kvm *kvm,
virt/kvm/kvm_main.c
2005
lockdep_assert_held(&kvm->slots_lock);
virt/kvm/kvm_main.c
2007
r = check_memory_region_flags(kvm, mem);
virt/kvm/kvm_main.c
2030
if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_MEM_SLOTS_NUM)
virt/kvm/kvm_main.c
2045
slots = __kvm_memslots(kvm, as_id);
virt/kvm/kvm_main.c
2057
if (WARN_ON_ONCE(kvm->nr_memslot_pages < old->npages))
virt/kvm/kvm_main.c
2060
return kvm_set_memslot(kvm, old, NULL, KVM_MR_DELETE);
virt/kvm/kvm_main.c
2073
if ((kvm->nr_memslot_pages + npages) < kvm->nr_memslot_pages)
virt/kvm/kvm_main.c
2108
r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset);
virt/kvm/kvm_main.c
2113
r = kvm_set_memslot(kvm, old, new, change);
virt/kvm/kvm_main.c
2127
int kvm_set_internal_memslot(struct kvm *kvm,
virt/kvm/kvm_main.c
2136
return kvm_set_memory_region(kvm, mem);
virt/kvm/kvm_main.c
2140
static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
virt/kvm/kvm_main.c
2146
guard(mutex)(&kvm->slots_lock);
virt/kvm/kvm_main.c
2147
return kvm_set_memory_region(kvm, mem);
virt/kvm/kvm_main.c
2158
int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log,
virt/kvm/kvm_main.c
2167
if (!kvm_use_dirty_bitmap(kvm))
virt/kvm/kvm_main.c
2175
if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
virt/kvm/kvm_main.c
2178
slots = __kvm_memslots(kvm, as_id);
virt/kvm/kvm_main.c
2183
kvm_arch_sync_dirty_log(kvm, *memslot);
virt/kvm/kvm_main.c
2221
static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log)
virt/kvm/kvm_main.c
2232
if (!kvm_use_dirty_bitmap(kvm))
virt/kvm/kvm_main.c
2237
if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
virt/kvm/kvm_main.c
2240
slots = __kvm_memslots(kvm, as_id);
virt/kvm/kvm_main.c
2247
kvm_arch_sync_dirty_log(kvm, memslot);
virt/kvm/kvm_main.c
2251
if (kvm->manual_dirty_log_protect) {
virt/kvm/kvm_main.c
2265
KVM_MMU_LOCK(kvm);
virt/kvm/kvm_main.c
2278
kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
virt/kvm/kvm_main.c
2281
KVM_MMU_UNLOCK(kvm);
virt/kvm/kvm_main.c
2285
kvm_flush_remote_tlbs_memslot(kvm, memslot);
virt/kvm/kvm_main.c
2312
static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
virt/kvm/kvm_main.c
2317
mutex_lock(&kvm->slots_lock);
virt/kvm/kvm_main.c
2319
r = kvm_get_dirty_log_protect(kvm, log);
virt/kvm/kvm_main.c
2321
mutex_unlock(&kvm->slots_lock);
virt/kvm/kvm_main.c
2331
static int kvm_clear_dirty_log_protect(struct kvm *kvm,
virt/kvm/kvm_main.c
2344
if (!kvm_use_dirty_bitmap(kvm))
virt/kvm/kvm_main.c
2349
if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
virt/kvm/kvm_main.c
2355
slots = __kvm_memslots(kvm, as_id);
virt/kvm/kvm_main.c
2369
kvm_arch_sync_dirty_log(kvm, memslot);
virt/kvm/kvm_main.c
2376
KVM_MMU_LOCK(kvm);
virt/kvm/kvm_main.c
2395
kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
virt/kvm/kvm_main.c
2399
KVM_MMU_UNLOCK(kvm);
virt/kvm/kvm_main.c
2402
kvm_flush_remote_tlbs_memslot(kvm, memslot);
virt/kvm/kvm_main.c
2407
static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm,
virt/kvm/kvm_main.c
2412
mutex_lock(&kvm->slots_lock);
virt/kvm/kvm_main.c
2414
r = kvm_clear_dirty_log_protect(kvm, log);
virt/kvm/kvm_main.c
2416
mutex_unlock(&kvm->slots_lock);
virt/kvm/kvm_main.c
2422
static u64 kvm_supported_mem_attributes(struct kvm *kvm)
virt/kvm/kvm_main.c
2424
if (!kvm || kvm_arch_has_private_mem(kvm))
virt/kvm/kvm_main.c
2434
bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
virt/kvm/kvm_main.c
2437
XA_STATE(xas, &kvm->mem_attr_array, start);
virt/kvm/kvm_main.c
244
bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
virt/kvm/kvm_main.c
2441
mask &= kvm_supported_mem_attributes(kvm);
virt/kvm/kvm_main.c
2446
return (kvm_get_memory_attributes(kvm, start) & mask) == attrs;
virt/kvm/kvm_main.c
2465
static __always_inline void kvm_handle_gfn_range(struct kvm *kvm,
virt/kvm/kvm_main.c
2487
for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
virt/kvm/kvm_main.c
2488
slots = __kvm_memslots(kvm, i);
virt/kvm/kvm_main.c
2501
KVM_MMU_LOCK(kvm);
virt/kvm/kvm_main.c
2503
range->on_lock(kvm);
virt/kvm/kvm_main.c
2506
ret |= range->handler(kvm, &gfn_range);
virt/kvm/kvm_main.c
2511
kvm_flush_remote_tlbs(kvm);
virt/kvm/kvm_main.c
2514
KVM_MMU_UNLOCK(kvm);
virt/kvm/kvm_main.c
2517
static bool kvm_pre_set_memory_attributes(struct kvm *kvm,
virt/kvm/kvm_main.c
2531
kvm_mmu_invalidate_range_add(kvm, range->start, range->end);
virt/kvm/kvm_main.c
2533
return kvm_arch_pre_set_memory_attributes(kvm, range);
virt/kvm/kvm_main.c
2537
static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
virt/kvm/kvm_main.c
2565
mutex_lock(&kvm->slots_lock);
virt/kvm/kvm_main.c
2568
if (kvm_range_has_memory_attributes(kvm, start, end, ~0, attributes))
virt/kvm/kvm_main.c
2576
r = xa_reserve(&kvm->mem_attr_array, i, GFP_KERNEL_ACCOUNT);
virt/kvm/kvm_main.c
258
vcpu = kvm_get_vcpu(kvm, i);
virt/kvm/kvm_main.c
2583
kvm_handle_gfn_range(kvm, &pre_set_range);
virt/kvm/kvm_main.c
2586
r = xa_err(xa_store(&kvm->mem_attr_array, i, entry,
virt/kvm/kvm_main.c
2588
KVM_BUG_ON(r, kvm);
virt/kvm/kvm_main.c
2592
kvm_handle_gfn_range(kvm, &post_set_range);
virt/kvm/kvm_main.c
2595
mutex_unlock(&kvm->slots_lock);
virt/kvm/kvm_main.c
2599
static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm,
virt/kvm/kvm_main.c
2607
if (attrs->attributes & ~kvm_supported_mem_attributes(kvm))
virt/kvm/kvm_main.c
2624
return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes);
virt/kvm/kvm_main.c
2628
struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
virt/kvm/kvm_main.c
2630
return __gfn_to_memslot(kvm_memslots(kvm), gfn);
virt/kvm/kvm_main.c
2668
bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
virt/kvm/kvm_main.c
2670
struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
virt/kvm/kvm_main.c
270
bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
virt/kvm/kvm_main.c
2741
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
virt/kvm/kvm_main.c
2743
return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
virt/kvm/kvm_main.c
2772
unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
virt/kvm/kvm_main.c
2774
struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
virt/kvm/kvm_main.c
283
kvm_for_each_vcpu(i, vcpu, kvm)
virt/kvm/kvm_main.c
293
void kvm_flush_remote_tlbs(struct kvm *kvm)
virt/kvm/kvm_main.c
295
++kvm->stat.generic.remote_tlb_flush_requests;
virt/kvm/kvm_main.c
308
if (!kvm_arch_flush_remote_tlbs(kvm)
virt/kvm/kvm_main.c
309
|| kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
virt/kvm/kvm_main.c
3095
struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn, bool write)
virt/kvm/kvm_main.c
3099
.slot = gfn_to_memslot(kvm, gfn),
virt/kvm/kvm_main.c
310
++kvm->stat.generic.remote_tlb_flush;
virt/kvm/kvm_main.c
314
void kvm_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages)
virt/kvm/kvm_main.c
316
if (!kvm_arch_flush_remote_tlbs_range(kvm, gfn, nr_pages))
virt/kvm/kvm_main.c
3199
int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
virt/kvm/kvm_main.c
3202
struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
virt/kvm/kvm_main.c
3217
int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
virt/kvm/kvm_main.c
3225
ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
virt/kvm/kvm_main.c
324
kvm_flush_remote_tlbs(kvm);
virt/kvm/kvm_main.c
327
void kvm_flush_remote_tlbs_memslot(struct kvm *kvm,
virt/kvm/kvm_main.c
3289
static int __kvm_write_guest_page(struct kvm *kvm,
virt/kvm/kvm_main.c
3305
mark_page_dirty_in_slot(kvm, memslot, gfn);
virt/kvm/kvm_main.c
3309
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
virt/kvm/kvm_main.c
3312
struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
virt/kvm/kvm_main.c
3314
return __kvm_write_guest_page(kvm, slot, gfn, data, offset, len);
virt/kvm/kvm_main.c
3323
return __kvm_write_guest_page(vcpu->kvm, slot, gfn, data, offset, len);
virt/kvm/kvm_main.c
3327
int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
virt/kvm/kvm_main.c
3336
ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
virt/kvm/kvm_main.c
337
lockdep_assert_held(&kvm->slots_lock);
virt/kvm/kvm_main.c
338
kvm_flush_remote_tlbs_range(kvm, memslot->base_gfn, memslot->npages);
virt/kvm/kvm_main.c
341
static void kvm_flush_shadow_all(struct kvm *kvm)
virt/kvm/kvm_main.c
3410
int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
virt/kvm/kvm_main.c
3413
struct kvm_memslots *slots = kvm_memslots(kvm);
virt/kvm/kvm_main.c
3418
int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
virt/kvm/kvm_main.c
3422
struct kvm_memslots *slots = kvm_memslots(kvm);
virt/kvm/kvm_main.c
343
kvm_arch_flush_shadow_all(kvm);
virt/kvm/kvm_main.c
3438
return kvm_write_guest(kvm, gpa, data, len);
virt/kvm/kvm_main.c
344
kvm_arch_guest_memory_reclaimed(kvm);
virt/kvm/kvm_main.c
3443
mark_page_dirty_in_slot(kvm, ghc->memslot, gpa >> PAGE_SHIFT);
virt/kvm/kvm_main.c
3449
int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
virt/kvm/kvm_main.c
3452
return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
virt/kvm/kvm_main.c
3456
int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
virt/kvm/kvm_main.c
3460
struct kvm_memslots *slots = kvm_memslots(kvm);
virt/kvm/kvm_main.c
3476
return kvm_read_guest(kvm, gpa, data, len);
virt/kvm/kvm_main.c
3486
int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
virt/kvm/kvm_main.c
3489
return kvm_read_guest_offset_cached(kvm, ghc, data, 0, len);
virt/kvm/kvm_main.c
3493
int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
virt/kvm/kvm_main.c
3502
ret = kvm_write_guest_page(kvm, gfn, zero_page, offset, seg);
virt/kvm/kvm_main.c
3513
void mark_page_dirty_in_slot(struct kvm *kvm,
virt/kvm/kvm_main.c
3520
if (WARN_ON_ONCE(vcpu && vcpu->kvm != kvm))
virt/kvm/kvm_main.c
3523
WARN_ON_ONCE(!vcpu && refcount_read(&kvm->users_count) &&
virt/kvm/kvm_main.c
3524
!kvm_arch_allow_write_without_running_vcpu(kvm));
virt/kvm/kvm_main.c
3531
if (kvm->dirty_ring_size && vcpu)
virt/kvm/kvm_main.c
3539
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
virt/kvm/kvm_main.c
3543
memslot = gfn_to_memslot(kvm, gfn);
virt/kvm/kvm_main.c
3544
mark_page_dirty_in_slot(kvm, memslot, gfn);
virt/kvm/kvm_main.c
3553
mark_page_dirty_in_slot(vcpu->kvm, memslot, gfn);
virt/kvm/kvm_main.c
3621
int idx = srcu_read_lock(&vcpu->kvm->srcu);
virt/kvm/kvm_main.c
3634
srcu_read_unlock(&vcpu->kvm->srcu, idx);
virt/kvm/kvm_main.c
3699
struct kvm *kvm = vcpu->kvm;
virt/kvm/kvm_main.c
3701
if (kvm->override_halt_poll_ns) {
virt/kvm/kvm_main.c
3709
return READ_ONCE(kvm->max_halt_poll_ns);
virt/kvm/kvm_main.c
3963
struct kvm *kvm = me->kvm;
virt/kvm/kvm_main.c
3967
nr_vcpus = atomic_read(&kvm->online_vcpus);
virt/kvm/kvm_main.c
3995
start = READ_ONCE(kvm->last_boosted_vcpu) + 1;
virt/kvm/kvm_main.c
4001
vcpu = xa_load(&kvm->vcpu_array, idx);
virt/kvm/kvm_main.c
4023
WRITE_ONCE(kvm->last_boosted_vcpu, idx);
virt/kvm/kvm_main.c
4036
static bool kvm_page_in_dirty_ring(struct kvm *kvm, unsigned long pgoff)
virt/kvm/kvm_main.c
4041
kvm->dirty_ring_size / PAGE_SIZE);
virt/kvm/kvm_main.c
4060
page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
virt/kvm/kvm_main.c
4062
else if (kvm_page_in_dirty_ring(vcpu->kvm, vmf->pgoff))
virt/kvm/kvm_main.c
4082
if ((kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff) ||
virt/kvm/kvm_main.c
4083
kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff + pages - 1)) &&
virt/kvm/kvm_main.c
4095
kvm_put_kvm(vcpu->kvm);
virt/kvm/kvm_main.c
4141
vcpu->kvm->debugfs_dentry);
virt/kvm/kvm_main.c
4152
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id)
virt/kvm/kvm_main.c
4170
mutex_lock(&kvm->lock);
virt/kvm/kvm_main.c
4171
if (kvm->created_vcpus >= kvm->max_vcpus) {
virt/kvm/kvm_main.c
4172
mutex_unlock(&kvm->lock);
virt/kvm/kvm_main.c
4176
r = kvm_arch_vcpu_precreate(kvm, id);
virt/kvm/kvm_main.c
4178
mutex_unlock(&kvm->lock);
virt/kvm/kvm_main.c
4182
kvm->created_vcpus++;
virt/kvm/kvm_main.c
4183
mutex_unlock(&kvm->lock);
virt/kvm/kvm_main.c
4199
kvm_vcpu_init(vcpu, kvm, id);
virt/kvm/kvm_main.c
4205
if (kvm->dirty_ring_size) {
virt/kvm/kvm_main.c
4206
r = kvm_dirty_ring_alloc(kvm, &vcpu->dirty_ring,
virt/kvm/kvm_main.c
4207
id, kvm->dirty_ring_size);
virt/kvm/kvm_main.c
4212
mutex_lock(&kvm->lock);
virt/kvm/kvm_main.c
4214
if (kvm_get_vcpu_by_id(kvm, id)) {
virt/kvm/kvm_main.c
4219
vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus);
virt/kvm/kvm_main.c
4220
r = xa_insert(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, GFP_KERNEL_ACCOUNT);
virt/kvm/kvm_main.c
4234
kvm_get_kvm(kvm);
virt/kvm/kvm_main.c
4244
atomic_inc(&kvm->online_vcpus);
virt/kvm/kvm_main.c
4247
mutex_unlock(&kvm->lock);
virt/kvm/kvm_main.c
4254
kvm_put_kvm_no_destroy(kvm);
virt/kvm/kvm_main.c
4255
xa_erase(&kvm->vcpu_array, vcpu->vcpu_idx);
virt/kvm/kvm_main.c
4257
mutex_unlock(&kvm->lock);
virt/kvm/kvm_main.c
4266
mutex_lock(&kvm->lock);
virt/kvm/kvm_main.c
4267
kvm->created_vcpus--;
virt/kvm/kvm_main.c
4268
mutex_unlock(&kvm->lock);
virt/kvm/kvm_main.c
4297
kvm_put_kvm(vcpu->kvm);
virt/kvm/kvm_main.c
4327
kvm_get_kvm(vcpu->kvm);
virt/kvm/kvm_main.c
4350
idx = srcu_read_lock(&vcpu->kvm->srcu);
virt/kvm/kvm_main.c
4371
srcu_read_unlock(&vcpu->kvm->srcu, idx);
virt/kvm/kvm_main.c
4381
struct kvm *kvm = vcpu->kvm;
virt/kvm/kvm_main.c
4387
if (likely(vcpu->vcpu_idx < atomic_read(&kvm->online_vcpus)))
virt/kvm/kvm_main.c
4400
if (WARN_ON_ONCE(!kvm_get_vcpu(kvm, vcpu->vcpu_idx)))
virt/kvm/kvm_main.c
441
static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
virt/kvm/kvm_main.c
4415
if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead)
virt/kvm/kvm_main.c
445
vcpu->kvm = kvm;
virt/kvm/kvm_main.c
4664
if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead)
virt/kvm/kvm_main.c
4730
if (dev->kvm->mm != current->mm || dev->kvm->vm_dead)
virt/kvm/kvm_main.c
4751
struct kvm *kvm = dev->kvm;
virt/kvm/kvm_main.c
4754
mutex_lock(&kvm->lock);
virt/kvm/kvm_main.c
4758
mutex_unlock(&kvm->lock);
virt/kvm/kvm_main.c
4761
kvm_put_kvm(kvm);
virt/kvm/kvm_main.c
4805
static int kvm_ioctl_create_device(struct kvm *kvm,
virt/kvm/kvm_main.c
482
void kvm_destroy_vcpus(struct kvm *kvm)
virt/kvm/kvm_main.c
4830
dev->kvm = kvm;
virt/kvm/kvm_main.c
4832
mutex_lock(&kvm->lock);
virt/kvm/kvm_main.c
4835
mutex_unlock(&kvm->lock);
virt/kvm/kvm_main.c
4839
list_add_rcu(&dev->vm_node, &kvm->devices);
virt/kvm/kvm_main.c
4840
mutex_unlock(&kvm->lock);
virt/kvm/kvm_main.c
4845
kvm_get_kvm(kvm);
virt/kvm/kvm_main.c
4848
kvm_put_kvm_no_destroy(kvm);
virt/kvm/kvm_main.c
4849
mutex_lock(&kvm->lock);
virt/kvm/kvm_main.c
4854
mutex_unlock(&kvm->lock);
virt/kvm/kvm_main.c
4864
static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
virt/kvm/kvm_main.c
487
kvm_for_each_vcpu(i, vcpu, kvm) {
virt/kvm/kvm_main.c
489
xa_erase(&kvm->vcpu_array, i);
virt/kvm/kvm_main.c
4900
if (kvm)
virt/kvm/kvm_main.c
4901
return kvm_arch_nr_memslot_as_ids(kvm);
virt/kvm/kvm_main.c
4927
return kvm_supported_mem_attributes(kvm);
virt/kvm/kvm_main.c
4933
return kvm_gmem_get_supported_flags(kvm);
virt/kvm/kvm_main.c
4938
return kvm_vm_ioctl_check_extension(kvm, arg);
virt/kvm/kvm_main.c
4941
static int kvm_vm_ioctl_enable_dirty_log_ring(struct kvm *kvm, u32 size)
virt/kvm/kvm_main.c
4953
if (size < kvm_dirty_ring_get_rsvd_entries(kvm) *
virt/kvm/kvm_main.c
4962
if (kvm->dirty_ring_size)
virt/kvm/kvm_main.c
4965
mutex_lock(&kvm->lock);
virt/kvm/kvm_main.c
4967
if (kvm->created_vcpus) {
virt/kvm/kvm_main.c
497
WARN_ON_ONCE(xa_load(&kvm->vcpu_array, i) || kvm_get_vcpu(kvm, i));
virt/kvm/kvm_main.c
4971
kvm->dirty_ring_size = size;
virt/kvm/kvm_main.c
4975
mutex_unlock(&kvm->lock);
virt/kvm/kvm_main.c
4979
static int kvm_vm_ioctl_reset_dirty_pages(struct kvm *kvm)
virt/kvm/kvm_main.c
4985
if (!kvm->dirty_ring_size)
virt/kvm/kvm_main.c
4988
mutex_lock(&kvm->slots_lock);
virt/kvm/kvm_main.c
4990
kvm_for_each_vcpu(i, vcpu, kvm) {
virt/kvm/kvm_main.c
4991
r = kvm_dirty_ring_reset(vcpu->kvm, &vcpu->dirty_ring, &cleared);
virt/kvm/kvm_main.c
4996
mutex_unlock(&kvm->slots_lock);
virt/kvm/kvm_main.c
4999
kvm_flush_remote_tlbs(kvm);
virt/kvm/kvm_main.c
500
atomic_set(&kvm->online_vcpus, 0);
virt/kvm/kvm_main.c
5004
int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm,
virt/kvm/kvm_main.c
5010
bool kvm_are_all_memslots_empty(struct kvm *kvm)
virt/kvm/kvm_main.c
5014
lockdep_assert_held(&kvm->slots_lock);
virt/kvm/kvm_main.c
5016
for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
virt/kvm/kvm_main.c
5017
if (!kvm_memslots_empty(__kvm_memslots(kvm, i)))
virt/kvm/kvm_main.c
5025
static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,
virt/kvm/kvm_main.c
5038
kvm->manual_dirty_log_protect = cap->args[0];
virt/kvm/kvm_main.c
504
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
virt/kvm/kvm_main.c
5046
kvm->max_halt_poll_ns = cap->args[0];
virt/kvm/kvm_main.c
5055
kvm->override_halt_poll_ns = true;
virt/kvm/kvm_main.c
506
return container_of(mn, struct kvm, mmu_notifier);
virt/kvm/kvm_main.c
5061
if (!kvm_vm_ioctl_check_extension_generic(kvm, cap->cap))
virt/kvm/kvm_main.c
5064
return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]);
virt/kvm/kvm_main.c
5069
!kvm->dirty_ring_size || cap->flags)
virt/kvm/kvm_main.c
5072
mutex_lock(&kvm->slots_lock);
virt/kvm/kvm_main.c
5079
if (kvm_are_all_memslots_empty(kvm)) {
virt/kvm/kvm_main.c
5080
kvm->dirty_ring_with_bitmap = true;
virt/kvm/kvm_main.c
5084
mutex_unlock(&kvm->slots_lock);
virt/kvm/kvm_main.c
5089
return kvm_vm_ioctl_enable_cap(kvm, cap);
virt/kvm/kvm_main.c
509
typedef bool (*gfn_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);
virt/kvm/kvm_main.c
5096
struct kvm *kvm = file->private_data;
virt/kvm/kvm_main.c
5098
return kvm_stats_read(kvm->stats_id, &kvm_vm_stats_header,
virt/kvm/kvm_main.c
5099
&kvm_vm_stats_desc[0], &kvm->stat,
virt/kvm/kvm_main.c
5100
sizeof(kvm->stat), user_buffer, size, offset);
virt/kvm/kvm_main.c
5105
struct kvm *kvm = file->private_data;
virt/kvm/kvm_main.c
5107
kvm_put_kvm(kvm);
virt/kvm/kvm_main.c
511
typedef void (*on_lock_fn_t)(struct kvm *kvm);
virt/kvm/kvm_main.c
5118
static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm)
virt/kvm/kvm_main.c
5128
&kvm_vm_stats_fops, kvm, O_RDONLY, FMODE_PREAD);
virt/kvm/kvm_main.c
5134
kvm_get_kvm(kvm);
virt/kvm/kvm_main.c
5151
struct kvm *kvm = filp->private_data;
virt/kvm/kvm_main.c
5155
if (kvm->mm != current->mm || kvm->vm_dead)
virt/kvm/kvm_main.c
5159
r = kvm_vm_ioctl_create_vcpu(kvm, arg);
virt/kvm/kvm_main.c
5167
r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);
virt/kvm/kvm_main.c
5202
r = kvm_vm_ioctl_set_memory_region(kvm, &mem);
virt/kvm/kvm_main.c
5211
r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
virt/kvm/kvm_main.c
5221
r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
virt/kvm/kvm_main.c
5232
r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
virt/kvm/kvm_main.c
5241
r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
virt/kvm/kvm_main.c
5251
r = kvm_irqfd(kvm, &data);
virt/kvm/kvm_main.c
5260
r = kvm_ioeventfd(kvm, &data);
virt/kvm/kvm_main.c
5270
r = kvm_send_userspace_msi(kvm, &msi);
virt/kvm/kvm_main.c
5283
r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
virt/kvm/kvm_main.c
5308
if (!kvm_arch_can_set_irq_routing(kvm))
virt/kvm/kvm_main.c
5323
r = kvm_set_irq_routing(kvm, entries, routing.nr,
virt/kvm/kvm_main.c
5337
r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs);
virt/kvm/kvm_main.c
5348
r = kvm_ioctl_create_device(kvm, &cd);
virt/kvm/kvm_main.c
5360
r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
virt/kvm/kvm_main.c
5363
r = kvm_vm_ioctl_reset_dirty_pages(kvm);
virt/kvm/kvm_main.c
5366
r = kvm_vm_ioctl_get_stats_fd(kvm);
virt/kvm/kvm_main.c
5376
r = kvm_gmem_create(kvm, &guest_memfd);
virt/kvm/kvm_main.c
5416
struct kvm *kvm = filp->private_data;
virt/kvm/kvm_main.c
5419
if (kvm->mm != current->mm || kvm->vm_dead)
virt/kvm/kvm_main.c
5441
r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
virt/kvm/kvm_main.c
5457
r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
virt/kvm/kvm_main.c
5484
struct kvm *kvm;
virt/kvm/kvm_main.c
5493
kvm = kvm_create_vm(type, fdname);
virt/kvm/kvm_main.c
5494
if (IS_ERR(kvm)) {
virt/kvm/kvm_main.c
5495
r = PTR_ERR(kvm);
virt/kvm/kvm_main.c
5499
file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
virt/kvm/kvm_main.c
5511
kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
virt/kvm/kvm_main.c
5517
kvm_put_kvm(kvm);
virt/kvm/kvm_main.c
560
static __always_inline kvm_mn_ret_t kvm_handle_hva_range(struct kvm *kvm,
virt/kvm/kvm_main.c
584
idx = srcu_read_lock(&kvm->srcu);
virt/kvm/kvm_main.c
586
for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
virt/kvm/kvm_main.c
5869
static struct kvm_io_bus *kvm_get_bus_srcu(struct kvm *kvm, enum kvm_bus idx)
virt/kvm/kvm_main.c
5878
return srcu_dereference(kvm->buses[idx], &kvm->srcu);
virt/kvm/kvm_main.c
589
slots = __kvm_memslots(kvm, i);
virt/kvm/kvm_main.c
5893
bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
virt/kvm/kvm_main.c
5912
bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
virt/kvm/kvm_main.c
5962
bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
virt/kvm/kvm_main.c
5977
int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
virt/kvm/kvm_main.c
5984
lockdep_assert_held(&kvm->slots_lock);
virt/kvm/kvm_main.c
5986
bus = kvm_get_bus(kvm, bus_idx);
virt/kvm/kvm_main.c
6014
rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
virt/kvm/kvm_main.c
6015
call_srcu(&kvm->srcu, &bus->rcu, __free_bus);
virt/kvm/kvm_main.c
6020
int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
virt/kvm/kvm_main.c
6026
lockdep_assert_held(&kvm->slots_lock);
virt/kvm/kvm_main.c
6028
bus = kvm_get_bus(kvm, bus_idx);
virt/kvm/kvm_main.c
6050
rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
virt/kvm/kvm_main.c
6051
synchronize_srcu_expedited(&kvm->srcu);
virt/kvm/kvm_main.c
6068
struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
virt/kvm/kvm_main.c
6074
lockdep_assert_held(&kvm->srcu);
virt/kvm/kvm_main.c
6076
bus = kvm_get_bus_srcu(kvm, bus_idx);
virt/kvm/kvm_main.c
6100
if (!kvm_get_kvm_safe(stat_data->kvm))
virt/kvm/kvm_main.c
6107
kvm_put_kvm(stat_data->kvm);
virt/kvm/kvm_main.c
6117
kvm_put_kvm(stat_data->kvm);
virt/kvm/kvm_main.c
6122
static int kvm_get_stat_per_vm(struct kvm *kvm, size_t offset, u64 *val)
virt/kvm/kvm_main.c
6124
*val = *(u64 *)((void *)(&kvm->stat) + offset);
virt/kvm/kvm_main.c
6129
static int kvm_clear_stat_per_vm(struct kvm *kvm, size_t offset)
virt/kvm/kvm_main.c
6131
*(u64 *)((void *)(&kvm->stat) + offset) = 0;
virt/kvm/kvm_main.c
6136
static int kvm_get_stat_per_vcpu(struct kvm *kvm, size_t offset, u64 *val)
virt/kvm/kvm_main.c
6143
kvm_for_each_vcpu(i, vcpu, kvm)
virt/kvm/kvm_main.c
6149
static int kvm_clear_stat_per_vcpu(struct kvm *kvm, size_t offset)
virt/kvm/kvm_main.c
6154
kvm_for_each_vcpu(i, vcpu, kvm)
virt/kvm/kvm_main.c
6167
r = kvm_get_stat_per_vm(stat_data->kvm,
virt/kvm/kvm_main.c
6171
r = kvm_get_stat_per_vcpu(stat_data->kvm,
virt/kvm/kvm_main.c
6189
r = kvm_clear_stat_per_vm(stat_data->kvm,
virt/kvm/kvm_main.c
6193
r = kvm_clear_stat_per_vcpu(stat_data->kvm,
virt/kvm/kvm_main.c
6219
struct kvm *kvm;
virt/kvm/kvm_main.c
6224
list_for_each_entry(kvm, &vm_list, vm_list) {
virt/kvm/kvm_main.c
6225
kvm_get_stat_per_vm(kvm, offset, &tmp_val);
virt/kvm/kvm_main.c
6235
struct kvm *kvm;
virt/kvm/kvm_main.c
6241
list_for_each_entry(kvm, &vm_list, vm_list) {
virt/kvm/kvm_main.c
6242
kvm_clear_stat_per_vm(kvm, offset);
virt/kvm/kvm_main.c
625
KVM_MMU_LOCK(kvm);
virt/kvm/kvm_main.c
6255
struct kvm *kvm;
virt/kvm/kvm_main.c
6260
list_for_each_entry(kvm, &vm_list, vm_list) {
virt/kvm/kvm_main.c
6261
kvm_get_stat_per_vcpu(kvm, offset, &tmp_val);
virt/kvm/kvm_main.c
627
range->on_lock(kvm);
virt/kvm/kvm_main.c
6271
struct kvm *kvm;
virt/kvm/kvm_main.c
6277
list_for_each_entry(kvm, &vm_list, vm_list) {
virt/kvm/kvm_main.c
6278
kvm_clear_stat_per_vcpu(kvm, offset);
virt/kvm/kvm_main.c
6289
static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
virt/kvm/kvm_main.c
6294
if (!kvm_dev.this_device || !kvm)
virt/kvm/kvm_main.c
6317
kvm->userspace_pid = task_pid_nr(current);
virt/kvm/kvm_main.c
6321
add_uevent_var(env, "PID=%d", kvm->userspace_pid);
virt/kvm/kvm_main.c
6323
if (!IS_ERR(kvm->debugfs_dentry)) {
virt/kvm/kvm_main.c
6327
tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
virt/kvm/kvm_main.c
633
r.ret |= range->handler(kvm, &gfn_range);
virt/kvm/kvm_main.c
638
kvm_flush_remote_tlbs(kvm);
virt/kvm/kvm_main.c
642
KVM_MMU_UNLOCK(kvm);
virt/kvm/kvm_main.c
644
srcu_read_unlock(&kvm->srcu, idx);
virt/kvm/kvm_main.c
653
struct kvm *kvm = mmu_notifier_to_kvm(mn);
virt/kvm/kvm_main.c
664
return kvm_handle_hva_range(kvm, &range).ret;
virt/kvm/kvm_main.c
673
void kvm_mmu_invalidate_start(struct kvm *kvm)
virt/kvm/kvm_main.c
675
lockdep_assert_held_write(&kvm->mmu_lock);
virt/kvm/kvm_main.c
681
kvm->mmu_invalidate_in_progress++;
virt/kvm/kvm_main.c
683
if (likely(kvm->mmu_invalidate_in_progress == 1)) {
virt/kvm/kvm_main.c
684
kvm->mmu_invalidate_range_start = INVALID_GPA;
virt/kvm/kvm_main.c
685
kvm->mmu_invalidate_range_end = INVALID_GPA;
virt/kvm/kvm_main.c
689
void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end)
virt/kvm/kvm_main.c
691
lockdep_assert_held_write(&kvm->mmu_lock);
virt/kvm/kvm_main.c
693
WARN_ON_ONCE(!kvm->mmu_invalidate_in_progress);
virt/kvm/kvm_main.c
695
if (likely(kvm->mmu_invalidate_range_start == INVALID_GPA)) {
virt/kvm/kvm_main.c
696
kvm->mmu_invalidate_range_start = start;
virt/kvm/kvm_main.c
697
kvm->mmu_invalidate_range_end = end;
virt/kvm/kvm_main.c
708
kvm->mmu_invalidate_range_start =
virt/kvm/kvm_main.c
709
min(kvm->mmu_invalidate_range_start, start);
virt/kvm/kvm_main.c
710
kvm->mmu_invalidate_range_end =
virt/kvm/kvm_main.c
711
max(kvm->mmu_invalidate_range_end, end);
virt/kvm/kvm_main.c
715
bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
virt/kvm/kvm_main.c
717
kvm_mmu_invalidate_range_add(kvm, range->start, range->end);
virt/kvm/kvm_main.c
718
return kvm_unmap_gfn_range(kvm, range);
virt/kvm/kvm_main.c
724
struct kvm *kvm = mmu_notifier_to_kvm(mn);
virt/kvm/kvm_main.c
744
spin_lock(&kvm->mn_invalidate_lock);
virt/kvm/kvm_main.c
745
kvm->mn_active_invalidate_count++;
virt/kvm/kvm_main.c
746
spin_unlock(&kvm->mn_invalidate_lock);
virt/kvm/kvm_main.c
758
gfn_to_pfn_cache_invalidate_start(kvm, range->start, range->end);
virt/kvm/kvm_main.c
765
if (kvm_handle_hva_range(kvm, &hva_range).found_memslot)
virt/kvm/kvm_main.c
766
kvm_arch_guest_memory_reclaimed(kvm);
virt/kvm/kvm_main.c
771
void kvm_mmu_invalidate_end(struct kvm *kvm)
virt/kvm/kvm_main.c
773
lockdep_assert_held_write(&kvm->mmu_lock);
virt/kvm/kvm_main.c
780
kvm->mmu_invalidate_seq++;
virt/kvm/kvm_main.c
787
kvm->mmu_invalidate_in_progress--;
virt/kvm/kvm_main.c
788
KVM_BUG_ON(kvm->mmu_invalidate_in_progress < 0, kvm);
virt/kvm/kvm_main.c
794
WARN_ON_ONCE(kvm->mmu_invalidate_range_start == INVALID_GPA);
virt/kvm/kvm_main.c
800
struct kvm *kvm = mmu_notifier_to_kvm(mn);
virt/kvm/kvm_main.c
811
kvm_handle_hva_range(kvm, &hva_range);
virt/kvm/kvm_main.c
814
spin_lock(&kvm->mn_invalidate_lock);
virt/kvm/kvm_main.c
815
if (!WARN_ON_ONCE(!kvm->mn_active_invalidate_count))
virt/kvm/kvm_main.c
816
--kvm->mn_active_invalidate_count;
virt/kvm/kvm_main.c
817
wake = !kvm->mn_active_invalidate_count;
virt/kvm/kvm_main.c
818
spin_unlock(&kvm->mn_invalidate_lock);
virt/kvm/kvm_main.c
825
rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait);
virt/kvm/kvm_main.c
870
struct kvm *kvm = mmu_notifier_to_kvm(mn);
virt/kvm/kvm_main.c
873
idx = srcu_read_lock(&kvm->srcu);
virt/kvm/kvm_main.c
874
kvm_flush_shadow_all(kvm);
virt/kvm/kvm_main.c
875
srcu_read_unlock(&kvm->srcu, idx);
virt/kvm/kvm_main.c
887
static int kvm_init_mmu_notifier(struct kvm *kvm)
virt/kvm/kvm_main.c
889
kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
virt/kvm/kvm_main.c
890
return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
virt/kvm/kvm_main.c
898
struct kvm *kvm = container_of(bl, struct kvm, pm_notifier);
virt/kvm/kvm_main.c
900
return kvm_arch_pm_notifier(kvm, state);
virt/kvm/kvm_main.c
903
static void kvm_init_pm_notifier(struct kvm *kvm)
virt/kvm/kvm_main.c
905
kvm->pm_notifier.notifier_call = kvm_pm_notifier_call;
virt/kvm/kvm_main.c
907
kvm->pm_notifier.priority = INT_MAX;
virt/kvm/kvm_main.c
908
register_pm_notifier(&kvm->pm_notifier);
virt/kvm/kvm_main.c
911
static void kvm_destroy_pm_notifier(struct kvm *kvm)
virt/kvm/kvm_main.c
913
unregister_pm_notifier(&kvm->pm_notifier);
virt/kvm/kvm_main.c
916
static void kvm_init_pm_notifier(struct kvm *kvm)
virt/kvm/kvm_main.c
920
static void kvm_destroy_pm_notifier(struct kvm *kvm)
virt/kvm/kvm_main.c
935
static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
virt/kvm/kvm_main.c
942
kvm_arch_free_memslot(kvm, slot);
virt/kvm/kvm_main.c
947
static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
virt/kvm/kvm_main.c
963
kvm_free_memslot(kvm, memslot);
virt/kvm/kvm_main.c
979
static void kvm_destroy_vm_debugfs(struct kvm *kvm)
virt/kvm/kvm_main.c
985
if (IS_ERR(kvm->debugfs_dentry))
virt/kvm/kvm_main.c
988
debugfs_remove_recursive(kvm->debugfs_dentry);
virt/kvm/kvm_main.c
990
if (kvm->debugfs_stat_data) {
virt/kvm/kvm_main.c
992
kfree(kvm->debugfs_stat_data[i]);
virt/kvm/kvm_main.c
993
kfree(kvm->debugfs_stat_data);
virt/kvm/kvm_main.c
997
static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname)
virt/kvm/kvm_mm.h
17
#define KVM_MMU_LOCK_INIT(kvm) rwlock_init(&(kvm)->mmu_lock)
virt/kvm/kvm_mm.h
18
#define KVM_MMU_LOCK(kvm) write_lock(&(kvm)->mmu_lock)
virt/kvm/kvm_mm.h
19
#define KVM_MMU_UNLOCK(kvm) write_unlock(&(kvm)->mmu_lock)
virt/kvm/kvm_mm.h
21
#define KVM_MMU_LOCK_INIT(kvm) spin_lock_init(&(kvm)->mmu_lock)
virt/kvm/kvm_mm.h
22
#define KVM_MMU_LOCK(kvm) spin_lock(&(kvm)->mmu_lock)
virt/kvm/kvm_mm.h
23
#define KVM_MMU_UNLOCK(kvm) spin_unlock(&(kvm)->mmu_lock)
virt/kvm/kvm_mm.h
62
void gfn_to_pfn_cache_invalidate_start(struct kvm *kvm,
virt/kvm/kvm_mm.h
66
static inline void gfn_to_pfn_cache_invalidate_start(struct kvm *kvm,
virt/kvm/kvm_mm.h
76
int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args);
virt/kvm/kvm_mm.h
77
int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
virt/kvm/kvm_mm.h
86
static inline int kvm_gmem_bind(struct kvm *kvm,
virt/kvm/pfncache.c
127
static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long mmu_seq)
virt/kvm/pfncache.c
141
if (kvm->mn_active_invalidate_count)
virt/kvm/pfncache.c
152
return kvm->mmu_invalidate_seq != mmu_seq;
virt/kvm/pfncache.c
184
mmu_seq = gpc->kvm->mmu_invalidate_seq;
virt/kvm/pfncache.c
235
} while (mmu_notifier_retry_cache(gpc->kvm, mmu_seq));
virt/kvm/pfncache.c
25
void gfn_to_pfn_cache_invalidate_start(struct kvm *kvm, unsigned long start,
virt/kvm/pfncache.c
293
struct kvm_memslots *slots = kvm_memslots(gpc->kvm);
virt/kvm/pfncache.c
30
spin_lock(&kvm->gpc_lock);
virt/kvm/pfncache.c
31
list_for_each_entry(gpc, &kvm->gpc_list, list) {
virt/kvm/pfncache.c
385
void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm)
virt/kvm/pfncache.c
390
gpc->kvm = kvm;
virt/kvm/pfncache.c
400
struct kvm *kvm = gpc->kvm;
virt/kvm/pfncache.c
408
if (KVM_BUG_ON(gpc->valid, kvm))
virt/kvm/pfncache.c
411
spin_lock(&kvm->gpc_lock);
virt/kvm/pfncache.c
412
list_add(&gpc->list, &kvm->gpc_list);
virt/kvm/pfncache.c
413
spin_unlock(&kvm->gpc_lock);
virt/kvm/pfncache.c
449
struct kvm *kvm = gpc->kvm;
virt/kvm/pfncache.c
478
spin_lock(&kvm->gpc_lock);
virt/kvm/pfncache.c
480
spin_unlock(&kvm->gpc_lock);
virt/kvm/pfncache.c
57
spin_unlock(&kvm->gpc_lock);
virt/kvm/pfncache.c
75
struct kvm_memslots *slots = kvm_memslots(gpc->kvm);
virt/kvm/vfio.c
100
static void kvm_spapr_tce_release_vfio_group(struct kvm *kvm,
virt/kvm/vfio.c
106
kvm_spapr_tce_release_iommu_group(kvm, kvf->iommu_group);
virt/kvm/vfio.c
137
kvm_arch_register_noncoherent_dma(dev->kvm);
virt/kvm/vfio.c
139
kvm_arch_unregister_noncoherent_dma(dev->kvm);
virt/kvm/vfio.c
171
kvm_vfio_file_set_kvm(kvf->file, dev->kvm);
virt/kvm/vfio.c
180
kvm_spapr_tce_release_vfio_group(dev->kvm, kvf);
virt/kvm/vfio.c
237
return kvm_spapr_tce_attach_iommu_group(dev->kvm, param.tablefd,
virt/kvm/vfio.c
332
lockdep_assert_held(&dev->kvm->lock);
virt/kvm/vfio.c
335
list_for_each_entry(tmp, &dev->kvm->devices, vm_node)
virt/kvm/vfio.c
38
static void kvm_vfio_file_set_kvm(struct file *file, struct kvm *kvm)
virt/kvm/vfio.c
40
void (*fn)(struct file *file, struct kvm *kvm);
virt/kvm/vfio.c
46
fn(file, kvm);