gmap
struct gmap *gmap;
struct gmap *gmap;
rc = gmap_link(mc, kvm->arch.gmap, f);
if (gmap_try_fixup_minor(kvm->arch.gmap, f) == 0)
static int walk_guest_tables(struct gmap *sg, unsigned long saddr, struct pgtwalk *w, bool wr)
struct gmap *parent = sg->parent;
static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union pte *ptep,
static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, union crste *table,
static int _gaccess_do_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *sg,
static inline int _gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr,
struct gmap *parent;
static int __gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr,
int gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr,
r = dat_get_storage_key(kvm->arch.gmap->asce, gpa_to_gfn(gpa), &storage_key);
r = dat_get_storage_key(vcpu->arch.gmap->asce, gpa_to_gfn(gpa), &storage_key);
int gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr,
int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn,
void gmap_set_cmma_all_dirty(struct gmap *gmap)
scoped_guard(read_lock, &gmap->kvm->mmu_lock)
gfn = _dat_walk_gfn_range(gfn, asce_end(gmap->asce), gmap->asce, &ops,
static void gmap_unshadow_level(struct gmap *sg, gfn_t r_gfn, int level)
static void gmap_unshadow(struct gmap *sg)
gmap_cache->gmap = NULL;
void _gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn)
struct gmap *sg, *next;
struct gmap *gmap_new_child(struct gmap *parent, gfn_t limit)
static struct gmap *gmap_find_shadow(struct gmap *parent, union asce asce, int edat_level)
struct gmap *sg;
struct gmap *res;
static inline int __gmap_protect_asce_top_level(struct kvm_s390_mmu_cache *mc, struct gmap *sg,
struct gmap *parent;
static inline int _gmap_protect_asce_top_level(struct kvm_s390_mmu_cache *mc, struct gmap *sg,
static int gmap_protect_asce_top_level(struct kvm_s390_mmu_cache *mc, struct gmap *sg)
struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *parent,
struct gmap *sg, *new;
int gmap_set_limit(struct gmap *gmap, gfn_t limit)
scoped_guard(write_lock, &gmap->kvm->mmu_lock)
rc = dat_set_asce_limit(mc, &gmap->asce, type);
void gmap_remove_child(struct gmap *child)
void gmap_dispose(struct gmap *gmap)
KVM_BUG_ON(gmap->parent, gmap->kvm);
KVM_BUG_ON(!list_empty(&gmap->children), gmap->kvm);
KVM_BUG_ON(!list_empty(&gmap->scb_users), gmap->kvm);
KVM_BUG_ON(!gmap->asce.val, gmap->kvm);
KVM_BUG_ON(refcount_read(&gmap->refcount), gmap->kvm);
asce_flush_tlb(gmap->asce);
dat_free_level(dereference_asce(gmap->asce), owns_page_tables(gmap));
if (is_shadow(gmap))
gmap_rmap_radix_tree_free(&gmap->host_to_rmap);
kfree(gmap);
int s390_replace_asce(struct gmap *gmap)
if (gmap->asce.dt == ASCE_TYPE_SEGMENT)
memcpy(table, dereference_asce(gmap->asce), sizeof(*table));
asce = gmap->asce;
WRITE_ONCE(gmap->asce, asce);
bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint)
struct kvm *kvm = gmap->kvm;
if (is_shadow(gmap))
struct gmap *gmap;
if (!pgste.prefix_notif || gmap_mkold_prefix(p->gmap, gfn, end)) {
pgste = __dat_ptep_xchg(ptep, pgste, new, gfn, walk->asce, uses_skeys(p->gmap));
if (crste_prefix(crste) && !gmap_mkold_prefix(priv->gmap, gfn, end))
bool gmap_age_gfn(struct gmap *gmap, gfn_t start, gfn_t end)
.gmap = gmap,
_dat_walk_gfn_range(start, end, gmap->asce, &ops, 0, &priv);
struct gmap *gmap;
gmap_helper_try_set_pte_unused(priv->gmap->kvm->mm, vmaddr);
if (ptep->s.pr && test_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &priv->gmap->flags))
pgste = gmap_ptep_xchg(priv->gmap, ptep, _PTE_EMPTY, pgste, gfn);
if (old.s.fc1.pr && test_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &priv->gmap->flags))
KVM_BUG_ON(!gmap_crstep_xchg_atomic(priv->gmap, crstep, old, _CRSTE_EMPTY(old.h.tt), gfn),
priv->gmap->kvm);
bool gmap_unmap_gfn_range(struct gmap *gmap, struct kvm_memory_slot *slot, gfn_t start, gfn_t end)
.gmap = gmap,
lockdep_assert_held_write(&gmap->kvm->mmu_lock);
_dat_walk_gfn_range(start, end, gmap->asce, &ops, 0, &priv);
struct gmap *gmap)
if (!pgste.prefix_notif || gmap_mkold_prefix(gmap, gfn, gfn + 1)) {
pgste = gmap_ptep_xchg(gmap, ptep, pte, pgste, gfn);
mark_page_dirty(gmap->kvm, gfn);
struct gmap *gmap = walk->priv;
pgste = __pte_test_and_clear_softdirty(ptep, pgste, gfn, gmap);
struct gmap *gmap = walk->priv;
if (crste.s.fc1.prefix_notif && !gmap_mkold_prefix(gmap, gfn, end))
} while (!gmap_crstep_xchg_atomic(gmap, table, crste, new, gfn));
mark_page_dirty(gmap->kvm, gfn);
void gmap_sync_dirty_log(struct gmap *gmap, gfn_t start, gfn_t end)
lockdep_assert_held(&gmap->kvm->mmu_lock);
_dat_walk_gfn_range(start, end, gmap->asce, &walk_ops, 0, gmap);
struct gmap *gmap_new(struct kvm *kvm, gfn_t limit)
static int gmap_handle_minor_crste_fault(struct gmap *gmap, struct guest_fault *f)
return !gmap_crstep_xchg_atomic(gmap, f->crstep, oldcrste, newcrste, f->gfn);
struct gmap *gmap;
static int _gmap_handle_minor_pte_fault(struct gmap *gmap, union pgste *pgste,
*pgste = gmap_ptep_xchg(gmap, f->ptep, newpte, *pgste, f->gfn);
int gmap_try_fixup_minor(struct gmap *gmap, struct guest_fault *fault)
lockdep_assert_held(&gmap->kvm->mmu_lock);
gmap = kzalloc_obj(*gmap, GFP_KERNEL_ACCOUNT);
rc = dat_entry_walk(NULL, fault->gfn, gmap->asce, DAT_WALK_LEAF, TABLE_TYPE_PAGE_TABLE,
rc = _gmap_handle_minor_pte_fault(gmap, &pgste, fault);
if (!gmap)
rc = gmap_handle_minor_crste_fault(gmap, fault);
static inline bool gmap_2g_allowed(struct gmap *gmap, gfn_t gfn)
static inline bool gmap_1m_allowed(struct gmap *gmap, gfn_t gfn)
return test_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &gmap->flags);
INIT_LIST_HEAD(&gmap->children);
static int _gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, int level,
rc = dat_entry_walk(mc, f->gfn, gmap->asce, DAT_WALK_ALLOC_CONTINUE, level,
INIT_LIST_HEAD(&gmap->list);
if (KVM_BUG_ON(rc == -EINVAL, gmap->kvm))
if (KVM_BUG_ON(get_level(f->crstep, f->ptep) > level, gmap->kvm))
INIT_LIST_HEAD(&gmap->scb_users);
pgste = gmap_ptep_xchg(gmap, f->ptep, newpte, pgste, f->gfn);
INIT_RADIX_TREE(&gmap->host_to_rmap, GFP_KVM_S390_MMU_CACHE);
spin_lock_init(&gmap->children_lock);
} while (!gmap_crstep_xchg_atomic(gmap, f->crstep, oldval, newval, f->gfn));
spin_lock_init(&gmap->host_to_rmap_lock);
int gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, struct guest_fault *f)
lockdep_assert_held(&gmap->kvm->mmu_lock);
refcount_set(&gmap->refcount, 1);
if (order >= get_order(_REGION3_SIZE) && gmap_2g_allowed(gmap, f->gfn))
else if (order >= get_order(_SEGMENT_SIZE) && gmap_1m_allowed(gmap, f->gfn))
return _gmap_link(mc, gmap, level, f);
static int gmap_ucas_map_one(struct kvm_s390_mmu_cache *mc, struct gmap *gmap,
rc = dat_entry_walk(mc, p_gfn, gmap->parent->asce, DAT_WALK_ALLOC,
rc = dat_entry_walk(mc, p_gfn, gmap->parent->asce, DAT_WALK_ALLOC_CONTINUE,
rc = dat_entry_walk(mc, c_gfn, gmap->asce, DAT_WALK_ALLOC, TABLE_TYPE_SEGMENT,
kfree(gmap);
} while (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, c_gfn, gmap->asce));
static int gmap_ucas_translate_simple(struct gmap *gmap, gpa_t *gaddr, union crste **crstepp)
rc = dat_entry_walk(NULL, gpa_to_gfn(*gaddr), gmap->asce, DAT_WALK_CONTINUE,
int gmap_ucas_translate(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, gpa_t *gaddr)
gmap->asce.val = __pa(table);
scoped_guard(read_lock, &gmap->kvm->mmu_lock) {
rc = gmap_ucas_translate_simple(gmap, gaddr, &crstep);
gmap->asce.dt = type;
scoped_guard(write_lock, &gmap->kvm->mmu_lock) {
rc = gmap_ucas_translate_simple(gmap, gaddr, &crstep);
rc = gmap_ucas_map_one(mc, gmap, gpa_to_gfn(translated_address), gfn, true);
gmap->asce.tl = _ASCE_TABLE_LENGTH;
gmap->asce.x = 1;
int gmap_ucas_map(struct gmap *gmap, gfn_t p_gfn, gfn_t c_gfn, unsigned long count)
gmap->asce.p = 1;
scoped_guard(write_lock, &gmap->kvm->mmu_lock)
rc = gmap_ucas_map_one(mc, gmap, p_gfn, c_gfn, false);
gmap->asce.s = 1;
gmap->kvm = kvm;
static void gmap_ucas_unmap_one(struct gmap *gmap, gfn_t c_gfn)
rc = dat_entry_walk(NULL, c_gfn, gmap->asce, 0, TABLE_TYPE_SEGMENT, &crstep, &ptep);
while (!dat_crstep_xchg_atomic(crstep, READ_ONCE(*crstep), _PMD_EMPTY, c_gfn, gmap->asce))
set_bit(GMAP_FLAG_OWNS_PAGETABLES, &gmap->flags);
void gmap_ucas_unmap(struct gmap *gmap, gfn_t c_gfn, unsigned long count)
guard(read_lock)(&gmap->kvm->mmu_lock);
gmap_ucas_unmap_one(gmap, c_gfn);
struct gmap *gmap = walk->priv;
return gmap;
gmap_unmap_prefix(gmap, gfn, next);
gmap_handle_vsie_unshadow_event(gmap, gfn);
void gmap_split_huge_pages(struct gmap *gmap)
scoped_guard(read_lock, &gmap->kvm->mmu_lock)
start = _dat_walk_gfn_range(start, asce_end(gmap->asce), gmap->asce,
&ops, DAT_WALK_IGN_HOLES, gmap);
static void gmap_add_child(struct gmap *parent, struct gmap *child)
static int _gmap_enable_skeys(struct gmap *gmap)
if (uses_skeys(gmap))
set_bit(GMAP_FLAG_USES_SKEYS, &gmap->flags);
clear_bit(GMAP_FLAG_USES_SKEYS, &gmap->flags);
scoped_guard(write_lock, &gmap->kvm->mmu_lock)
start = dat_reset_skeys(gmap->asce, start);
int gmap_enable_skeys(struct gmap *gmap)
mmap_write_lock(gmap->kvm->mm);
rc = _gmap_enable_skeys(gmap);
mmap_write_unlock(gmap->kvm->mm);
int gmap_pv_destroy_range(struct gmap *gmap, gfn_t start, gfn_t end, bool interruptible)
scoped_guard(read_lock, &gmap->kvm->mmu_lock)
start = _dat_walk_gfn_range(start, end, gmap->asce, &ops,
int gmap_insert_rmap(struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, int level)
int gmap_pv_destroy_range(struct gmap *gmap, gfn_t start, gfn_t end, bool interruptible);
int gmap_insert_rmap(struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, int level);
int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn,
void gmap_set_cmma_all_dirty(struct gmap *gmap);
void _gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn);
struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *gmap,
void gmap_split_huge_pages(struct gmap *gmap);
static inline bool uses_skeys(struct gmap *gmap)
return test_bit(GMAP_FLAG_USES_SKEYS, &gmap->flags);
static inline bool uses_cmm(struct gmap *gmap)
return test_bit(GMAP_FLAG_USES_CMM, &gmap->flags);
static inline bool pfault_enabled(struct gmap *gmap)
return test_bit(GMAP_FLAG_PFAULT_ENABLED, &gmap->flags);
static inline bool is_ucontrol(struct gmap *gmap)
return test_bit(GMAP_FLAG_IS_UCONTROL, &gmap->flags);
static inline bool is_shadow(struct gmap *gmap)
return test_bit(GMAP_FLAG_SHADOW, &gmap->flags);
static inline bool owns_page_tables(struct gmap *gmap)
return test_bit(GMAP_FLAG_OWNS_PAGETABLES, &gmap->flags);
static inline struct gmap *gmap_put(struct gmap *gmap)
if (refcount_dec_and_test(&gmap->refcount))
gmap_dispose(gmap);
static inline void gmap_get(struct gmap *gmap)
WARN_ON_ONCE(unlikely(!refcount_inc_not_zero(&gmap->refcount)));
static inline void gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn)
static inline bool gmap_mkold_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end)
return _gmap_unmap_prefix(gmap, gfn, end, true);
static inline bool gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end)
return _gmap_unmap_prefix(gmap, gfn, end, false);
static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, union pte newpte,
lockdep_assert_held(&gmap->kvm->mmu_lock);
lockdep_assert_held(&gmap->children_lock);
lockdep_assert_not_held(&gmap->children_lock);
gmap_unmap_prefix(gmap, gfn, gfn + 1);
gmap_handle_vsie_unshadow_event(gmap, gfn);
_gmap_handle_vsie_unshadow_event(gmap, gfn);
return __dat_ptep_xchg(ptep, pgste, newpte, gfn, gmap->asce, uses_skeys(gmap));
static inline union pgste gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, union pte newpte,
return _gmap_ptep_xchg(gmap, ptep, newpte, pgste, gfn, true);
static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep,
if (KVM_BUG_ON(crstep->h.tt != oldcrste.h.tt || newcrste.h.tt != oldcrste.h.tt, gmap->kvm))
lockdep_assert_held(&gmap->kvm->mmu_lock);
lockdep_assert_held(&gmap->children_lock);
gmap_unmap_prefix(gmap, gfn, gfn + align);
gmap_handle_vsie_unshadow_event(gmap, gfn);
_gmap_handle_vsie_unshadow_event(gmap, gfn);
return dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, gmap->asce);
static inline bool __must_check gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep,
return _gmap_crstep_xchg_atomic(gmap, crstep, oldcrste, newcrste, gfn, true);
static inline bool gmap_is_shadow_valid(struct gmap *sg, union asce asce, int edat_level)
struct gmap *parent;
struct gmap *gmap;
int s390_replace_asce(struct gmap *gmap);
bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint);
bool gmap_age_gfn(struct gmap *gmap, gfn_t start, gfn_t end);
bool gmap_unmap_gfn_range(struct gmap *gmap, struct kvm_memory_slot *slot, gfn_t start, gfn_t end);
int gmap_try_fixup_minor(struct gmap *gmap, struct guest_fault *fault);
struct gmap *gmap_new(struct kvm *kvm, gfn_t limit);
struct gmap *gmap_new_child(struct gmap *parent, gfn_t limit);
void gmap_remove_child(struct gmap *child);
void gmap_dispose(struct gmap *gmap);
int gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, struct guest_fault *fault);
void gmap_sync_dirty_log(struct gmap *gmap, gfn_t start, gfn_t end);
int gmap_set_limit(struct gmap *gmap, gfn_t limit);
int gmap_ucas_translate(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, gpa_t *gaddr);
int gmap_ucas_map(struct gmap *gmap, gfn_t p_gfn, gfn_t c_gfn, unsigned long count);
void gmap_ucas_unmap(struct gmap *gmap, gfn_t c_gfn, unsigned long count);
int gmap_enable_skeys(struct gmap *gmap);
set_bit(GMAP_FLAG_PFAULT_ENABLED, &dev->kvm->arch.gmap->flags);
clear_bit(GMAP_FLAG_PFAULT_ENABLED, &dev->kvm->arch.gmap->flags);
ret = gmap_set_limit(kvm->arch.gmap, gpa_to_gfn(new_limit));
(void *)kvm->arch.gmap->asce.val);
gmap_set_cmma_all_dirty(kvm->arch.gmap);
if (!uses_skeys(kvm->arch.gmap))
r = dat_get_storage_key(kvm->arch.gmap->asce,
r = gmap_enable_skeys(kvm->arch.gmap);
r = dat_set_storage_key(mc, kvm->arch.gmap->asce,
if (!args->count || !uses_cmm(kvm->arch.gmap)) {
ret = dat_peek_cmma(args->start_gfn, kvm->arch.gmap->asce, &args->count,
ret = dat_get_cmma(kvm->arch.gmap->asce, &args->start_gfn, &args->count,
r = dat_set_cmma_bits(mc, kvm->arch.gmap->asce, args->start_gfn,
set_bit(GMAP_FLAG_USES_CMM, &kvm->arch.gmap->flags);
set_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &kvm->arch.gmap->flags);
kvm->arch.gmap = gmap_new(kvm, gpa_to_gfn(kvm->arch.mem_limit));
if (!kvm->arch.gmap)
clear_bit(GMAP_FLAG_PFAULT_ENABLED, &kvm->arch.gmap->flags);
set_bit(GMAP_FLAG_IS_UCONTROL, &kvm->arch.gmap->flags);
struct crst_table *table = dereference_asce(kvm->arch.gmap->asce);
scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock)
gmap_remove_child(vcpu->arch.gmap);
vcpu->arch.gmap = gmap_put(vcpu->arch.gmap);
kvm->arch.gmap = gmap_put(kvm->arch.gmap);
vcpu->arch.gmap = vcpu->kvm->arch.gmap;
vcpu->arch.gmap = gmap_new_child(vcpu->kvm->arch.gmap, -1UL);
if (!vcpu->arch.gmap)
gmap_remove_child(vcpu->arch.gmap);
vcpu->arch.gmap = gmap_put(vcpu->arch.gmap);
rc = gmap_ucas_translate(vcpu->arch.mc, vcpu->arch.gmap, gaddr);
rc = dat_set_prefix_notif_bit(vcpu->kvm->arch.gmap->asce, gfn);
if (vcpu->kvm->arch.use_cmma && uses_cmm(vcpu->arch.gmap))
if (!pfault_enabled(vcpu->arch.gmap))
.attempt_pfault = pfault_enabled(vcpu->arch.gmap),
vcpu->arch.gmap->asce.val);
r = gmap_ucas_map(vcpu->arch.gmap, gpa_to_gfn(ucas.user_addr),
gmap_ucas_unmap(vcpu->arch.gmap, gpa_to_gfn(ucas.vcpu_addr),
union asce asce = kvm->arch.gmap->asce;
rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages);
rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages);
rc = dat_create_slot(mc, kvm->arch.gmap->asce, new->base_gfn, new->npages);
return dat_test_age_gfn(kvm->arch.gmap->asce, range->start, range->end);
return gmap_age_gfn(kvm->arch.gmap, range->start, range->end);
return gmap_unmap_gfn_range(kvm->arch.gmap, range->slot, range->start, range->end);
gmap_sync_dirty_log(kvm->arch.gmap, memslot->base_gfn, last_gfn);
set_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags);
start_gfn = dat_reset_cmma(kvm->arch.gmap->asce, start_gfn);
return test_bit(GMAP_FLAG_IS_UCONTROL, &kvm->arch.gmap->flags);
void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, gpa_t start, gpa_t end);
int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned int prot,
rc = dat_cond_set_storage_key(vcpu->arch.mc, vcpu->arch.gmap->asce,
nappended = dat_perform_essa(vcpu->arch.gmap->asce, gfn, orc, &state, &dirtied);
if (dat_entry_walk(NULL, gpa_to_gfn(cbrl[i]), vcpu->arch.gmap->asce,
set_bit(GMAP_FLAG_USES_CMM, &vcpu->arch.gmap->flags);
rc = gmap_enable_skeys(vcpu->arch.gmap);
rc = dat_get_storage_key(vcpu->arch.gmap->asce, gpa_to_gfn(gaddr), &key);
rc = dat_reset_reference_bit(vcpu->arch.gmap->asce, gpa_to_gfn(gaddr));
rc = dat_cond_set_storage_key(vcpu->arch.mc, vcpu->arch.gmap->asce,
if (kvm->arch.gmap->asce.dt == TABLE_TYPE_SEGMENT)
priv->old_gmap_table = (unsigned long)dereference_asce(kvm->arch.gmap->asce);
if (s390_replace_asce(kvm->arch.gmap))
gmap_pv_destroy_range(kvm->arch.gmap, 0, gpa_to_gfn(SZ_2G), false);
s390_replace_asce(kvm->arch.gmap);
gmap_pv_destroy_range(kvm->arch.gmap, 0, asce_end(kvm->arch.gmap->asce), false);
if (gmap_pv_destroy_range(kvm->arch.gmap, 0, asce_end(kvm->arch.gmap->asce), true))
set_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &kvm->arch.gmap->flags);
uvcb.guest_asce = kvm->arch.gmap->asce.val;
clear_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags);
gmap_split_huge_pages(kvm->arch.gmap);
static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
struct gmap *gmap = vsie_page->gmap_cache.gmap;
lockdep_assert_held(&gmap->kvm->arch.gmap->children_lock);
vsie_page->gmap_cache.gmap = NULL;
if (list_empty(&gmap->scb_users)) {
gmap_remove_child(gmap);
gmap_put(gmap);
static struct gmap *acquire_gmap_shadow(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
struct gmap *gmap;
scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock) {
gmap = vsie_page->gmap_cache.gmap;
if (gmap) {
if (gmap_is_shadow_valid(gmap, asce, edat)) {
gmap_get(gmap);
return gmap;
gmap = gmap_create_shadow(vcpu->arch.mc, vcpu->kvm->arch.gmap, asce, edat);
if (IS_ERR(gmap))
return gmap;
scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock) {
if (vsie_page->gmap_cache.gmap)
if (!gmap->parent) {
gmap_put(gmap);
list_add(&vsie_page->gmap_cache.list, &gmap->scb_users);
vsie_page->gmap_cache.gmap = gmap;
return gmap;
struct gmap *sg = NULL;
if (vsie_page->gmap_cache.gmap) {
scoped_guard(spinlock, &kvm->arch.gmap->children_lock)
if (vsie_page->gmap_cache.gmap)
scoped_guard(spinlock, &kvm->arch.gmap->children_lock)
if (vsie_page->gmap_cache.gmap)
void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, gpa_t start, gpa_t end)
KVM_BUG_ON(!test_bit(GMAP_FLAG_SHADOW, &gmap->flags), gmap->kvm);
list_for_each_entry_safe(cur, next, &gmap->scb_users, gmap_cache.list) {
static int map_prefix(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
static void handle_last_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
tbl->gmap = (crtl & GSWIP_PCE_TBL_CTRL_GMAP_MASK) >> 7;
crtl |= (tbl->gmap << 7) & GSWIP_PCE_TBL_CTRL_GMAP_MASK;
u8 gmap;
gbe->gmap[i] = (i << 24) | (i << 16) | (i << 8);
volatile uint32_t gmap[256]; /* gamma map */