vsie_page
struct vsie_page;
struct vsie_page *pages[KVM_MAX_VCPUS];
clear_vsie_icpt(vsie_page);
static int handle_stfle_0(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
if (read_guest_real(vcpu, fac_list_origin, &vsie_page->fac,
scb_s->fac = (u32)virt_to_phys(&vsie_page->fac);
static void prefix_mapped(struct vsie_page *vsie_page)
static int handle_stfle_2(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, u32 fac_list_origin)
struct kvm_s390_flcb2 *flcb_s = (struct kvm_s390_flcb2 *)vsie_page->fac;
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
atomic_andnot(PROG_REQUEST, &vsie_page->scb_s.prog20);
scb_s->fac = (u32)virt_to_phys(&vsie_page->fac) | S390_ARCH_FAC_FORMAT_2;
static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
u32 fac = READ_ONCE(vsie_page->scb_o->fac);
BUILD_BUG_ON(sizeof(vsie_page->fac) < ((S390_ARCH_FAC_LIST_SIZE_U64 + 1) * sizeof(u64)));
BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct vsie_page, fac), 8));
retry_vsie_icpt(vsie_page);
return handle_stfle_0(vcpu, vsie_page, origin);
return set_validity_icpt(&vsie_page->scb_s, 0x1330U);
return handle_stfle_2(vcpu, vsie_page, origin);
static int prefix_is_mapped(struct vsie_page *vsie_page)
return set_validity_icpt(&vsie_page->scb_s, 0x1330U);
static u64 vsie_get_register(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, u8 reg)
return vsie_page->scb_s.gg15;
return !(atomic_read(&vsie_page->scb_s.prog20) & PROG_REQUEST);
return vsie_page->scb_s.gg14;
static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
u64 *pei_block = &vsie_page->scb_o->mcic;
dest = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 20) & mask;
src = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 16) & mask;
retry_vsie_icpt(vsie_page);
static void update_intervention_requests(struct vsie_page *vsie_page)
clear_vsie_icpt(vsie_page);
retry_vsie_icpt(vsie_page);
static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
cpuflags = atomic_read(&vsie_page->scb_o->cpuflags);
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
handle_last_fault(vcpu, vsie_page, sg);
atomic_andnot(bits, &vsie_page->scb_s.cpuflags);
atomic_or(cpuflags & bits, &vsie_page->scb_s.cpuflags);
static int prepare_cpuflags(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
kvm_s390_reinject_machine_check(vcpu, &vsie_page->mcck_info);
return handle_fault(vcpu, vsie_page, sg);
rc = handle_stfle(vcpu, vsie_page);
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
clear_vsie_icpt(vsie_page);
rc = vsie_handle_mvpg(vcpu, vsie_page, sg);
struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
static void release_gmap_shadow(struct vsie_page *vsie_page)
struct gmap *gmap = vsie_page->gmap_cache.gmap;
list_del(&vsie_page->gmap_cache.list);
vsie_page->gmap_cache.gmap = NULL;
prefix_unmapped(vsie_page);
static struct gmap *acquire_gmap_shadow(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
gmap = vsie_page->gmap_cache.gmap;
release_gmap_shadow(vsie_page);
if (vsie_page->gmap_cache.gmap)
release_gmap_shadow(vsie_page);
list_add(&vsie_page->gmap_cache.list, &gmap->scb_users);
vsie_page->gmap_cache.gmap = gmap;
prefix_unmapped(vsie_page);
struct vsie_page *vsie_page)
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
WRITE_ONCE(vcpu->arch.vsie_block, &vsie_page->scb_s);
static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
sg = acquire_gmap_shadow(vcpu, vsie_page);
rc = map_prefix(vcpu, vsie_page, sg);
update_intervention_requests(vsie_page);
rc = do_vsie_run(vcpu, vsie_page, sg);
static inline bool try_get_vsie_page(struct vsie_page *vsie_page)
if (test_bit(VSIE_PAGE_IN_USE, &vsie_page->flags))
return !test_and_set_bit(VSIE_PAGE_IN_USE, &vsie_page->flags);
static void put_vsie_page(struct vsie_page *vsie_page)
clear_bit(VSIE_PAGE_IN_USE, &vsie_page->flags);
static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr)
struct vsie_page *vsie_page;
vsie_page = radix_tree_lookup(&kvm->arch.vsie.addr_to_page, addr >> 9);
if (vsie_page) {
if (try_get_vsie_page(vsie_page)) {
if (vsie_page->scb_gpa == addr)
return vsie_page;
put_vsie_page(vsie_page);
vsie_page = (void *)__get_free_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO | GFP_DMA);
if (!vsie_page) {
__set_bit(VSIE_PAGE_IN_USE, &vsie_page->flags);
kvm->arch.vsie.pages[kvm->arch.vsie.page_count] = vsie_page;
vsie_page = kvm->arch.vsie.pages[kvm->arch.vsie.next];
if (try_get_vsie_page(vsie_page))
if (vsie_page->scb_gpa != ULONG_MAX)
vsie_page->scb_gpa >> 9);
vsie_page->scb_gpa = ULONG_MAX;
if (radix_tree_insert(&kvm->arch.vsie.addr_to_page, addr >> 9, vsie_page)) {
put_vsie_page(vsie_page);
vsie_page->scb_gpa = addr;
memset(&vsie_page->scb_s, 0, sizeof(struct kvm_s390_sie_block));
if (vsie_page->gmap_cache.gmap) {
if (vsie_page->gmap_cache.gmap)
release_gmap_shadow(vsie_page);
prefix_unmapped(vsie_page);
vsie_page->fault_addr = 0;
vsie_page->scb_s.ihcpu = 0xffffU;
return vsie_page;
struct vsie_page *vsie_page;
BUILD_BUG_ON(sizeof(struct vsie_page) != PAGE_SIZE);
vsie_page = get_vsie_page(vcpu->kvm, scb_addr);
if (IS_ERR(vsie_page)) {
return PTR_ERR(vsie_page);
} else if (!vsie_page) {
rc = pin_scb(vcpu, vsie_page, scb_addr);
rc = shadow_scb(vcpu, vsie_page);
rc = pin_blocks(vcpu, vsie_page);
register_shadow_scb(vcpu, vsie_page);
rc = vsie_run(vcpu, vsie_page);
unpin_blocks(vcpu, vsie_page);
unshadow_scb(vcpu, vsie_page);
unpin_scb(vcpu, vsie_page, scb_addr);
put_vsie_page(vsie_page);
struct vsie_page *vsie_page;
vsie_page = kvm->arch.vsie.pages[i];
if (vsie_page->gmap_cache.gmap)
release_gmap_shadow(vsie_page);
if (vsie_page->scb_gpa != ULONG_MAX)
vsie_page->scb_gpa >> 9);
free_page((unsigned long)vsie_page);
static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
ret = setup_apcb(vcpu, &vsie_page->crycb, crycb_addr,
vsie_page->crycb.dea_wrapping_key_mask, 56))
b1 = (unsigned long *) vsie_page->crycb.dea_wrapping_key_mask;
scb_s->crycbd = (u32)virt_to_phys(&vsie_page->crycb) | CRYCB_FORMAT2;
static void shadow_esa(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
static void prepare_ibc(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
static void unshadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
rc = prepare_cpuflags(vcpu, vsie_page);
prefix_unmapped(vsie_page);
prefix_unmapped(vsie_page);
shadow_esa(vcpu, vsie_page);
prepare_ibc(vcpu, vsie_page);
rc = shadow_crycb(vcpu, vsie_page);
unshadow_scb(vcpu, vsie_page);
struct vsie_page *cur, *next;
static int map_prefix(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
if (prefix_is_mapped(vsie_page))
prefix_mapped(vsie_page);
prefix_unmapped(vsie_page);
static void unpin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
unpin_guest_page(vcpu->kvm, vsie_page->sca_gpa, hpa);
vsie_page->sca_gpa = 0;
unpin_guest_page(vcpu->kvm, vsie_page->itdba_gpa, hpa);
vsie_page->itdba_gpa = 0;
static_assert(sizeof(struct vsie_page) == PAGE_SIZE);
unpin_guest_page(vcpu->kvm, vsie_page->gvrd_gpa, hpa);
vsie_page->gvrd_gpa = 0;
unpin_guest_page(vcpu->kvm, vsie_page->riccbd_gpa, hpa);
vsie_page->riccbd_gpa = 0;
unpin_guest_page(vcpu->kvm, vsie_page->sdnx_gpa, hpa);
vsie_page->sdnx_gpa = 0;
static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
vsie_page->sca_gpa = gpa;
vsie_page->itdba_gpa = gpa;
vsie_page->gvrd_gpa = gpa;
vsie_page->riccbd_gpa = gpa;
static void prefix_unmapped(struct vsie_page *vsie_page)
vsie_page->sdnx_gpa = gpa;
unpin_blocks(vcpu, vsie_page);
atomic_or(PROG_REQUEST, &vsie_page->scb_s.prog20);
static void unpin_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,
hpa_t hpa = virt_to_phys(vsie_page->scb_o);
vsie_page->scb_o = NULL;
static int pin_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,
vsie_page->scb_o = phys_to_virt(hpa);
static void prefix_unmapped_sync(struct vsie_page *vsie_page)
prefix_unmapped(vsie_page);
if (vsie_page->scb_s.prog0c & PROG_IN_SIE)
static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
atomic_or(CPUSTAT_STOP_INT, &vsie_page->scb_s.cpuflags);
vsie_page->fault_addr = current->thread.gmap_teid.addr * PAGE_SIZE;
while (vsie_page->scb_s.prog0c & PROG_IN_SIE)
static void handle_last_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
if (vsie_page->fault_addr)
gaccess_shadow_fault(vcpu, sg, vsie_page->fault_addr, NULL, true);
vsie_page->fault_addr = 0;
static inline void clear_vsie_icpt(struct vsie_page *vsie_page)
vsie_page->scb_s.icptcode = 0;
static void retry_vsie_icpt(struct vsie_page *vsie_page)
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;