crste
static long dat_set_pn_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
union crste newcrste, oldcrste;
union crste *crstep;
union crste crste;
crste = table->crstes[0];
if (crste.h.fc)
if (!crste.h.i) {
asce->rsto = crste.h.fc0.to;
static long __dat_peek_cmma_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
crste.h.tt--;
crst_table_init((void *)table, crste.val);
crste = _crste_fc0(asce->rsto, asce->dt + 1);
crst_table_init((void *)table, _CRSTE_HOLE(crste.h.tt).val);
table->crstes[0] = crste;
union crste *crstep;
bool __must_check dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new,
static int dat_split_crste(struct kvm_s390_mmu_cache *mc, union crste *crstep,
union crste old, new, init;
int walk_level, union crste **last, union pte **ptepp)
union crste entry;
union crste crste;
crste = READ_ONCE(*walk->last);
if (crste_hole(crste)) {
the_op = walk->ops->crste_ops[crste.h.tt];
crste = READ_ONCE(*walk->last);
if (!crste.h.i && !crste.h.fc) {
if (!is_pmd(crste))
_dereference_crste(crste), walk);
dereference_pmd(crste.pmd), walk);
union crste *crstep;
union crste crste;
crste = READ_ONCE(*crstep);
if (!crste.h.fc || !crste.s.fc1.pr)
skey->skey = page_get_storage_key(large_crste_to_phys(crste, gfn));
union crste *crstep;
union crste *crstep;
union crste *crstep;
union crste crste = READ_ONCE(*crstep);
if (!crste.h.fc || !crste.s.fc1.pr)
static long dat_reset_skeys_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
union crste dummy = { .val = p->token };
static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
union crste new_crste, crste = READ_ONCE(*crstep);
new_crste.h.tt = crste.h.tt;
if (crste.val == new_crste.val)
if (!dat_crstep_xchg_atomic(crstep, crste, new_crste, gfn, walk->asce))
if (!crste.h.fc && !crste.h.i) {
if (is_pmd(crste))
dat_free_pt(dereference_pmd(crste.pmd));
dat_free_level(dereference_crste(crste), true);
if (!crste.h.fc && !crste.h.i)
static long _dat_test_young_crste(union crste *crstep, gfn_t start, gfn_t end,
static_assert(sizeof(union crste) == sizeof(unsigned long));
union crste crstes[_CRST_ENTRIES];
typedef long (*dat_walk_op)(union crste *crste, gfn_t gfn, gfn_t next, struct dat_walk *w);
union crste *last;
#define _CRSTE_TOK(l, t, p) ((union crste) { \
static inline union crste _crste_fc0(kvm_pfn_t pfn, int tt)
union crste res = { .val = PFN_PHYS(pfn) };
static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool writable, bool dirty)
union crste res = { .val = PFN_PHYS(pfn) & _SEGMENT_MASK };
union crste *crstep; /* Used to resolve the fault, or NULL */
bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce);
int walk_level, union crste **last, union pte **ptepp);
static inline struct crst_table *crste_table_start(union crste *crstep)
static inline bool crdte_crste(union crste *crstep, union crste old, union crste new, gfn_t gfn,
static __always_inline void idte_crste(union crste *crstep, gfn_t gfn, unsigned long opt,
#define _CRSTE(x) ((union crste) { .val = _Generic((x), \
union crste : (x).val)})
#define _CRSTEP(x) ((union crste *)_Generic((*(x)), \
union crste : (x)))
static inline bool is_pmd(union crste crste)
return crste.h.tt == TABLE_TYPE_SEGMENT;
static inline bool is_pud(union crste crste)
return crste.h.tt == TABLE_TYPE_REGION3;
static inline bool is_p4d(union crste crste)
return crste.h.tt == TABLE_TYPE_REGION2;
static inline bool is_pgd(union crste crste)
return crste.h.tt == TABLE_TYPE_REGION1;
static inline phys_addr_t crste_origin_large(union crste crste)
if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3))
if (is_pmd(crste))
return pmd_origin_large(crste.pmd);
return pud_origin_large(crste.pud);
static inline bool crste_leaf(union crste crste)
return (crste.h.tt <= TABLE_TYPE_REGION3) && crste.h.fc;
static inline bool crste_prefix(union crste crste)
return crste_leaf(crste) && crste.s.fc1.prefix_notif;
static inline bool crste_dirty(union crste crste)
return crste_leaf(crste) && crste.s.fc1.d;
static inline bool _crste_hole(union crste crste)
return crste.h.i && !crste.tok.pr && crste.tok.type != _DAT_TOKEN_NONE;
static inline bool _crste_none(union crste crste)
return crste.h.i && !crste.tok.pr && crste.tok.type == _DAT_TOKEN_NONE;
static inline phys_addr_t large_crste_to_phys(union crste crste, gfn_t gfn)
if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3))
if (is_pmd(crste))
return large_pmd_to_phys(crste.pmd, gfn);
return large_pud_to_phys(crste.pud, gfn);
static inline bool cspg_crste(union crste *crstep, union crste old, union crste new)
static inline struct crst_table *_dereference_crste(union crste crste)
if (unlikely(is_pmd(crste)))
return phys_to_virt(crste_origin(crste.pud));
union crste : _dereference_crste(_CRSTE(x))))
static inline union crste dat_crstep_clear_atomic(union crste *crstep, gfn_t gfn, union asce asce)
union crste oldcrste, empty = _CRSTE_EMPTY(crstep->h.tt);
static inline int get_level(union crste *crstep, union pte *ptep)
static inline bool crste_is_ucas(union crste crste)
return is_pmd(crste) && crste.h.i && crste.h.fc0.tl == 1 && crste.h.fc == 0;
static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, union crste *table,
union crste newcrste, oldcrste;
union crste *table, *host;
union crste *crstep;
union crste *crstep;
union crste crste;
crste = dat_crstep_clear_atomic(crstep, r_gfn, sg->asce);
if (crste_leaf(crste) || crste.h.i)
if (is_pmd(crste))
dat_free_pt(dereference_pmd(crste.pmd));
dat_free_level(dereference_crste(crste), true);
static long gmap_clear_young_crste(union crste *crstep, gfn_t gfn, gfn_t end, struct dat_walk *walk)
union crste crste, new;
crste = READ_ONCE(*crstep);
if (!crste.h.fc)
if (!crste.s.fc1.y && crste.h.i)
if (crste_prefix(crste) && !gmap_mkold_prefix(priv->gmap, gfn, end))
new = crste;
folio_set_dirty(phys_to_folio(crste_origin_large(crste)));
} while (!dat_crstep_xchg_atomic(crstep, crste, new, gfn, walk->asce));
static long _gmap_unmap_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
union crste old = *crstep;
static long _crste_test_and_clear_softdirty(union crste *table, gfn_t gfn, gfn_t end,
union crste crste, new;
crste = READ_ONCE(*table);
if (!crste.h.fc)
if (crste.h.p && !crste.s.fc1.sd)
if (crste.s.fc1.prefix_notif && !gmap_mkold_prefix(gmap, gfn, end))
new = crste;
} while (!gmap_crstep_xchg_atomic(gmap, table, crste, new, gfn));
union crste newcrste, oldcrste = READ_ONCE(*f->crstep);
union crste oldval, newval;
union crste newcrste, oldcrste;
union crste *crstep;
static int gmap_ucas_translate_simple(struct gmap *gmap, gpa_t *gaddr, union crste **crstepp)
union crste *crstep;
union crste *crstep;
static long _gmap_split_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
union crste crste, newcrste;
crste = READ_ONCE(*crstep);
newcrste = _CRSTE_EMPTY(crste.h.tt);
while (crste_leaf(crste)) {
if (crste_prefix(crste))
if (crste.s.fc1.vsie_notif)
if (dat_crstep_xchg_atomic(crstep, crste, newcrste, gfn, walk->asce))
crste = READ_ONCE(*crstep);
static long _destroy_pages_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep,
union crste oldcrste, union crste newcrste,
static inline bool __must_check gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep,
union crste oldcrste, union crste newcrste,
union crste *crstep;