/* SPDX-License-Identifier: GPL-2.0 */
/*
 *  KVM guest address space mapping code
 *
 *    Copyright IBM Corp. 2024, 2025
 *    Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com>
 */

#ifndef __KVM_S390_DAT_H
#define __KVM_S390_DAT_H

#include <linux/radix-tree.h>
#include <linux/refcount.h>
#include <linux/io.h>
#include <linux/kvm_types.h>
#include <linux/pgalloc.h>
#include <asm/tlbflush.h>
#include <asm/dat-bits.h>

/*
 * Base address and length must be sent at the start of each block, therefore
 * it's cheaper to send some clean data, as long as it's less than the size of
 * two longs.
 */
#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
/* For consistency */
#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)

#define _ASCE(x) ((union asce) { .val = (x), })
#define NULL_ASCE _ASCE(0)

enum {
        _DAT_TOKEN_NONE = 0,
        _DAT_TOKEN_PIC,
};

#define _CRSTE_TOK(l, t, p) ((union crste) {    \
                .tok.i = 1,                     \
                .tok.tt = (l),                  \
                .tok.type = (t),                \
                .tok.par = (p)                  \
        })
#define _CRSTE_PIC(l, p) _CRSTE_TOK(l, _DAT_TOKEN_PIC, p)

#define _CRSTE_HOLE(l) _CRSTE_PIC(l, PGM_ADDRESSING)
#define _CRSTE_EMPTY(l) _CRSTE_TOK(l, _DAT_TOKEN_NONE, 0)

#define _PMD_EMPTY _CRSTE_EMPTY(TABLE_TYPE_SEGMENT)

#define _PTE_TOK(t, p) ((union pte) { .tok.i = 1, .tok.type = (t), .tok.par = (p) })
#define _PTE_EMPTY _PTE_TOK(_DAT_TOKEN_NONE, 0)

/* This fake table type is used for page table walks (both for normal page tables and vSIE) */
#define TABLE_TYPE_PAGE_TABLE -1

enum dat_walk_flags {
        DAT_WALK_USES_SKEYS     = 0x40,
        DAT_WALK_CONTINUE       = 0x20,
        DAT_WALK_IGN_HOLES      = 0x10,
        DAT_WALK_SPLIT          = 0x08,
        DAT_WALK_ALLOC          = 0x04,
        DAT_WALK_ANY            = 0x02,
        DAT_WALK_LEAF           = 0x01,
        DAT_WALK_DEFAULT        = 0
};

#define DAT_WALK_SPLIT_ALLOC (DAT_WALK_SPLIT | DAT_WALK_ALLOC)
#define DAT_WALK_ALLOC_CONTINUE (DAT_WALK_CONTINUE | DAT_WALK_ALLOC)
#define DAT_WALK_LEAF_ALLOC (DAT_WALK_LEAF | DAT_WALK_ALLOC)

union pte {
        unsigned long val;
        union page_table_entry h;
        struct {
                unsigned long   :56; /* Hardware bits */
                unsigned long u : 1; /* Page unused */
                unsigned long s : 1; /* Special */
                unsigned long w : 1; /* Writable */
                unsigned long r : 1; /* Readable */
                unsigned long d : 1; /* Dirty */
                unsigned long y : 1; /* Young */
                unsigned long sd: 1; /* Soft dirty */
                unsigned long pr: 1; /* Present */
        } s;
        struct {
                unsigned char hwbytes[7];
                unsigned char swbyte;
        };
        union {
                struct {
                        unsigned long type :16; /* Token type */
                        unsigned long par  :16; /* Token parameter */
                        unsigned long      :20;
                        unsigned long      : 1; /* Must be 0 */
                        unsigned long i    : 1; /* Must be 1 */
                        unsigned long      : 2;
                        unsigned long      : 7;
                        unsigned long pr   : 1; /* Must be 0 */
                };
                struct {
                        unsigned long token:32; /* Token and parameter */
                        unsigned long      :32;
                };
        } tok;
};

/* Soft dirty, needed as macro for atomic operations on ptes */
#define _PAGE_SD 0x002

/* Needed as macro to perform atomic operations */
#define PGSTE_PCL_BIT           0x0080000000000000UL    /* PCL lock, HW bit */
#define PGSTE_CMMA_D_BIT        0x0000000000008000UL    /* CMMA dirty soft-bit */

enum pgste_gps_usage {
        PGSTE_GPS_USAGE_STABLE = 0,
        PGSTE_GPS_USAGE_UNUSED,
        PGSTE_GPS_USAGE_POT_VOLATILE,
        PGSTE_GPS_USAGE_VOLATILE,
};

union pgste {
        unsigned long val;
        struct {
                unsigned long acc          : 4;
                unsigned long fp           : 1;
                unsigned long              : 3;
                unsigned long pcl          : 1;
                unsigned long hr           : 1;
                unsigned long hc           : 1;
                unsigned long              : 2;
                unsigned long gr           : 1;
                unsigned long gc           : 1;
                unsigned long              : 1;
                unsigned long              :16; /* val16 */
                unsigned long zero         : 1;
                unsigned long nodat        : 1;
                unsigned long              : 4;
                unsigned long usage        : 2;
                unsigned long              : 8;
                unsigned long cmma_d       : 1; /* Dirty flag for CMMA bits */
                unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
                unsigned long vsie_notif   : 1; /* Referenced in a shadow table */
                unsigned long              : 5;
                unsigned long              : 8;
        };
        struct {
                unsigned short hwbytes0;
                unsigned short val16;   /* Used to store chunked values, see dat_{s,g}et_ptval() */
                unsigned short hwbytes4;
                unsigned char flags;    /* Maps to the software bits */
                unsigned char hwbyte7;
        } __packed;
};

union pmd {
        unsigned long val;
        union segment_table_entry h;
        struct {
                struct {
                        unsigned long              :44; /* HW */
                        unsigned long              : 3; /* Unused */
                        unsigned long              : 1; /* HW */
                        unsigned long s            : 1; /* Special */
                        unsigned long w            : 1; /* Writable soft-bit */
                        unsigned long r            : 1; /* Readable soft-bit */
                        unsigned long d            : 1; /* Dirty */
                        unsigned long y            : 1; /* Young */
                        unsigned long              : 3; /* HW */
                        unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
                        unsigned long vsie_notif   : 1; /* Referenced in a shadow table */
                        unsigned long              : 4; /* HW */
                        unsigned long sd           : 1; /* Soft-Dirty */
                        unsigned long pr           : 1; /* Present */
                } fc1;
        } s;
};

union pud {
        unsigned long val;
        union region3_table_entry h;
        struct {
                struct {
                        unsigned long              :33; /* HW */
                        unsigned long              :14; /* Unused */
                        unsigned long              : 1; /* HW */
                        unsigned long s            : 1; /* Special */
                        unsigned long w            : 1; /* Writable soft-bit */
                        unsigned long r            : 1; /* Readable soft-bit */
                        unsigned long d            : 1; /* Dirty */
                        unsigned long y            : 1; /* Young */
                        unsigned long              : 3; /* HW */
                        unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
                        unsigned long vsie_notif   : 1; /* Referenced in a shadow table */
                        unsigned long              : 4; /* HW */
                        unsigned long sd           : 1; /* Soft-Dirty */
                        unsigned long pr           : 1; /* Present */
                } fc1;
        } s;
};

union p4d {
        unsigned long val;
        union region2_table_entry h;
};

union pgd {
        unsigned long val;
        union region1_table_entry h;
};

union crste {
        unsigned long val;
        union {
                struct {
                        unsigned long   :52;
                        unsigned long   : 1;
                        unsigned long fc: 1;
                        unsigned long p : 1;
                        unsigned long   : 1;
                        unsigned long   : 2;
                        unsigned long i : 1;
                        unsigned long   : 1;
                        unsigned long tt: 2;
                        unsigned long   : 2;
                };
                struct {
                        unsigned long to:52;
                        unsigned long   : 1;
                        unsigned long fc: 1;
                        unsigned long p : 1;
                        unsigned long   : 1;
                        unsigned long tf: 2;
                        unsigned long i : 1;
                        unsigned long   : 1;
                        unsigned long tt: 2;
                        unsigned long tl: 2;
                } fc0;
                struct {
                        unsigned long    :47;
                        unsigned long av : 1; /* ACCF-Validity Control */
                        unsigned long acc: 4; /* Access-Control Bits */
                        unsigned long f  : 1; /* Fetch-Protection Bit */
                        unsigned long fc : 1; /* Format-Control */
                        unsigned long p  : 1; /* DAT-Protection Bit */
                        unsigned long iep: 1; /* Instruction-Execution-Protection */
                        unsigned long    : 2;
                        unsigned long i  : 1; /* Segment-Invalid Bit */
                        unsigned long cs : 1; /* Common-Segment Bit */
                        unsigned long tt : 2; /* Table-Type Bits */
                        unsigned long    : 2;
                } fc1;
        } h;
        struct {
                struct {
                        unsigned long              :47;
                        unsigned long              : 1; /* HW (should be 0) */
                        unsigned long s            : 1; /* Special */
                        unsigned long w            : 1; /* Writable */
                        unsigned long r            : 1; /* Readable */
                        unsigned long d            : 1; /* Dirty */
                        unsigned long y            : 1; /* Young */
                        unsigned long              : 3; /* HW */
                        unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
                        unsigned long vsie_notif   : 1; /* Referenced in a shadow table */
                        unsigned long              : 4; /* HW */
                        unsigned long sd           : 1; /* Soft-Dirty */
                        unsigned long pr           : 1; /* Present */
                } fc1;
        } s;
        union {
                struct {
                        unsigned long type :16; /* Token type */
                        unsigned long par  :16; /* Token parameter */
                        unsigned long      :26;
                        unsigned long i    : 1; /* Must be 1 */
                        unsigned long      : 1;
                        unsigned long tt   : 2;
                        unsigned long      : 1;
                        unsigned long pr   : 1; /* Must be 0 */
                };
                struct {
                        unsigned long token:32; /* Token and parameter */
                        unsigned long      :32;
                };
        } tok;
        union pmd pmd;
        union pud pud;
        union p4d p4d;
        union pgd pgd;
};

union skey {
        unsigned char skey;
        struct {
                unsigned char acc :4;
                unsigned char fp  :1;
                unsigned char r   :1;
                unsigned char c   :1;
                unsigned char zero:1;
        };
};

static_assert(sizeof(union pgste) == sizeof(unsigned long));
static_assert(sizeof(union pte) == sizeof(unsigned long));
static_assert(sizeof(union pmd) == sizeof(unsigned long));
static_assert(sizeof(union pud) == sizeof(unsigned long));
static_assert(sizeof(union p4d) == sizeof(unsigned long));
static_assert(sizeof(union pgd) == sizeof(unsigned long));
static_assert(sizeof(union crste) == sizeof(unsigned long));
static_assert(sizeof(union skey) == sizeof(char));

struct segment_table {
        union pmd pmds[_CRST_ENTRIES];
};

struct region3_table {
        union pud puds[_CRST_ENTRIES];
};

struct region2_table {
        union p4d p4ds[_CRST_ENTRIES];
};

struct region1_table {
        union pgd pgds[_CRST_ENTRIES];
};

struct crst_table {
        union {
                union crste crstes[_CRST_ENTRIES];
                struct segment_table segment;
                struct region3_table region3;
                struct region2_table region2;
                struct region1_table region1;
        };
};

struct page_table {
        union pte ptes[_PAGE_ENTRIES];
        union pgste pgstes[_PAGE_ENTRIES];
};

static_assert(sizeof(struct crst_table) == _CRST_TABLE_SIZE);
static_assert(sizeof(struct page_table) == PAGE_SIZE);

struct dat_walk;

typedef long (*dat_walk_op)(union crste *crste, gfn_t gfn, gfn_t next, struct dat_walk *w);

struct dat_walk_ops {
        union {
                dat_walk_op crste_ops[4];
                struct {
                        dat_walk_op pmd_entry;
                        dat_walk_op pud_entry;
                        dat_walk_op p4d_entry;
                        dat_walk_op pgd_entry;
                };
        };
        long (*pte_entry)(union pte *pte, gfn_t gfn, gfn_t next, struct dat_walk *w);
};

struct dat_walk {
        const struct dat_walk_ops *ops;
        union crste *last;
        union pte *last_pte;
        union asce asce;
        gfn_t start;
        gfn_t end;
        int flags;
        void *priv;
};

struct ptval_param {
        unsigned char offset : 6;
        unsigned char len : 2;
};

/**
 * _pte() - Useful constructor for union pte
 * @pfn: the pfn this pte should point to.
 * @writable: whether the pte should be writable.
 * @dirty: whether the pte should be dirty.
 * @special: whether the pte should be marked as special
 *
 * The pte is also marked as young and present. If the pte is marked as dirty,
 * it gets marked as soft-dirty too. If the pte is not dirty, the hardware
 * protect bit is set (independently of the write softbit); this way proper
 * dirty tracking can be performed.
 *
 * Return: a union pte value.
 */
static inline union pte _pte(kvm_pfn_t pfn, bool writable, bool dirty, bool special)
{
        union pte res = { .val = PFN_PHYS(pfn) };

        res.h.p = !dirty;
        res.s.y = 1;
        res.s.pr = 1;
        res.s.w = writable;
        res.s.d = dirty;
        res.s.sd = dirty;
        res.s.s = special;
        return res;
}

static inline union crste _crste_fc0(kvm_pfn_t pfn, int tt)
{
        union crste res = { .val = PFN_PHYS(pfn) };

        res.h.tt = tt;
        res.h.fc0.tl = _REGION_ENTRY_LENGTH;
        res.h.fc0.tf = 0;
        return res;
}

/**
 * _crste() - Useful constructor for union crste with FC=1
 * @pfn: the pfn this pte should point to.
 * @tt: the table type
 * @writable: whether the pte should be writable.
 * @dirty: whether the pte should be dirty.
 *
 * The crste is also marked as young and present. If the crste is marked as
 * dirty, it gets marked as soft-dirty too. If the crste is not dirty, the
 * hardware protect bit is set (independently of the write softbit); this way
 * proper dirty tracking can be performed.
 *
 * Return: a union crste value.
 */
static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool writable, bool dirty)
{
        union crste res = { .val = PFN_PHYS(pfn) & _SEGMENT_MASK };

        res.h.tt = tt;
        res.h.p = !dirty;
        res.h.fc = 1;
        res.s.fc1.y = 1;
        res.s.fc1.pr = 1;
        res.s.fc1.w = writable;
        res.s.fc1.d = dirty;
        res.s.fc1.sd = dirty;
        return res;
}

union essa_state {
        unsigned char val;
        struct {
                unsigned char           : 2;
                unsigned char nodat     : 1;
                unsigned char exception : 1;
                unsigned char usage     : 2;
                unsigned char content   : 2;
        };
};

/**
 * struct vsie_rmap - reverse mapping for shadow page table entries
 * @next: pointer to next rmap in the list
 * @r_gfn: virtual rmap address in the shadow guest address space
 */
struct vsie_rmap {
        struct vsie_rmap *next;
        union {
                unsigned long val;
                struct {
                        long          level: 8;
                        unsigned long      : 4;
                        unsigned long r_gfn:52;
                };
        };
};

static_assert(sizeof(struct vsie_rmap) == 2 * sizeof(long));

#define KVM_S390_MMU_CACHE_N_CRSTS      6
#define KVM_S390_MMU_CACHE_N_PTS        2
#define KVM_S390_MMU_CACHE_N_RMAPS      16
struct kvm_s390_mmu_cache {
        void *crsts[KVM_S390_MMU_CACHE_N_CRSTS];
        void *pts[KVM_S390_MMU_CACHE_N_PTS];
        void *rmaps[KVM_S390_MMU_CACHE_N_RMAPS];
        short int n_crsts;
        short int n_pts;
        short int n_rmaps;
};

struct guest_fault {
        gfn_t gfn;              /* Guest frame */
        kvm_pfn_t pfn;          /* Host PFN */
        struct page *page;      /* Host page */
        union pte *ptep;        /* Used to resolve the fault, or NULL */
        union crste *crstep;    /* Used to resolve the fault, or NULL */
        bool writable;          /* Mapping is writable */
        bool write_attempt;     /* Write access attempted */
        bool attempt_pfault;    /* Attempt a pfault first */
        bool valid;             /* This entry contains valid data */
        void (*callback)(struct guest_fault *f);
        void *priv;
};

/*
 *      0       1       2       3       4       5       6       7
 *      +-------+-------+-------+-------+-------+-------+-------+-------+
 *  0   |                               |           PGT_ADDR            |
 *  8   |        VMADDR         |                                       |
 * 16   |                                                               |
 * 24   |                                                               |
 */
#define MKPTVAL(o, l) ((struct ptval_param) { .offset = (o), .len = ((l) + 1) / 2 - 1})
#define PTVAL_PGT_ADDR  MKPTVAL(4, 8)
#define PTVAL_VMADDR    MKPTVAL(8, 6)

union pgste __must_check __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new,
                                         gfn_t gfn, union asce asce, bool uses_skeys);
bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
                            union asce asce);
void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce);

long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce,
                         const struct dat_walk_ops *ops, int flags, void *priv);

int dat_entry_walk(struct kvm_s390_mmu_cache *mc, gfn_t gfn, union asce asce, int flags,
                   int walk_level, union crste **last, union pte **ptepp);
void dat_free_level(struct crst_table *table, bool owns_ptes);
struct crst_table *dat_alloc_crst_sleepable(unsigned long init);
int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newtype);
int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey);
int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn,
                        union skey skey, bool nq);
int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn,
                             union skey skey, union skey *oldkey, bool nq, bool mr, bool mc);
int dat_reset_reference_bit(union asce asce, gfn_t gfn);
long dat_reset_skeys(union asce asce, gfn_t start);

unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param);
void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val);

int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end,
                 u16 type, u16 param);
int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn);
bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end);

int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty);
long dat_reset_cmma(union asce asce, gfn_t start_gfn);
int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values);
int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem);
int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn,
                      unsigned long count, unsigned long mask, const uint8_t *bits);

int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc);

#define GFP_KVM_S390_MMU_CACHE (GFP_ATOMIC | __GFP_ACCOUNT | __GFP_NOWARN)

static inline struct page_table *kvm_s390_mmu_cache_alloc_pt(struct kvm_s390_mmu_cache *mc)
{
        if (mc->n_pts)
                return mc->pts[--mc->n_pts];
        return (void *)__get_free_page(GFP_KVM_S390_MMU_CACHE);
}

static inline struct crst_table *kvm_s390_mmu_cache_alloc_crst(struct kvm_s390_mmu_cache *mc)
{
        if (mc->n_crsts)
                return mc->crsts[--mc->n_crsts];
        return (void *)__get_free_pages(GFP_KVM_S390_MMU_CACHE | __GFP_COMP, CRST_ALLOC_ORDER);
}

static inline struct vsie_rmap *kvm_s390_mmu_cache_alloc_rmap(struct kvm_s390_mmu_cache *mc)
{
        if (mc->n_rmaps)
                return mc->rmaps[--mc->n_rmaps];
        return kzalloc_obj(struct vsie_rmap, GFP_KVM_S390_MMU_CACHE);
}

static inline struct crst_table *crste_table_start(union crste *crstep)
{
        return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE);
}

static inline struct page_table *pte_table_start(union pte *ptep)
{
        return (struct page_table *)ALIGN_DOWN((unsigned long)ptep, _PAGE_TABLE_SIZE);
}

static inline bool crdte_crste(union crste *crstep, union crste old, union crste new, gfn_t gfn,
                               union asce asce)
{
        unsigned long dtt = 0x10 | new.h.tt << 2;
        void *table = crste_table_start(crstep);

        return crdte(old.val, new.val, table, dtt, gfn_to_gpa(gfn), asce.val);
}

/**
 * idte_crste() - invalidate a crste entry using idte
 * @crstep: pointer to the crste to be invalidated
 * @gfn: a gfn mapped by the crste
 * @opt: options for the idte instruction
 * @asce: the asce
 * @local: whether the operation is cpu-local
 */
static __always_inline void idte_crste(union crste *crstep, gfn_t gfn, unsigned long opt,
                                       union asce asce, int local)
{
        unsigned long table_origin = __pa(crste_table_start(crstep));
        unsigned long gaddr = gfn_to_gpa(gfn) & HPAGE_MASK;

        if (__builtin_constant_p(opt) && opt == 0) {
                /* flush without guest asce */
                asm volatile("idte      %[table_origin],0,%[gaddr],%[local]"
                        : "+m" (*crstep)
                        : [table_origin] "a" (table_origin), [gaddr] "a" (gaddr),
                          [local] "i" (local)
                        : "cc");
        } else {
                /* flush with guest asce */
                asm volatile("idte %[table_origin],%[asce],%[gaddr_opt],%[local]"
                        : "+m" (*crstep)
                        : [table_origin] "a" (table_origin), [gaddr_opt] "a" (gaddr | opt),
                          [asce] "a" (asce.val), [local] "i" (local)
                        : "cc");
        }
}

static inline void dat_init_pgstes(struct page_table *pt, unsigned long val)
{
        memset64((void *)pt->pgstes, val, PTRS_PER_PTE);
}

static inline void dat_init_page_table(struct page_table *pt, unsigned long ptes,
                                       unsigned long pgstes)
{
        memset64((void *)pt->ptes, ptes, PTRS_PER_PTE);
        dat_init_pgstes(pt, pgstes);
}

static inline gfn_t asce_end(union asce asce)
{
        return 1ULL << ((asce.dt + 1) * 11 + _SEGMENT_SHIFT - PAGE_SHIFT);
}

#define _CRSTE(x) ((union crste) { .val = _Generic((x), \
                        union pgd : (x).val,            \
                        union p4d : (x).val,            \
                        union pud : (x).val,            \
                        union pmd : (x).val,            \
                        union crste : (x).val)})

#define _CRSTEP(x) ((union crste *)_Generic((*(x)),     \
                                union pgd : (x),        \
                                union p4d : (x),        \
                                union pud : (x),        \
                                union pmd : (x),        \
                                union crste : (x)))

#define _CRSTP(x) ((struct crst_table *)_Generic((*(x)),        \
                struct crst_table : (x),                        \
                struct segment_table : (x),                     \
                struct region3_table : (x),                     \
                struct region2_table : (x),                     \
                struct region1_table : (x)))

static inline bool asce_contains_gfn(union asce asce, gfn_t gfn)
{
        return gfn < asce_end(asce);
}

static inline bool is_pmd(union crste crste)
{
        return crste.h.tt == TABLE_TYPE_SEGMENT;
}

static inline bool is_pud(union crste crste)
{
        return crste.h.tt == TABLE_TYPE_REGION3;
}

static inline bool is_p4d(union crste crste)
{
        return crste.h.tt == TABLE_TYPE_REGION2;
}

static inline bool is_pgd(union crste crste)
{
        return crste.h.tt == TABLE_TYPE_REGION1;
}

static inline phys_addr_t pmd_origin_large(union pmd pmd)
{
        return pmd.val & _SEGMENT_ENTRY_ORIGIN_LARGE;
}

static inline phys_addr_t pud_origin_large(union pud pud)
{
        return pud.val & _REGION3_ENTRY_ORIGIN_LARGE;
}

/**
 * crste_origin_large() - Return the large frame origin of a large crste
 * @crste: The crste whose origin is to be returned. Should be either a
 *         region-3 table entry or a segment table entry, in both cases with
 *         FC set to 1 (large pages).
 *
 * Return: The origin of the large frame pointed to by @crste, or -1 if the
 *         crste was not large (wrong table type, or FC==0)
 */
static inline phys_addr_t crste_origin_large(union crste crste)
{
        if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3))
                return -1;
        if (is_pmd(crste))
                return pmd_origin_large(crste.pmd);
        return pud_origin_large(crste.pud);
}

#define crste_origin(x) (_Generic((x),                          \
                union pmd : (x).val & _SEGMENT_ENTRY_ORIGIN,    \
                union pud : (x).val & _REGION_ENTRY_ORIGIN,     \
                union p4d : (x).val & _REGION_ENTRY_ORIGIN,     \
                union pgd : (x).val & _REGION_ENTRY_ORIGIN))

static inline unsigned long pte_origin(union pte pte)
{
        return pte.val & PAGE_MASK;
}

static inline bool pmd_prefix(union pmd pmd)
{
        return pmd.h.fc && pmd.s.fc1.prefix_notif;
}

static inline bool pud_prefix(union pud pud)
{
        return pud.h.fc && pud.s.fc1.prefix_notif;
}

static inline bool crste_leaf(union crste crste)
{
        return (crste.h.tt <= TABLE_TYPE_REGION3) && crste.h.fc;
}

static inline bool crste_prefix(union crste crste)
{
        return crste_leaf(crste) && crste.s.fc1.prefix_notif;
}

static inline bool crste_dirty(union crste crste)
{
        return crste_leaf(crste) && crste.s.fc1.d;
}

static inline union pgste *pgste_of(union pte *pte)
{
        return (union pgste *)(pte + _PAGE_ENTRIES);
}

static inline bool pte_hole(union pte pte)
{
        return pte.h.i && !pte.tok.pr && pte.tok.type != _DAT_TOKEN_NONE;
}

static inline bool _crste_hole(union crste crste)
{
        return crste.h.i && !crste.tok.pr && crste.tok.type != _DAT_TOKEN_NONE;
}

#define crste_hole(x) _crste_hole(_CRSTE(x))

static inline bool _crste_none(union crste crste)
{
        return crste.h.i && !crste.tok.pr && crste.tok.type == _DAT_TOKEN_NONE;
}

#define crste_none(x) _crste_none(_CRSTE(x))

static inline phys_addr_t large_pud_to_phys(union pud pud, gfn_t gfn)
{
        return pud_origin_large(pud) | (gfn_to_gpa(gfn) & ~_REGION3_MASK);
}

static inline phys_addr_t large_pmd_to_phys(union pmd pmd, gfn_t gfn)
{
        return pmd_origin_large(pmd) | (gfn_to_gpa(gfn) & ~_SEGMENT_MASK);
}

static inline phys_addr_t large_crste_to_phys(union crste crste, gfn_t gfn)
{
        if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3))
                return -1;
        if (is_pmd(crste))
                return large_pmd_to_phys(crste.pmd, gfn);
        return large_pud_to_phys(crste.pud, gfn);
}

static inline bool cspg_crste(union crste *crstep, union crste old, union crste new)
{
        return cspg(&crstep->val, old.val, new.val);
}

static inline struct page_table *dereference_pmd(union pmd pmd)
{
        return phys_to_virt(crste_origin(pmd));
}

static inline struct segment_table *dereference_pud(union pud pud)
{
        return phys_to_virt(crste_origin(pud));
}

static inline struct region3_table *dereference_p4d(union p4d p4d)
{
        return phys_to_virt(crste_origin(p4d));
}

static inline struct region2_table *dereference_pgd(union pgd pgd)
{
        return phys_to_virt(crste_origin(pgd));
}

static inline struct crst_table *_dereference_crste(union crste crste)
{
        if (unlikely(is_pmd(crste)))
                return NULL;
        return phys_to_virt(crste_origin(crste.pud));
}

#define dereference_crste(x) (_Generic((x),                     \
                union pud : _dereference_crste(_CRSTE(x)),      \
                union p4d : _dereference_crste(_CRSTE(x)),      \
                union pgd : _dereference_crste(_CRSTE(x)),      \
                union crste : _dereference_crste(_CRSTE(x))))

static inline struct crst_table *dereference_asce(union asce asce)
{
        return phys_to_virt(asce.val & _ASCE_ORIGIN);
}

static inline void asce_flush_tlb(union asce asce)
{
        __tlb_flush_idte(asce.val);
}

static inline bool pgste_get_trylock(union pte *ptep, union pgste *res)
{
        union pgste *pgstep = pgste_of(ptep);
        union pgste old_pgste;

        if (READ_ONCE(pgstep->val) & PGSTE_PCL_BIT)
                return false;
        old_pgste.val = __atomic64_or_barrier(PGSTE_PCL_BIT, &pgstep->val);
        if (old_pgste.pcl)
                return false;
        old_pgste.pcl = 1;
        *res = old_pgste;
        return true;
}

static inline union pgste pgste_get_lock(union pte *ptep)
{
        union pgste res;

        while (!pgste_get_trylock(ptep, &res))
                cpu_relax();
        return res;
}

static inline void pgste_set_unlock(union pte *ptep, union pgste pgste)
{
        pgste.pcl = 0;
        barrier();
        WRITE_ONCE(*pgste_of(ptep), pgste);
}

static inline void dat_ptep_xchg(union pte *ptep, union pte new, gfn_t gfn, union asce asce,
                                 bool has_skeys)
{
        union pgste pgste;

        pgste = pgste_get_lock(ptep);
        pgste = __dat_ptep_xchg(ptep, pgste, new, gfn, asce, has_skeys);
        pgste_set_unlock(ptep, pgste);
}

static inline void dat_ptep_clear(union pte *ptep, gfn_t gfn, union asce asce, bool has_skeys)
{
        dat_ptep_xchg(ptep, _PTE_EMPTY, gfn, asce, has_skeys);
}

static inline void dat_free_pt(struct page_table *pt)
{
        free_page((unsigned long)pt);
}

static inline void _dat_free_crst(struct crst_table *table)
{
        free_pages((unsigned long)table, CRST_ALLOC_ORDER);
}

#define dat_free_crst(x) _dat_free_crst(_CRSTP(x))

static inline void kvm_s390_free_mmu_cache(struct kvm_s390_mmu_cache *mc)
{
        if (!mc)
                return;
        while (mc->n_pts)
                dat_free_pt(mc->pts[--mc->n_pts]);
        while (mc->n_crsts)
                _dat_free_crst(mc->crsts[--mc->n_crsts]);
        while (mc->n_rmaps)
                kfree(mc->rmaps[--mc->n_rmaps]);
        kfree(mc);
}

DEFINE_FREE(kvm_s390_mmu_cache, struct kvm_s390_mmu_cache *, if (_T) kvm_s390_free_mmu_cache(_T))

static inline struct kvm_s390_mmu_cache *kvm_s390_new_mmu_cache(void)
{
        struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;

        mc = kzalloc_obj(*mc, GFP_KERNEL_ACCOUNT);
        if (mc && !kvm_s390_mmu_cache_topup(mc))
                return_ptr(mc);
        return NULL;
}

static inline bool dat_pmdp_xchg_atomic(union pmd *pmdp, union pmd old, union pmd new,
                                        gfn_t gfn, union asce asce)
{
        return dat_crstep_xchg_atomic(_CRSTEP(pmdp), _CRSTE(old), _CRSTE(new), gfn, asce);
}

static inline bool dat_pudp_xchg_atomic(union pud *pudp, union pud old, union pud new,
                                        gfn_t gfn, union asce asce)
{
        return dat_crstep_xchg_atomic(_CRSTEP(pudp), _CRSTE(old), _CRSTE(new), gfn, asce);
}

static inline union crste dat_crstep_clear_atomic(union crste *crstep, gfn_t gfn, union asce asce)
{
        union crste oldcrste, empty = _CRSTE_EMPTY(crstep->h.tt);

        do {
                oldcrste = READ_ONCE(*crstep);
        } while (!dat_crstep_xchg_atomic(crstep, oldcrste, empty, gfn, asce));
        return oldcrste;
}

static inline int get_level(union crste *crstep, union pte *ptep)
{
        return ptep ? TABLE_TYPE_PAGE_TABLE : crstep->h.tt;
}

static inline int dat_delete_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start,
                                  unsigned long npages)
{
        return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_PIC, PGM_ADDRESSING);
}

static inline int dat_create_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start,
                                  unsigned long npages)
{
        return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_NONE, 0);
}

static inline bool crste_is_ucas(union crste crste)
{
        return is_pmd(crste) && crste.h.i && crste.h.fc0.tl == 1 && crste.h.fc == 0;
}

#endif /* __KVM_S390_DAT_H */