/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
 */

#include <linux/linkage.h>
#include <asm/asm.h>
#include <asm/asmmacro.h>
#include <asm/loongarch.h>
#include <asm/regdef.h>
#include <asm/unwind_hints.h>

#define HGPR_OFFSET(x)          (PT_R0 + 8*x)
#define GGPR_OFFSET(x)          (KVM_ARCH_GGPR + 8*x)

.macro kvm_save_host_gpr base
        .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
        st.d    $r\n, \base, HGPR_OFFSET(\n)
        .endr
.endm

.macro kvm_restore_host_gpr base
        .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
        ld.d    $r\n, \base, HGPR_OFFSET(\n)
        .endr
.endm

/*
 * Save and restore all GPRs except base register,
 * and default value of base register is a2.
 */
.macro kvm_save_guest_gprs base
        .irp n,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
        st.d    $r\n, \base, GGPR_OFFSET(\n)
        .endr
.endm

.macro kvm_restore_guest_gprs base
        .irp n,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
        ld.d    $r\n, \base, GGPR_OFFSET(\n)
        .endr
.endm

/*
 * Prepare switch to guest, save host regs and restore guest regs.
 * a2: kvm_vcpu_arch, don't touch it until 'ertn'
 * t0, t1: temp register
 */
.macro kvm_switch_to_guest
        /* Set host ECFG.VS=0, all exceptions share one exception entry */
        csrrd           t0, LOONGARCH_CSR_ECFG
        bstrins.w       t0, zero, CSR_ECFG_VS_SHIFT_END, CSR_ECFG_VS_SHIFT
        csrwr           t0, LOONGARCH_CSR_ECFG

        /* Load up the new EENTRY */
        ld.d    t0, a2, KVM_ARCH_GEENTRY
        csrwr   t0, LOONGARCH_CSR_EENTRY

        /* Set Guest ERA */
        ld.d    t0, a2, KVM_ARCH_GPC
        csrwr   t0, LOONGARCH_CSR_ERA

        /* Load PGD for KVM hypervisor */
        ld.d    t0, a2, KVM_ARCH_KVMPGD
        csrwr   t0, LOONGARCH_CSR_PGDL

        /* Mix GID and RID */
        csrrd           t1, LOONGARCH_CSR_GSTAT
        bstrpick.w      t1, t1, CSR_GSTAT_GID_SHIFT_END, CSR_GSTAT_GID_SHIFT
        csrrd           t0, LOONGARCH_CSR_GTLBC
        bstrins.w       t0, t1, CSR_GTLBC_TGID_SHIFT_END, CSR_GTLBC_TGID_SHIFT
        csrwr           t0, LOONGARCH_CSR_GTLBC

        /*
         * Enable intr in root mode with future ertn so that host interrupt
         * can be responsed during VM runs
         * Guest CRMD comes from separate GCSR_CRMD register
         */
        ori     t0, zero, CSR_PRMD_PIE
        csrwr   t0, LOONGARCH_CSR_PRMD

        /* Set PVM bit to setup ertn to guest context */
        ori     t0, zero, CSR_GSTAT_PVM
        csrxchg t0, t0,   LOONGARCH_CSR_GSTAT

        /* Load Guest GPRs */
        kvm_restore_guest_gprs a2
        /* Load KVM_ARCH register */
        ld.d    a2, a2, (KVM_ARCH_GGPR + 8 * REG_A2)

        ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0, TLBRPRMD.ISTLBR = 0 */
.endm

        /*
         * Exception entry for general exception from guest mode
         *  - IRQ is disabled
         *  - kernel privilege in root mode
         *  - page mode keep unchanged from previous PRMD in root mode
         *  - Fixme: tlb exception cannot happen since registers relative with TLB
         *  -        is still in guest mode, such as pgd table/vmid registers etc,
         *  -        will fix with hw page walk enabled in future
         * load kvm_vcpu from reserved CSR KVM_VCPU_KS, and save a2 to KVM_TEMP_KS
         */
        .text
        .cfi_sections   .debug_frame
SYM_CODE_START(kvm_exc_entry)
        UNWIND_HINT_UNDEFINED
        csrwr   a2,   KVM_TEMP_KS
        csrrd   a2,   KVM_VCPU_KS
        addi.d  a2,   a2, KVM_VCPU_ARCH

        /* After save GPRs, free to use any GPR */
        kvm_save_guest_gprs a2
        /* Save guest A2 */
        csrrd   t0,     KVM_TEMP_KS
        st.d    t0,     a2,     (KVM_ARCH_GGPR + 8 * REG_A2)

        /* A2 is kvm_vcpu_arch, A1 is free to use */
        csrrd   s1,   KVM_VCPU_KS
        ld.d    s0,   s1, KVM_VCPU_RUN

        csrrd   t0,   LOONGARCH_CSR_ESTAT
        st.d    t0,   a2, KVM_ARCH_HESTAT
        csrrd   t0,   LOONGARCH_CSR_ERA
        st.d    t0,   a2, KVM_ARCH_GPC
        csrrd   t0,   LOONGARCH_CSR_BADV
        st.d    t0,   a2, KVM_ARCH_HBADV
        csrrd   t0,   LOONGARCH_CSR_BADI
        st.d    t0,   a2, KVM_ARCH_HBADI

        /* Restore host ECFG.VS */
        csrrd   t0, LOONGARCH_CSR_ECFG
        ld.d    t1, a2, KVM_ARCH_HECFG
        or      t0, t0, t1
        csrwr   t0, LOONGARCH_CSR_ECFG

        /* Restore host EENTRY */
        ld.d    t0, a2, KVM_ARCH_HEENTRY
        csrwr   t0, LOONGARCH_CSR_EENTRY

        /* Restore host pgd table */
        ld.d    t0, a2, KVM_ARCH_HPGD
        csrwr   t0, LOONGARCH_CSR_PGDL

        /*
         * Disable PGM bit to enter root mode by default with next ertn
         */
        ori     t0, zero, CSR_GSTAT_PVM
        csrxchg zero, t0, LOONGARCH_CSR_GSTAT

        /*
         * Clear GTLBC.TGID field
         *       0: for root  tlb update in future tlb instr
         *  others: for guest tlb update like gpa to hpa in future tlb instr
         */
        csrrd   t0, LOONGARCH_CSR_GTLBC
        bstrins.w       t0, zero, CSR_GTLBC_TGID_SHIFT_END, CSR_GTLBC_TGID_SHIFT
        csrwr   t0, LOONGARCH_CSR_GTLBC
        ld.d    tp, a2, KVM_ARCH_HTP
        ld.d    sp, a2, KVM_ARCH_HSP
        /* restore per cpu register */
        ld.d    u0, a2, KVM_ARCH_HPERCPU
        addi.d  sp, sp, -PT_SIZE

        /* Prepare handle exception */
        or      a0, s0, zero
        or      a1, s1, zero
        ld.d    t8, a2, KVM_ARCH_HANDLE_EXIT
        jirl    ra, t8, 0

        or      a2, s1, zero
        addi.d  a2, a2, KVM_VCPU_ARCH

        /* Resume host when ret <= 0 */
        blez    a0, ret_to_host

        /*
         * Return to guest
         * Save per cpu register again, maybe switched to another cpu
         */
        st.d    u0, a2, KVM_ARCH_HPERCPU

        /* Save kvm_vcpu to kscratch */
        csrwr   s1, KVM_VCPU_KS
        kvm_switch_to_guest

ret_to_host:
        ld.d    a2, a2, KVM_ARCH_HSP
        addi.d  a2, a2, -PT_SIZE
        kvm_restore_host_gpr    a2
        jr      ra

SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
SYM_CODE_END(kvm_exc_entry)

/*
 * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
 *
 * @register_param:
 *  a0: kvm_run* run
 *  a1: kvm_vcpu* vcpu
 */
SYM_FUNC_START(kvm_enter_guest)
        /* Allocate space in stack bottom */
        addi.d  a2, sp, -PT_SIZE
        /* Save host GPRs */
        kvm_save_host_gpr a2

        addi.d  a2, a1, KVM_VCPU_ARCH
        st.d    sp, a2, KVM_ARCH_HSP
        st.d    tp, a2, KVM_ARCH_HTP
        /* Save per cpu register */
        st.d    u0, a2, KVM_ARCH_HPERCPU

        /* Save kvm_vcpu to kscratch */
        csrwr   a1, KVM_VCPU_KS
        kvm_switch_to_guest
SYM_INNER_LABEL(kvm_enter_guest_end, SYM_L_LOCAL)
SYM_FUNC_END(kvm_enter_guest)

SYM_FUNC_START(kvm_save_fpu)
        fpu_save_csr    a0 t1
        fpu_save_double a0 t1
        fpu_save_cc     a0 t1 t2
        jr              ra
SYM_FUNC_END(kvm_save_fpu)

SYM_FUNC_START(kvm_restore_fpu)
        fpu_restore_double a0 t1
        fpu_restore_csr    a0 t1 t2
        fpu_restore_cc     a0 t1 t2
        jr                 ra
SYM_FUNC_END(kvm_restore_fpu)

#ifdef CONFIG_CPU_HAS_LSX
SYM_FUNC_START(kvm_save_lsx)
        fpu_save_csr    a0 t1
        fpu_save_cc     a0 t1 t2
        lsx_save_data   a0 t1
        jr              ra
SYM_FUNC_END(kvm_save_lsx)

SYM_FUNC_START(kvm_restore_lsx)
        lsx_restore_data a0 t1
        fpu_restore_cc   a0 t1 t2
        fpu_restore_csr  a0 t1 t2
        jr               ra
SYM_FUNC_END(kvm_restore_lsx)
#endif

#ifdef CONFIG_CPU_HAS_LASX
SYM_FUNC_START(kvm_save_lasx)
        fpu_save_csr    a0 t1
        fpu_save_cc     a0 t1 t2
        lasx_save_data  a0 t1
        jr              ra
SYM_FUNC_END(kvm_save_lasx)

SYM_FUNC_START(kvm_restore_lasx)
        lasx_restore_data a0 t1
        fpu_restore_cc    a0 t1 t2
        fpu_restore_csr   a0 t1 t2
        jr                ra
SYM_FUNC_END(kvm_restore_lasx)
#endif
        .section ".rodata"
SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry)
SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest)

#ifdef CONFIG_CPU_HAS_LBT
STACK_FRAME_NON_STANDARD kvm_restore_fpu
#ifdef CONFIG_CPU_HAS_LSX
STACK_FRAME_NON_STANDARD kvm_restore_lsx
#endif
#ifdef CONFIG_CPU_HAS_LASX
STACK_FRAME_NON_STANDARD kvm_restore_lasx
#endif
#endif