// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
 */

#include <linux/err.h>
#include <linux/errno.h>
#include <linux/kvm_host.h>
#include <linux/module.h>
#include <linux/preempt.h>
#include <linux/vmalloc.h>
#include <trace/events/kvm.h>
#include <asm/fpu.h>
#include <asm/inst.h>
#include <asm/loongarch.h>
#include <asm/mmzone.h>
#include <asm/numa.h>
#include <asm/time.h>
#include <asm/tlb.h>
#include <asm/kvm_csr.h>
#include <asm/kvm_vcpu.h>
#include "trace.h"

static int kvm_emu_cpucfg(struct kvm_vcpu *vcpu, larch_inst inst)
{
        int rd, rj;
        unsigned int index, ret;

        if (inst.reg2_format.opcode != cpucfg_op)
                return EMULATE_FAIL;

        rd = inst.reg2_format.rd;
        rj = inst.reg2_format.rj;
        ++vcpu->stat.cpucfg_exits;
        index = vcpu->arch.gprs[rj];

        /*
         * By LoongArch Reference Manual 2.2.10.5
         * Return value is 0 for undefined CPUCFG index
         *
         * Disable preemption since hw gcsr is accessed
         */
        preempt_disable();
        switch (index) {
        case 0 ... (KVM_MAX_CPUCFG_REGS - 1):
                vcpu->arch.gprs[rd] = vcpu->arch.cpucfg[index];
                break;
        case CPUCFG_KVM_SIG:
                /* CPUCFG emulation between 0x40000000 -- 0x400000ff */
                vcpu->arch.gprs[rd] = *(unsigned int *)KVM_SIGNATURE;
                break;
        case CPUCFG_KVM_FEATURE:
                ret = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK;
                vcpu->arch.gprs[rd] = ret;
                break;
        default:
                vcpu->arch.gprs[rd] = 0;
                break;
        }
        preempt_enable();

        return EMULATE_DONE;
}

static unsigned long kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid)
{
        unsigned long val = 0;
        struct loongarch_csrs *csr = vcpu->arch.csr;

        /*
         * From LoongArch Reference Manual Volume 1 Chapter 4.2.1
         * For undefined CSR id, return value is 0
         */
        if (get_gcsr_flag(csrid) & SW_GCSR)
                val = kvm_read_sw_gcsr(csr, csrid);
        else
                pr_warn_once("Unsupported csrrd 0x%x with pc %lx\n", csrid, vcpu->arch.pc);

        return val;
}

static unsigned long kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, unsigned long val)
{
        unsigned long old = 0;
        struct loongarch_csrs *csr = vcpu->arch.csr;

        if (get_gcsr_flag(csrid) & SW_GCSR) {
                old = kvm_read_sw_gcsr(csr, csrid);
                kvm_write_sw_gcsr(csr, csrid, val);
        } else
                pr_warn_once("Unsupported csrwr 0x%x with pc %lx\n", csrid, vcpu->arch.pc);

        return old;
}

static unsigned long kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid,
                                unsigned long csr_mask, unsigned long val)
{
        unsigned long old = 0;
        struct loongarch_csrs *csr = vcpu->arch.csr;

        if (get_gcsr_flag(csrid) & SW_GCSR) {
                old = kvm_read_sw_gcsr(csr, csrid);
                val = (old & ~csr_mask) | (val & csr_mask);
                kvm_write_sw_gcsr(csr, csrid, val);
                old = old & csr_mask;
        } else
                pr_warn_once("Unsupported csrxchg 0x%x with pc %lx\n", csrid, vcpu->arch.pc);

        return old;
}

static int kvm_handle_csr(struct kvm_vcpu *vcpu, larch_inst inst)
{
        unsigned int rd, rj, csrid;
        unsigned long csr_mask, val = 0;

        /*
         * CSR value mask imm
         * rj = 0 means csrrd
         * rj = 1 means csrwr
         * rj != 0,1 means csrxchg
         */
        rd = inst.reg2csr_format.rd;
        rj = inst.reg2csr_format.rj;
        csrid = inst.reg2csr_format.csr;

        if (csrid >= LOONGARCH_CSR_PERFCTRL0 && csrid <= vcpu->arch.max_pmu_csrid) {
                if (kvm_guest_has_pmu(&vcpu->arch)) {
                        vcpu->arch.pc -= 4;
                        kvm_make_request(KVM_REQ_PMU, vcpu);
                        return EMULATE_DONE;
                }
        }

        /* Process CSR ops */
        switch (rj) {
        case 0: /* process csrrd */
                val = kvm_emu_read_csr(vcpu, csrid);
                vcpu->arch.gprs[rd] = val;
                break;
        case 1: /* process csrwr */
                val = vcpu->arch.gprs[rd];
                val = kvm_emu_write_csr(vcpu, csrid, val);
                vcpu->arch.gprs[rd] = val;
                break;
        default: /* process csrxchg */
                val = vcpu->arch.gprs[rd];
                csr_mask = vcpu->arch.gprs[rj];
                val = kvm_emu_xchg_csr(vcpu, csrid, csr_mask, val);
                vcpu->arch.gprs[rd] = val;
        }

        return EMULATE_DONE;
}

int kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu)
{
        int idx, ret;
        unsigned long *val;
        u32 addr, rd, rj, opcode;

        /*
         * Each IOCSR with different opcode
         */
        rd = inst.reg2_format.rd;
        rj = inst.reg2_format.rj;
        opcode = inst.reg2_format.opcode;
        addr = vcpu->arch.gprs[rj];
        run->iocsr_io.phys_addr = addr;
        run->iocsr_io.is_write = 0;
        val = &vcpu->arch.gprs[rd];

        /* LoongArch is Little endian */
        switch (opcode) {
        case iocsrrdb_op:
                run->iocsr_io.len = 1;
                break;
        case iocsrrdh_op:
                run->iocsr_io.len = 2;
                break;
        case iocsrrdw_op:
                run->iocsr_io.len = 4;
                break;
        case iocsrrdd_op:
                run->iocsr_io.len = 8;
                break;
        case iocsrwrb_op:
                run->iocsr_io.len = 1;
                run->iocsr_io.is_write = 1;
                break;
        case iocsrwrh_op:
                run->iocsr_io.len = 2;
                run->iocsr_io.is_write = 1;
                break;
        case iocsrwrw_op:
                run->iocsr_io.len = 4;
                run->iocsr_io.is_write = 1;
                break;
        case iocsrwrd_op:
                run->iocsr_io.len = 8;
                run->iocsr_io.is_write = 1;
                break;
        default:
                return EMULATE_FAIL;
        }

        if (run->iocsr_io.is_write) {
                idx = srcu_read_lock(&vcpu->kvm->srcu);
                ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, run->iocsr_io.len, val);
                srcu_read_unlock(&vcpu->kvm->srcu, idx);
                if (ret == 0)
                        ret = EMULATE_DONE;
                else {
                        ret = EMULATE_DO_IOCSR;
                        /* Save data and let user space to write it */
                        memcpy(run->iocsr_io.data, val, run->iocsr_io.len);
                }
                trace_kvm_iocsr(KVM_TRACE_IOCSR_WRITE, run->iocsr_io.len, addr, val);
        } else {
                vcpu->arch.io_gpr = rd; /* Set register id for iocsr read completion */
                idx = srcu_read_lock(&vcpu->kvm->srcu);
                ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr,
                                      run->iocsr_io.len, run->iocsr_io.data);
                srcu_read_unlock(&vcpu->kvm->srcu, idx);
                if (ret == 0) {
                        kvm_complete_iocsr_read(vcpu, run);
                        ret = EMULATE_DONE;
                } else
                        ret = EMULATE_DO_IOCSR;
                trace_kvm_iocsr(KVM_TRACE_IOCSR_READ, run->iocsr_io.len, addr, NULL);
        }

        return ret;
}

int kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run)
{
        enum emulation_result er = EMULATE_DONE;
        unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];

        switch (run->iocsr_io.len) {
        case 1:
                *gpr = *(s8 *)run->iocsr_io.data;
                break;
        case 2:
                *gpr = *(s16 *)run->iocsr_io.data;
                break;
        case 4:
                *gpr = *(s32 *)run->iocsr_io.data;
                break;
        case 8:
                *gpr = *(s64 *)run->iocsr_io.data;
                break;
        default:
                kvm_err("Bad IOCSR length: %d, addr is 0x%lx\n",
                                run->iocsr_io.len, vcpu->arch.badv);
                er = EMULATE_FAIL;
                break;
        }

        return er;
}

int kvm_emu_idle(struct kvm_vcpu *vcpu)
{
        ++vcpu->stat.idle_exits;
        trace_kvm_exit_idle(vcpu, KVM_TRACE_EXIT_IDLE);

        if (!kvm_arch_vcpu_runnable(vcpu))
                kvm_vcpu_halt(vcpu);

        return EMULATE_DONE;
}

static int kvm_trap_handle_gspr(struct kvm_vcpu *vcpu)
{
        unsigned long curr_pc;
        larch_inst inst;
        enum emulation_result er = EMULATE_DONE;
        struct kvm_run *run = vcpu->run;

        /* Fetch the instruction */
        inst.word = vcpu->arch.badi;
        curr_pc = vcpu->arch.pc;
        update_pc(&vcpu->arch);

        trace_kvm_exit_gspr(vcpu, inst.word);
        er = EMULATE_FAIL;
        switch (((inst.word >> 24) & 0xff)) {
        case 0x0: /* CPUCFG GSPR */
                trace_kvm_exit_cpucfg(vcpu, KVM_TRACE_EXIT_CPUCFG);
                er = kvm_emu_cpucfg(vcpu, inst);
                break;
        case 0x4: /* CSR{RD,WR,XCHG} GSPR */
                trace_kvm_exit_csr(vcpu, KVM_TRACE_EXIT_CSR);
                er = kvm_handle_csr(vcpu, inst);
                break;
        case 0x6: /* Cache, Idle and IOCSR GSPR */
                switch (((inst.word >> 22) & 0x3ff)) {
                case 0x18: /* Cache GSPR */
                        er = EMULATE_DONE;
                        trace_kvm_exit_cache(vcpu, KVM_TRACE_EXIT_CACHE);
                        break;
                case 0x19: /* Idle/IOCSR GSPR */
                        switch (((inst.word >> 15) & 0x1ffff)) {
                        case 0xc90: /* IOCSR GSPR */
                                er = kvm_emu_iocsr(inst, run, vcpu);
                                break;
                        case 0xc91: /* Idle GSPR */
                                er = kvm_emu_idle(vcpu);
                                break;
                        default:
                                er = EMULATE_FAIL;
                                break;
                        }
                        break;
                default:
                        er = EMULATE_FAIL;
                        break;
                }
                break;
        default:
                er = EMULATE_FAIL;
                break;
        }

        /* Rollback PC only if emulation was unsuccessful */
        if (er == EMULATE_FAIL) {
                kvm_err("[%#lx]%s: unsupported gspr instruction 0x%08x\n",
                        curr_pc, __func__, inst.word);

                kvm_arch_vcpu_dump_regs(vcpu);
                vcpu->arch.pc = curr_pc;
        }

        return er;
}

/*
 * Trigger GSPR:
 * 1) Execute CPUCFG instruction;
 * 2) Execute CACOP/IDLE instructions;
 * 3) Access to unimplemented CSRs/IOCSRs.
 */
static int kvm_handle_gspr(struct kvm_vcpu *vcpu, int ecode)
{
        int ret = RESUME_GUEST;
        enum emulation_result er = EMULATE_DONE;

        er = kvm_trap_handle_gspr(vcpu);

        if (er == EMULATE_DONE) {
                ret = RESUME_GUEST;
        } else if (er == EMULATE_DO_MMIO) {
                vcpu->run->exit_reason = KVM_EXIT_MMIO;
                ret = RESUME_HOST;
        } else if (er == EMULATE_DO_IOCSR) {
                vcpu->run->exit_reason = KVM_EXIT_LOONGARCH_IOCSR;
                ret = RESUME_HOST;
        } else {
                kvm_queue_exception(vcpu, EXCCODE_INE, 0);
                ret = RESUME_GUEST;
        }

        return ret;
}

int kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst)
{
        int idx, ret;
        unsigned int op8, opcode, rd;
        struct kvm_run *run = vcpu->run;

        run->mmio.phys_addr = vcpu->arch.badv;
        vcpu->mmio_needed = 2;  /* signed */
        op8 = (inst.word >> 24) & 0xff;
        ret = EMULATE_DO_MMIO;

        switch (op8) {
        case 0x24 ... 0x27:     /* ldptr.w/d process */
                rd = inst.reg2i14_format.rd;
                opcode = inst.reg2i14_format.opcode;

                switch (opcode) {
                case ldptrw_op:
                        run->mmio.len = 4;
                        break;
                case ldptrd_op:
                        run->mmio.len = 8;
                        break;
                default:
                        break;
                }
                break;
        case 0x28 ... 0x2e:     /* ld.b/h/w/d, ld.bu/hu/wu process */
                rd = inst.reg2i12_format.rd;
                opcode = inst.reg2i12_format.opcode;

                switch (opcode) {
                case ldb_op:
                        run->mmio.len = 1;
                        break;
                case ldbu_op:
                        vcpu->mmio_needed = 1;  /* unsigned */
                        run->mmio.len = 1;
                        break;
                case ldh_op:
                        run->mmio.len = 2;
                        break;
                case ldhu_op:
                        vcpu->mmio_needed = 1;  /* unsigned */
                        run->mmio.len = 2;
                        break;
                case ldw_op:
                        run->mmio.len = 4;
                        break;
                case ldwu_op:
                        vcpu->mmio_needed = 1;  /* unsigned */
                        run->mmio.len = 4;
                        break;
                case ldd_op:
                        run->mmio.len = 8;
                        break;
                default:
                        ret = EMULATE_FAIL;
                        break;
                }
                break;
        case 0x38:      /* ldx.b/h/w/d, ldx.bu/hu/wu process */
                rd = inst.reg3_format.rd;
                opcode = inst.reg3_format.opcode;

                switch (opcode) {
                case ldxb_op:
                        run->mmio.len = 1;
                        break;
                case ldxbu_op:
                        run->mmio.len = 1;
                        vcpu->mmio_needed = 1;  /* unsigned */
                        break;
                case ldxh_op:
                        run->mmio.len = 2;
                        break;
                case ldxhu_op:
                        run->mmio.len = 2;
                        vcpu->mmio_needed = 1;  /* unsigned */
                        break;
                case ldxw_op:
                        run->mmio.len = 4;
                        break;
                case ldxwu_op:
                        run->mmio.len = 4;
                        vcpu->mmio_needed = 1;  /* unsigned */
                        break;
                case ldxd_op:
                        run->mmio.len = 8;
                        break;
                default:
                        ret = EMULATE_FAIL;
                        break;
                }
                break;
        default:
                ret = EMULATE_FAIL;
        }

        if (ret == EMULATE_DO_MMIO) {
                trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len, run->mmio.phys_addr, NULL);

                vcpu->arch.io_gpr = rd; /* Set for kvm_complete_mmio_read() use */

                /*
                 * If mmio device such as PCH-PIC is emulated in KVM,
                 * it need not return to user space to handle the mmio
                 * exception.
                 */
                idx = srcu_read_lock(&vcpu->kvm->srcu);
                ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, vcpu->arch.badv,
                                      run->mmio.len, run->mmio.data);
                srcu_read_unlock(&vcpu->kvm->srcu, idx);
                if (!ret) {
                        kvm_complete_mmio_read(vcpu, run);
                        update_pc(&vcpu->arch);
                        vcpu->mmio_needed = 0;
                        return EMULATE_DONE;
                }

                run->mmio.is_write = 0;
                vcpu->mmio_is_write = 0;
                return EMULATE_DO_MMIO;
        }

        kvm_err("Read not supported Inst=0x%08x @%lx BadVaddr:%#lx\n",
                        inst.word, vcpu->arch.pc, vcpu->arch.badv);
        kvm_arch_vcpu_dump_regs(vcpu);
        vcpu->mmio_needed = 0;

        return ret;
}

int kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run)
{
        enum emulation_result er = EMULATE_DONE;
        unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];

        /* Update with new PC */
        update_pc(&vcpu->arch);
        switch (run->mmio.len) {
        case 1:
                if (vcpu->mmio_needed == 2)
                        *gpr = *(s8 *)run->mmio.data;
                else
                        *gpr = *(u8 *)run->mmio.data;
                break;
        case 2:
                if (vcpu->mmio_needed == 2)
                        *gpr = *(s16 *)run->mmio.data;
                else
                        *gpr = *(u16 *)run->mmio.data;
                break;
        case 4:
                if (vcpu->mmio_needed == 2)
                        *gpr = *(s32 *)run->mmio.data;
                else
                        *gpr = *(u32 *)run->mmio.data;
                break;
        case 8:
                *gpr = *(s64 *)run->mmio.data;
                break;
        default:
                kvm_err("Bad MMIO length: %d, addr is 0x%lx\n",
                                run->mmio.len, vcpu->arch.badv);
                er = EMULATE_FAIL;
                break;
        }

        trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len,
                        run->mmio.phys_addr, run->mmio.data);

        return er;
}

int kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst)
{
        int idx, ret;
        unsigned int rd, op8, opcode;
        unsigned long curr_pc, rd_val = 0;
        struct kvm_run *run = vcpu->run;
        void *data = run->mmio.data;

        /*
         * Update PC and hold onto current PC in case there is
         * an error and we want to rollback the PC
         */
        curr_pc = vcpu->arch.pc;
        update_pc(&vcpu->arch);

        op8 = (inst.word >> 24) & 0xff;
        run->mmio.phys_addr = vcpu->arch.badv;
        ret = EMULATE_DO_MMIO;
        switch (op8) {
        case 0x24 ... 0x27:     /* stptr.w/d process */
                rd = inst.reg2i14_format.rd;
                opcode = inst.reg2i14_format.opcode;

                switch (opcode) {
                case stptrw_op:
                        run->mmio.len = 4;
                        *(unsigned int *)data = vcpu->arch.gprs[rd];
                        break;
                case stptrd_op:
                        run->mmio.len = 8;
                        *(unsigned long *)data = vcpu->arch.gprs[rd];
                        break;
                default:
                        ret = EMULATE_FAIL;
                        break;
                }
                break;
        case 0x28 ... 0x2e:     /* st.b/h/w/d  process */
                rd = inst.reg2i12_format.rd;
                opcode = inst.reg2i12_format.opcode;
                rd_val = vcpu->arch.gprs[rd];

                switch (opcode) {
                case stb_op:
                        run->mmio.len = 1;
                        *(unsigned char *)data = rd_val;
                        break;
                case sth_op:
                        run->mmio.len = 2;
                        *(unsigned short *)data = rd_val;
                        break;
                case stw_op:
                        run->mmio.len = 4;
                        *(unsigned int *)data = rd_val;
                        break;
                case std_op:
                        run->mmio.len = 8;
                        *(unsigned long *)data = rd_val;
                        break;
                default:
                        ret = EMULATE_FAIL;
                        break;
                }
                break;
        case 0x38:      /* stx.b/h/w/d process */
                rd = inst.reg3_format.rd;
                opcode = inst.reg3_format.opcode;

                switch (opcode) {
                case stxb_op:
                        run->mmio.len = 1;
                        *(unsigned char *)data = vcpu->arch.gprs[rd];
                        break;
                case stxh_op:
                        run->mmio.len = 2;
                        *(unsigned short *)data = vcpu->arch.gprs[rd];
                        break;
                case stxw_op:
                        run->mmio.len = 4;
                        *(unsigned int *)data = vcpu->arch.gprs[rd];
                        break;
                case stxd_op:
                        run->mmio.len = 8;
                        *(unsigned long *)data = vcpu->arch.gprs[rd];
                        break;
                default:
                        ret = EMULATE_FAIL;
                        break;
                }
                break;
        default:
                ret = EMULATE_FAIL;
        }

        if (ret == EMULATE_DO_MMIO) {
                trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, run->mmio.len, run->mmio.phys_addr, data);

                /*
                 * If mmio device such as PCH-PIC is emulated in KVM,
                 * it need not return to user space to handle the mmio
                 * exception.
                 */
                idx = srcu_read_lock(&vcpu->kvm->srcu);
                ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, vcpu->arch.badv, run->mmio.len, data);
                srcu_read_unlock(&vcpu->kvm->srcu, idx);
                if (!ret)
                        return EMULATE_DONE;

                run->mmio.is_write = 1;
                vcpu->mmio_needed = 1;
                vcpu->mmio_is_write = 1;
                return EMULATE_DO_MMIO;
        }

        vcpu->arch.pc = curr_pc;
        kvm_err("Write not supported Inst=0x%08x @%lx BadVaddr:%#lx\n",
                        inst.word, vcpu->arch.pc, vcpu->arch.badv);
        kvm_arch_vcpu_dump_regs(vcpu);
        /* Rollback PC if emulation was unsuccessful */

        return ret;
}

static int kvm_handle_rdwr_fault(struct kvm_vcpu *vcpu, bool write, int ecode)
{
        int ret;
        larch_inst inst;
        enum emulation_result er = EMULATE_DONE;
        struct kvm_run *run = vcpu->run;
        unsigned long badv = vcpu->arch.badv;

        /* Inject ADE exception if exceed max GPA size */
        if (unlikely(badv >= vcpu->kvm->arch.gpa_size)) {
                kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM);
                return RESUME_GUEST;
        }

        ret = kvm_handle_mm_fault(vcpu, badv, write, ecode);
        if (ret) {
                /* Treat as MMIO */
                inst.word = vcpu->arch.badi;
                if (write) {
                        er = kvm_emu_mmio_write(vcpu, inst);
                } else {
                        /* A code fetch fault doesn't count as an MMIO */
                        if (kvm_is_ifetch_fault(&vcpu->arch)) {
                                kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEF);
                                return RESUME_GUEST;
                        }

                        er = kvm_emu_mmio_read(vcpu, inst);
                }
        }

        if (er == EMULATE_DONE) {
                ret = RESUME_GUEST;
        } else if (er == EMULATE_DO_MMIO) {
                run->exit_reason = KVM_EXIT_MMIO;
                ret = RESUME_HOST;
        } else {
                kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM);
                ret = RESUME_GUEST;
        }

        return ret;
}

static int kvm_handle_read_fault(struct kvm_vcpu *vcpu, int ecode)
{
        return kvm_handle_rdwr_fault(vcpu, false, ecode);
}

static int kvm_handle_write_fault(struct kvm_vcpu *vcpu, int ecode)
{
        return kvm_handle_rdwr_fault(vcpu, true, ecode);
}

int kvm_complete_user_service(struct kvm_vcpu *vcpu, struct kvm_run *run)
{
        update_pc(&vcpu->arch);
        kvm_write_reg(vcpu, LOONGARCH_GPR_A0, run->hypercall.ret);

        return 0;
}

/**
 * kvm_handle_fpu_disabled() - Guest used fpu however it is disabled at host
 * @vcpu:       Virtual CPU context.
 * @ecode:      Exception code.
 *
 * Handle when the guest attempts to use fpu which hasn't been allowed
 * by the root context.
 */
static int kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu, int ecode)
{
        struct kvm_run *run = vcpu->run;

        if (!kvm_guest_has_fpu(&vcpu->arch)) {
                kvm_queue_exception(vcpu, EXCCODE_INE, 0);
                return RESUME_GUEST;
        }

        /*
         * If guest FPU not present, the FPU operation should have been
         * treated as a reserved instruction!
         * If FPU already in use, we shouldn't get this at all.
         */
        if (WARN_ON(vcpu->arch.aux_inuse & KVM_LARCH_FPU)) {
                kvm_err("%s internal error\n", __func__);
                run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
                return RESUME_HOST;
        }

        vcpu->arch.aux_ldtype = KVM_LARCH_FPU;
        kvm_make_request(KVM_REQ_AUX_LOAD, vcpu);

        return RESUME_GUEST;
}

static long kvm_save_notify(struct kvm_vcpu *vcpu)
{
        unsigned long id, data;

        id   = kvm_read_reg(vcpu, LOONGARCH_GPR_A1);
        data = kvm_read_reg(vcpu, LOONGARCH_GPR_A2);
        switch (id) {
        case BIT(KVM_FEATURE_STEAL_TIME):
                if (data & ~(KVM_STEAL_PHYS_MASK | KVM_STEAL_PHYS_VALID))
                        return KVM_HCALL_INVALID_PARAMETER;

                vcpu->arch.st.guest_addr = data;
                if (!(data & KVM_STEAL_PHYS_VALID))
                        return 0;

                vcpu->arch.st.last_steal = current->sched_info.run_delay;
                kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
                return 0;
        default:
                return KVM_HCALL_INVALID_CODE;
        }
}

/*
 * kvm_handle_lsx_disabled() - Guest used LSX while disabled in root.
 * @vcpu:      Virtual CPU context.
 * @ecode:      Exception code.
 *
 * Handle when the guest attempts to use LSX when it is disabled in the root
 * context.
 */
static int kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu, int ecode)
{
        if (!kvm_guest_has_lsx(&vcpu->arch))
                kvm_queue_exception(vcpu, EXCCODE_INE, 0);
        else {
                vcpu->arch.aux_ldtype = KVM_LARCH_LSX;
                kvm_make_request(KVM_REQ_AUX_LOAD, vcpu);
        }

        return RESUME_GUEST;
}

/*
 * kvm_handle_lasx_disabled() - Guest used LASX while disabled in root.
 * @vcpu:       Virtual CPU context.
 * @ecode:      Exception code.
 *
 * Handle when the guest attempts to use LASX when it is disabled in the root
 * context.
 */
static int kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu, int ecode)
{
        if (!kvm_guest_has_lasx(&vcpu->arch))
                kvm_queue_exception(vcpu, EXCCODE_INE, 0);
        else {
                vcpu->arch.aux_ldtype = KVM_LARCH_LASX;
                kvm_make_request(KVM_REQ_AUX_LOAD, vcpu);
        }

        return RESUME_GUEST;
}

static int kvm_handle_lbt_disabled(struct kvm_vcpu *vcpu, int ecode)
{
        if (!kvm_guest_has_lbt(&vcpu->arch))
                kvm_queue_exception(vcpu, EXCCODE_INE, 0);
        else {
                vcpu->arch.aux_ldtype = KVM_LARCH_LBT;
                kvm_make_request(KVM_REQ_AUX_LOAD, vcpu);
        }

        return RESUME_GUEST;
}

static void kvm_send_pv_ipi(struct kvm_vcpu *vcpu)
{
        unsigned int min, cpu;
        struct kvm_vcpu *dest;
        DECLARE_BITMAP(ipi_bitmap, BITS_PER_LONG * 2) = {
                kvm_read_reg(vcpu, LOONGARCH_GPR_A1),
                kvm_read_reg(vcpu, LOONGARCH_GPR_A2)
        };

        min = kvm_read_reg(vcpu, LOONGARCH_GPR_A3);
        for_each_set_bit(cpu, ipi_bitmap, BITS_PER_LONG * 2) {
                dest = kvm_get_vcpu_by_cpuid(vcpu->kvm, cpu + min);
                if (!dest)
                        continue;

                /* Send SWI0 to dest vcpu to emulate IPI interrupt */
                kvm_queue_irq(dest, INT_SWI0);
                kvm_vcpu_kick(dest);
        }
}

/*
 * Hypercall emulation always return to guest, Caller should check retval.
 */
static void kvm_handle_service(struct kvm_vcpu *vcpu)
{
        long ret = KVM_HCALL_INVALID_CODE;
        unsigned long func = kvm_read_reg(vcpu, LOONGARCH_GPR_A0);

        switch (func) {
        case KVM_HCALL_FUNC_IPI:
                if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_IPI)) {
                        kvm_send_pv_ipi(vcpu);
                        ret = KVM_HCALL_SUCCESS;
                }
                break;
        case KVM_HCALL_FUNC_NOTIFY:
                if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME))
                        ret = kvm_save_notify(vcpu);
                break;
        default:
                break;
        }

        kvm_write_reg(vcpu, LOONGARCH_GPR_A0, ret);
}

static int kvm_handle_hypercall(struct kvm_vcpu *vcpu, int ecode)
{
        int ret;
        larch_inst inst;
        unsigned int code;

        inst.word = vcpu->arch.badi;
        code = inst.reg0i15_format.immediate;
        ret = RESUME_GUEST;

        switch (code) {
        case KVM_HCALL_SERVICE:
                vcpu->stat.hypercall_exits++;
                kvm_handle_service(vcpu);
                break;
        case KVM_HCALL_USER_SERVICE:
                if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_USER_HCALL)) {
                        kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE);
                        break;
                }

                vcpu->stat.hypercall_exits++;
                vcpu->run->exit_reason = KVM_EXIT_HYPERCALL;
                vcpu->run->hypercall.nr = KVM_HCALL_USER_SERVICE;
                vcpu->run->hypercall.args[0] = kvm_read_reg(vcpu, LOONGARCH_GPR_A0);
                vcpu->run->hypercall.args[1] = kvm_read_reg(vcpu, LOONGARCH_GPR_A1);
                vcpu->run->hypercall.args[2] = kvm_read_reg(vcpu, LOONGARCH_GPR_A2);
                vcpu->run->hypercall.args[3] = kvm_read_reg(vcpu, LOONGARCH_GPR_A3);
                vcpu->run->hypercall.args[4] = kvm_read_reg(vcpu, LOONGARCH_GPR_A4);
                vcpu->run->hypercall.args[5] = kvm_read_reg(vcpu, LOONGARCH_GPR_A5);
                vcpu->run->hypercall.flags = 0;
                /*
                 * Set invalid return value by default, let user-mode VMM modify it.
                 */
                vcpu->run->hypercall.ret = KVM_HCALL_INVALID_CODE;
                ret = RESUME_HOST;
                break;
        case KVM_HCALL_SWDBG:
                /* KVM_HCALL_SWDBG only in effective when SW_BP is enabled */
                if (vcpu->guest_debug & KVM_GUESTDBG_SW_BP_MASK) {
                        vcpu->run->exit_reason = KVM_EXIT_DEBUG;
                        ret = RESUME_HOST;
                        break;
                }
                fallthrough;
        default:
                /* Treat it as noop intruction, only set return value */
                kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE);
                break;
        }

        if (ret == RESUME_GUEST)
                update_pc(&vcpu->arch);

        return ret;
}

/*
 * LoongArch KVM callback handling for unimplemented guest exiting
 */
static int kvm_fault_ni(struct kvm_vcpu *vcpu, int ecode)
{
        unsigned int inst;
        unsigned long badv;

        /* Fetch the instruction */
        inst = vcpu->arch.badi;
        badv = vcpu->arch.badv;
        kvm_err("ECode: %d PC=%#lx Inst=0x%08x BadVaddr=%#lx ESTAT=%#lx\n",
                        ecode, vcpu->arch.pc, inst, badv, read_gcsr_estat());
        kvm_arch_vcpu_dump_regs(vcpu);
        kvm_queue_exception(vcpu, EXCCODE_INE, 0);

        return RESUME_GUEST;
}

static exit_handle_fn kvm_fault_tables[EXCCODE_INT_START] = {
        [0 ... EXCCODE_INT_START - 1]   = kvm_fault_ni,
        [EXCCODE_TLBI]                  = kvm_handle_read_fault,
        [EXCCODE_TLBL]                  = kvm_handle_read_fault,
        [EXCCODE_TLBS]                  = kvm_handle_write_fault,
        [EXCCODE_TLBM]                  = kvm_handle_write_fault,
        [EXCCODE_FPDIS]                 = kvm_handle_fpu_disabled,
        [EXCCODE_LSXDIS]                = kvm_handle_lsx_disabled,
        [EXCCODE_LASXDIS]               = kvm_handle_lasx_disabled,
        [EXCCODE_BTDIS]                 = kvm_handle_lbt_disabled,
        [EXCCODE_GSPR]                  = kvm_handle_gspr,
        [EXCCODE_HVC]                   = kvm_handle_hypercall,
};

int kvm_handle_fault(struct kvm_vcpu *vcpu, int fault)
{
        return kvm_fault_tables[fault](vcpu, fault);
}