// SPDX-License-Identifier: GPL-2.0-only
/*
 * AMD Secure AVIC Support (SEV-SNP Guests)
 *
 * Copyright (C) 2024 Advanced Micro Devices, Inc.
 *
 * Author: Neeraj Upadhyay <Neeraj.Upadhyay@amd.com>
 */

#include <linux/cc_platform.h>
#include <linux/cpumask.h>
#include <linux/percpu-defs.h>
#include <linux/align.h>

#include <asm/apic.h>
#include <asm/sev.h>

#include "local.h"

struct secure_avic_page {
        u8 regs[PAGE_SIZE];
} __aligned(PAGE_SIZE);

static struct secure_avic_page __percpu *savic_page __ro_after_init;

static int savic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
{
        return x2apic_enabled() && cc_platform_has(CC_ATTR_SNP_SECURE_AVIC);
}

static inline void *get_reg_bitmap(unsigned int cpu, unsigned int offset)
{
        return &per_cpu_ptr(savic_page, cpu)->regs[offset];
}

static inline void update_vector(unsigned int cpu, unsigned int offset,
                                 unsigned int vector, bool set)
{
        void *bitmap = get_reg_bitmap(cpu, offset);

        if (set)
                apic_set_vector(vector, bitmap);
        else
                apic_clear_vector(vector, bitmap);
}

#define SAVIC_ALLOWED_IRR       0x204

/*
 * When Secure AVIC is enabled, RDMSR/WRMSR of the APIC registers
 * result in #VC exception (for non-accelerated register accesses)
 * with VMEXIT_AVIC_NOACCEL error code. The #VC exception handler
 * can read/write the x2APIC register in the guest APIC backing page.
 *
 * Since doing this would increase the latency of accessing x2APIC
 * registers, instead of doing RDMSR/WRMSR based accesses and
 * handling the APIC register reads/writes in the #VC exception handler,
 * the read() and write() callbacks directly read/write the APIC register
 * from/to the vCPU's APIC backing page.
 */
static u32 savic_read(u32 reg)
{
        void *ap = this_cpu_ptr(savic_page);

        switch (reg) {
        case APIC_LVTT:
        case APIC_TMICT:
        case APIC_TMCCT:
        case APIC_TDCR:
        case APIC_LVTTHMR:
        case APIC_LVTPC:
        case APIC_LVT0:
        case APIC_LVT1:
        case APIC_LVTERR:
                return savic_ghcb_msr_read(reg);
        case APIC_ID:
        case APIC_LVR:
        case APIC_TASKPRI:
        case APIC_ARBPRI:
        case APIC_PROCPRI:
        case APIC_LDR:
        case APIC_SPIV:
        case APIC_ESR:
        case APIC_EFEAT:
        case APIC_ECTRL:
        case APIC_SEOI:
        case APIC_IER:
        case APIC_EILVTn(0) ... APIC_EILVTn(3):
                return apic_get_reg(ap, reg);
        case APIC_ICR:
                return (u32)apic_get_reg64(ap, reg);
        case APIC_ISR ... APIC_ISR + 0x70:
        case APIC_TMR ... APIC_TMR + 0x70:
                if (WARN_ONCE(!IS_ALIGNED(reg, 16),
                              "APIC register read offset 0x%x not aligned at 16 bytes", reg))
                        return 0;
                return apic_get_reg(ap, reg);
        /* IRR and ALLOWED_IRR offset range */
        case APIC_IRR ... APIC_IRR + 0x74:
                /*
                 * Valid APIC_IRR/SAVIC_ALLOWED_IRR registers are at 16 bytes strides from
                 * their respective base offset. APIC_IRRs are in the range
                 *
                 * (0x200, 0x210,  ..., 0x270)
                 *
                 * while the SAVIC_ALLOWED_IRR range starts 4 bytes later, in the range
                 *
                 * (0x204, 0x214, ..., 0x274).
                 *
                 * Filter out everything else.
                 */
                if (WARN_ONCE(!(IS_ALIGNED(reg, 16) ||
                                IS_ALIGNED(reg - 4, 16)),
                              "Misaligned APIC_IRR/ALLOWED_IRR APIC register read offset 0x%x", reg))
                        return 0;
                return apic_get_reg(ap, reg);
        default:
                pr_err("Error reading unknown Secure AVIC reg offset 0x%x\n", reg);
                return 0;
        }
}

#define SAVIC_NMI_REQ           0x278

/*
 * On WRMSR to APIC_SELF_IPI register by the guest, Secure AVIC hardware
 * updates the APIC_IRR in the APIC backing page of the vCPU. In addition,
 * hardware evaluates the new APIC_IRR update for interrupt injection to
 * the vCPU. So, self IPIs are hardware-accelerated.
 */
static inline void self_ipi_reg_write(unsigned int vector)
{
        native_apic_msr_write(APIC_SELF_IPI, vector);
}

static void send_ipi_dest(unsigned int cpu, unsigned int vector, bool nmi)
{
        if (nmi)
                apic_set_reg(per_cpu_ptr(savic_page, cpu), SAVIC_NMI_REQ, 1);
        else
                update_vector(cpu, APIC_IRR, vector, true);
}

static void send_ipi_allbut(unsigned int vector, bool nmi)
{
        unsigned int cpu, src_cpu;

        guard(irqsave)();

        src_cpu = raw_smp_processor_id();

        for_each_cpu(cpu, cpu_online_mask) {
                if (cpu == src_cpu)
                        continue;
                send_ipi_dest(cpu, vector, nmi);
        }
}

static inline void self_ipi(unsigned int vector, bool nmi)
{
        u32 icr_low = APIC_SELF_IPI | vector;

        if (nmi)
                icr_low |= APIC_DM_NMI;

        native_x2apic_icr_write(icr_low, 0);
}

static void savic_icr_write(u32 icr_low, u32 icr_high)
{
        unsigned int dsh, vector;
        u64 icr_data;
        bool nmi;

        dsh = icr_low & APIC_DEST_ALLBUT;
        vector = icr_low & APIC_VECTOR_MASK;
        nmi = ((icr_low & APIC_DM_FIXED_MASK) == APIC_DM_NMI);

        switch (dsh) {
        case APIC_DEST_SELF:
                self_ipi(vector, nmi);
                break;
        case APIC_DEST_ALLINC:
                self_ipi(vector, nmi);
                fallthrough;
        case APIC_DEST_ALLBUT:
                send_ipi_allbut(vector, nmi);
                break;
        default:
                send_ipi_dest(icr_high, vector, nmi);
                break;
        }

        icr_data = ((u64)icr_high) << 32 | icr_low;
        if (dsh != APIC_DEST_SELF)
                savic_ghcb_msr_write(APIC_ICR, icr_data);
        apic_set_reg64(this_cpu_ptr(savic_page), APIC_ICR, icr_data);
}

static void savic_write(u32 reg, u32 data)
{
        void *ap = this_cpu_ptr(savic_page);

        switch (reg) {
        case APIC_LVTT:
        case APIC_TMICT:
        case APIC_TDCR:
        case APIC_LVT0:
        case APIC_LVT1:
        case APIC_LVTTHMR:
        case APIC_LVTPC:
        case APIC_LVTERR:
                savic_ghcb_msr_write(reg, data);
                break;
        case APIC_TASKPRI:
        case APIC_EOI:
        case APIC_SPIV:
        case SAVIC_NMI_REQ:
        case APIC_ESR:
        case APIC_ECTRL:
        case APIC_SEOI:
        case APIC_IER:
        case APIC_EILVTn(0) ... APIC_EILVTn(3):
                apic_set_reg(ap, reg, data);
                break;
        case APIC_ICR:
                savic_icr_write(data, 0);
                break;
        case APIC_SELF_IPI:
                self_ipi_reg_write(data);
                break;
        /* ALLOWED_IRR offsets are writable */
        case SAVIC_ALLOWED_IRR ... SAVIC_ALLOWED_IRR + 0x70:
                if (IS_ALIGNED(reg - 4, 16)) {
                        apic_set_reg(ap, reg, data);
                        break;
                }
                fallthrough;
        default:
                pr_err("Error writing unknown Secure AVIC reg offset 0x%x\n", reg);
        }
}

static void send_ipi(u32 dest, unsigned int vector, unsigned int dsh)
{
        unsigned int icr_low;

        icr_low = __prepare_ICR(dsh, vector, APIC_DEST_PHYSICAL);
        savic_icr_write(icr_low, dest);
}

static void savic_send_ipi(int cpu, int vector)
{
        u32 dest = per_cpu(x86_cpu_to_apicid, cpu);

        send_ipi(dest, vector, 0);
}

static void send_ipi_mask(const struct cpumask *mask, unsigned int vector, bool excl_self)
{
        unsigned int cpu, this_cpu;

        guard(irqsave)();

        this_cpu = raw_smp_processor_id();

        for_each_cpu(cpu, mask) {
                if (excl_self && cpu == this_cpu)
                        continue;
                send_ipi(per_cpu(x86_cpu_to_apicid, cpu), vector, 0);
        }
}

static void savic_send_ipi_mask(const struct cpumask *mask, int vector)
{
        send_ipi_mask(mask, vector, false);
}

static void savic_send_ipi_mask_allbutself(const struct cpumask *mask, int vector)
{
        send_ipi_mask(mask, vector, true);
}

static void savic_send_ipi_allbutself(int vector)
{
        send_ipi(0, vector, APIC_DEST_ALLBUT);
}

static void savic_send_ipi_all(int vector)
{
        send_ipi(0, vector, APIC_DEST_ALLINC);
}

static void savic_send_ipi_self(int vector)
{
        self_ipi_reg_write(vector);
}

static void savic_update_vector(unsigned int cpu, unsigned int vector, bool set)
{
        update_vector(cpu, SAVIC_ALLOWED_IRR, vector, set);
}

static void savic_eoi(void)
{
        unsigned int cpu;
        int vec;

        cpu = raw_smp_processor_id();
        vec = apic_find_highest_vector(get_reg_bitmap(cpu, APIC_ISR));
        if (WARN_ONCE(vec == -1, "EOI write while no active interrupt in APIC_ISR"))
                return;

        /* Is level-triggered interrupt? */
        if (apic_test_vector(vec, get_reg_bitmap(cpu, APIC_TMR))) {
                update_vector(cpu, APIC_ISR, vec, false);
                /*
                 * Propagate the EOI write to the hypervisor for level-triggered
                 * interrupts. Return to the guest from GHCB protocol event takes
                 * care of re-evaluating interrupt state.
                 */
                savic_ghcb_msr_write(APIC_EOI, 0);
        } else {
                /*
                 * Hardware clears APIC_ISR and re-evaluates the interrupt state
                 * to determine if there is any pending interrupt which can be
                 * delivered to CPU.
                 */
                native_apic_msr_eoi();
        }
}

static void savic_teardown(void)
{
        /* Disable Secure AVIC */
        native_wrmsrq(MSR_AMD64_SAVIC_CONTROL, 0);
        savic_unregister_gpa(NULL);
}

static void savic_setup(void)
{
        void *ap = this_cpu_ptr(savic_page);
        enum es_result res;
        unsigned long gpa;

        /*
         * Before Secure AVIC is enabled, APIC MSR reads are intercepted.
         * APIC_ID MSR read returns the value from the hypervisor.
         */
        apic_set_reg(ap, APIC_ID, native_apic_msr_read(APIC_ID));

        gpa = __pa(ap);

        /*
         * The NPT entry for a vCPU's APIC backing page must always be
         * present when the vCPU is running in order for Secure AVIC to
         * function. A VMEXIT_BUSY is returned on VMRUN and the vCPU cannot
         * be resumed if the NPT entry for the APIC backing page is not
         * present. Notify GPA of the vCPU's APIC backing page to the
         * hypervisor by calling savic_register_gpa(). Before executing
         * VMRUN, the hypervisor makes use of this information to make sure
         * the APIC backing page is mapped in NPT.
         */
        res = savic_register_gpa(gpa);
        if (res != ES_OK)
                sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL);

        native_wrmsrq(MSR_AMD64_SAVIC_CONTROL,
                      gpa | MSR_AMD64_SAVIC_EN | MSR_AMD64_SAVIC_ALLOWEDNMI);
}

static int savic_probe(void)
{
        if (!cc_platform_has(CC_ATTR_SNP_SECURE_AVIC))
                return 0;

        if (!x2apic_mode) {
                pr_err("Secure AVIC enabled in non x2APIC mode\n");
                sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL);
                /* unreachable */
        }

        savic_page = alloc_percpu(struct secure_avic_page);
        if (!savic_page)
                sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL);

        return 1;
}

static struct apic apic_x2apic_savic __ro_after_init = {

        .name                           = "secure avic x2apic",
        .probe                          = savic_probe,
        .acpi_madt_oem_check            = savic_acpi_madt_oem_check,
        .setup                          = savic_setup,
        .teardown                       = savic_teardown,

        .dest_mode_logical              = false,

        .disable_esr                    = 0,

        .cpu_present_to_apicid          = default_cpu_present_to_apicid,

        .max_apic_id                    = UINT_MAX,
        .x2apic_set_max_apicid          = true,
        .get_apic_id                    = x2apic_get_apic_id,

        .calc_dest_apicid               = apic_default_calc_apicid,

        .send_IPI                       = savic_send_ipi,
        .send_IPI_mask                  = savic_send_ipi_mask,
        .send_IPI_mask_allbutself       = savic_send_ipi_mask_allbutself,
        .send_IPI_allbutself            = savic_send_ipi_allbutself,
        .send_IPI_all                   = savic_send_ipi_all,
        .send_IPI_self                  = savic_send_ipi_self,

        .nmi_to_offline_cpu             = true,

        .read                           = savic_read,
        .write                          = savic_write,
        .eoi                            = savic_eoi,
        .icr_read                       = native_x2apic_icr_read,
        .icr_write                      = savic_icr_write,

        .update_vector                  = savic_update_vector,
};

apic_driver(apic_x2apic_savic);