// SPDX-License-Identifier: GPL-2.0
/*
 * ARM Generic Interrupt Controller (GIC) v3 support
 */

#include <linux/sizes.h>

#include "kvm_util.h"
#include "processor.h"
#include "delay.h"

#include "gic.h"
#include "gic_v3.h"
#include "gic_private.h"

#define GICV3_MAX_CPUS                  512

#define GICD_INT_DEF_PRI                0xa0
#define GICD_INT_DEF_PRI_X4             ((GICD_INT_DEF_PRI << 24) |\
                                        (GICD_INT_DEF_PRI << 16) |\
                                        (GICD_INT_DEF_PRI << 8) |\
                                        GICD_INT_DEF_PRI)

#define ICC_PMR_DEF_PRIO                0xf0

struct gicv3_data {
        unsigned int nr_cpus;
        unsigned int nr_spis;
};

#define sgi_base_from_redist(redist_base)       (redist_base + SZ_64K)
#define DIST_BIT                                (1U << 31)

enum gicv3_intid_range {
        SGI_RANGE,
        PPI_RANGE,
        SPI_RANGE,
        INVALID_RANGE,
};

static struct gicv3_data gicv3_data;

static void gicv3_gicd_wait_for_rwp(void)
{
        unsigned int count = 100000; /* 1s */

        while (readl(GICD_BASE_GVA + GICD_CTLR) & GICD_CTLR_RWP) {
                GUEST_ASSERT(count--);
                udelay(10);
        }
}

static inline volatile void *gicr_base_cpu(u32 cpu)
{
        /* Align all the redistributors sequentially */
        return GICR_BASE_GVA + cpu * SZ_64K * 2;
}

static void gicv3_gicr_wait_for_rwp(u32 cpu)
{
        unsigned int count = 100000; /* 1s */

        while (readl(gicr_base_cpu(cpu) + GICR_CTLR) & GICR_CTLR_RWP) {
                GUEST_ASSERT(count--);
                udelay(10);
        }
}

static void gicv3_wait_for_rwp(u32 cpu_or_dist)
{
        if (cpu_or_dist & DIST_BIT)
                gicv3_gicd_wait_for_rwp();
        else
                gicv3_gicr_wait_for_rwp(cpu_or_dist);
}

static enum gicv3_intid_range get_intid_range(unsigned int intid)
{
        switch (intid) {
        case 0 ... 15:
                return SGI_RANGE;
        case 16 ... 31:
                return PPI_RANGE;
        case 32 ... 1019:
                return SPI_RANGE;
        }

        /* We should not be reaching here */
        GUEST_ASSERT(0);

        return INVALID_RANGE;
}

static u64 gicv3_read_iar(void)
{
        u64 irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1);

        dsb(sy);
        return irqstat;
}

static void gicv3_write_eoir(u32 irq)
{
        write_sysreg_s(irq, SYS_ICC_EOIR1_EL1);
        isb();
}

static void gicv3_write_dir(u32 irq)
{
        write_sysreg_s(irq, SYS_ICC_DIR_EL1);
        isb();
}

static void gicv3_set_priority_mask(u64 mask)
{
        write_sysreg_s(mask, SYS_ICC_PMR_EL1);
}

static void gicv3_set_eoi_split(bool split)
{
        u32 val;

        /*
         * All other fields are read-only, so no need to read CTLR first. In
         * fact, the kernel does the same.
         */
        val = split ? (1U << 1) : 0;
        write_sysreg_s(val, SYS_ICC_CTLR_EL1);
        isb();
}

u32 gicv3_reg_readl(u32 cpu_or_dist, u64 offset)
{
        volatile void *base = cpu_or_dist & DIST_BIT ? GICD_BASE_GVA
                        : sgi_base_from_redist(gicr_base_cpu(cpu_or_dist));
        return readl(base + offset);
}

void gicv3_reg_writel(u32 cpu_or_dist, u64 offset, u32 reg_val)
{
        volatile void *base = cpu_or_dist & DIST_BIT ? GICD_BASE_GVA
                        : sgi_base_from_redist(gicr_base_cpu(cpu_or_dist));
        writel(reg_val, base + offset);
}

u32 gicv3_getl_fields(u32 cpu_or_dist, u64 offset, u32 mask)
{
        return gicv3_reg_readl(cpu_or_dist, offset) & mask;
}

void gicv3_setl_fields(u32 cpu_or_dist, u64 offset,
                       u32 mask, u32 reg_val)
{
        u32 tmp = gicv3_reg_readl(cpu_or_dist, offset) & ~mask;

        tmp |= (reg_val & mask);
        gicv3_reg_writel(cpu_or_dist, offset, tmp);
}

/*
 * We use a single offset for the distributor and redistributor maps as they
 * have the same value in both. The only exceptions are registers that only
 * exist in one and not the other, like GICR_WAKER that doesn't exist in the
 * distributor map. Such registers are conveniently marked as reserved in the
 * map that doesn't implement it; like GICR_WAKER's offset of 0x0014 being
 * marked as "Reserved" in the Distributor map.
 */
static void gicv3_access_reg(u32 intid, u64 offset,
                             u32 reg_bits, u32 bits_per_field,
                             bool write, u32 *val)
{
        u32 cpu = guest_get_vcpuid();
        enum gicv3_intid_range intid_range = get_intid_range(intid);
        u32 fields_per_reg, index, mask, shift;
        u32 cpu_or_dist;

        GUEST_ASSERT(bits_per_field <= reg_bits);
        GUEST_ASSERT(!write || *val < (1U << bits_per_field));
        /*
         * This function does not support 64 bit accesses. Just asserting here
         * until we implement readq/writeq.
         */
        GUEST_ASSERT(reg_bits == 32);

        fields_per_reg = reg_bits / bits_per_field;
        index = intid % fields_per_reg;
        shift = index * bits_per_field;
        mask = ((1U << bits_per_field) - 1) << shift;

        /* Set offset to the actual register holding intid's config. */
        offset += (intid / fields_per_reg) * (reg_bits / 8);

        cpu_or_dist = (intid_range == SPI_RANGE) ? DIST_BIT : cpu;

        if (write)
                gicv3_setl_fields(cpu_or_dist, offset, mask, *val << shift);
        *val = gicv3_getl_fields(cpu_or_dist, offset, mask) >> shift;
}

static void gicv3_write_reg(u32 intid, u64 offset,
                            u32 reg_bits, u32 bits_per_field, u32 val)
{
        gicv3_access_reg(intid, offset, reg_bits,
                        bits_per_field, true, &val);
}

static u32 gicv3_read_reg(u32 intid, u64 offset,
                          u32 reg_bits, u32 bits_per_field)
{
        u32 val;

        gicv3_access_reg(intid, offset, reg_bits,
                        bits_per_field, false, &val);
        return val;
}

static void gicv3_set_priority(u32 intid, u32 prio)
{
        gicv3_write_reg(intid, GICD_IPRIORITYR, 32, 8, prio);
}

/* Sets the intid to be level-sensitive or edge-triggered. */
static void gicv3_irq_set_config(u32 intid, bool is_edge)
{
        u32 val;

        /* N/A for private interrupts. */
        GUEST_ASSERT(get_intid_range(intid) == SPI_RANGE);
        val = is_edge ? 2 : 0;
        gicv3_write_reg(intid, GICD_ICFGR, 32, 2, val);
}

static void gicv3_irq_enable(u32 intid)
{
        bool is_spi = get_intid_range(intid) == SPI_RANGE;
        u32 cpu = guest_get_vcpuid();

        gicv3_write_reg(intid, GICD_ISENABLER, 32, 1, 1);
        gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);
}

static void gicv3_irq_disable(u32 intid)
{
        bool is_spi = get_intid_range(intid) == SPI_RANGE;
        u32 cpu = guest_get_vcpuid();

        gicv3_write_reg(intid, GICD_ICENABLER, 32, 1, 1);
        gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);
}

static void gicv3_irq_set_active(u32 intid)
{
        gicv3_write_reg(intid, GICD_ISACTIVER, 32, 1, 1);
}

static void gicv3_irq_clear_active(u32 intid)
{
        gicv3_write_reg(intid, GICD_ICACTIVER, 32, 1, 1);
}

static bool gicv3_irq_get_active(u32 intid)
{
        return gicv3_read_reg(intid, GICD_ISACTIVER, 32, 1);
}

static void gicv3_irq_set_pending(u32 intid)
{
        gicv3_write_reg(intid, GICD_ISPENDR, 32, 1, 1);
}

static void gicv3_irq_clear_pending(u32 intid)
{
        gicv3_write_reg(intid, GICD_ICPENDR, 32, 1, 1);
}

static bool gicv3_irq_get_pending(u32 intid)
{
        return gicv3_read_reg(intid, GICD_ISPENDR, 32, 1);
}

static void gicv3_enable_redist(volatile void *redist_base)
{
        u32 val = readl(redist_base + GICR_WAKER);
        unsigned int count = 100000; /* 1s */

        val &= ~GICR_WAKER_ProcessorSleep;
        writel(val, redist_base + GICR_WAKER);

        /* Wait until the processor is 'active' */
        while (readl(redist_base + GICR_WAKER) & GICR_WAKER_ChildrenAsleep) {
                GUEST_ASSERT(count--);
                udelay(10);
        }
}

static void gicv3_set_group(u32 intid, bool grp)
{
        u32 cpu_or_dist;
        u32 val;

        cpu_or_dist = (get_intid_range(intid) == SPI_RANGE) ? DIST_BIT : guest_get_vcpuid();
        val = gicv3_reg_readl(cpu_or_dist, GICD_IGROUPR + (intid / 32) * 4);
        if (grp)
                val |= BIT(intid % 32);
        else
                val &= ~BIT(intid % 32);
        gicv3_reg_writel(cpu_or_dist, GICD_IGROUPR + (intid / 32) * 4, val);
}

static void gicv3_cpu_init(unsigned int cpu)
{
        volatile void *sgi_base;
        unsigned int i;
        volatile void *redist_base_cpu;
        u64 typer;

        GUEST_ASSERT(cpu < gicv3_data.nr_cpus);

        redist_base_cpu = gicr_base_cpu(cpu);
        sgi_base = sgi_base_from_redist(redist_base_cpu);

        /* Verify assumption that GICR_TYPER.Processor_number == cpu */
        typer = readq_relaxed(redist_base_cpu + GICR_TYPER);
        GUEST_ASSERT_EQ(GICR_TYPER_CPU_NUMBER(typer), cpu);

        gicv3_enable_redist(redist_base_cpu);

        /*
         * Mark all the SGI and PPI interrupts as non-secure Group-1.
         * Also, deactivate and disable them.
         */
        writel(~0, sgi_base + GICR_IGROUPR0);
        writel(~0, sgi_base + GICR_ICACTIVER0);
        writel(~0, sgi_base + GICR_ICENABLER0);

        /* Set a default priority for all the SGIs and PPIs */
        for (i = 0; i < 32; i += 4)
                writel(GICD_INT_DEF_PRI_X4,
                                sgi_base + GICR_IPRIORITYR0 + i);

        gicv3_gicr_wait_for_rwp(cpu);

        /* Enable the GIC system register (ICC_*) access */
        write_sysreg_s(read_sysreg_s(SYS_ICC_SRE_EL1) | ICC_SRE_EL1_SRE,
                        SYS_ICC_SRE_EL1);

        /* Set a default priority threshold */
        write_sysreg_s(ICC_PMR_DEF_PRIO, SYS_ICC_PMR_EL1);

        /* Disable Group-0 interrupts */
        write_sysreg_s(ICC_IGRPEN0_EL1_MASK, SYS_ICC_IGRPEN1_EL1);
        /* Enable non-secure Group-1 interrupts */
        write_sysreg_s(ICC_IGRPEN1_EL1_MASK, SYS_ICC_IGRPEN1_EL1);
}

static void gicv3_dist_init(void)
{
        unsigned int i;

        /* Disable the distributor until we set things up */
        writel(0, GICD_BASE_GVA + GICD_CTLR);
        gicv3_gicd_wait_for_rwp();

        /*
         * Mark all the SPI interrupts as non-secure Group-1.
         * Also, deactivate and disable them.
         */
        for (i = 32; i < gicv3_data.nr_spis; i += 32) {
                writel(~0, GICD_BASE_GVA + GICD_IGROUPR + i / 8);
                writel(~0, GICD_BASE_GVA + GICD_ICACTIVER + i / 8);
                writel(~0, GICD_BASE_GVA + GICD_ICENABLER + i / 8);
        }

        /* Set a default priority for all the SPIs */
        for (i = 32; i < gicv3_data.nr_spis; i += 4)
                writel(GICD_INT_DEF_PRI_X4,
                                GICD_BASE_GVA + GICD_IPRIORITYR + i);

        /* Wait for the settings to sync-in */
        gicv3_gicd_wait_for_rwp();

        /* Finally, enable the distributor globally with ARE */
        writel(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A |
                        GICD_CTLR_ENABLE_G1, GICD_BASE_GVA + GICD_CTLR);
        gicv3_gicd_wait_for_rwp();
}

static void gicv3_init(unsigned int nr_cpus)
{
        GUEST_ASSERT(nr_cpus <= GICV3_MAX_CPUS);

        gicv3_data.nr_cpus = nr_cpus;
        gicv3_data.nr_spis = GICD_TYPER_SPIS(
                                readl(GICD_BASE_GVA + GICD_TYPER));
        if (gicv3_data.nr_spis > 1020)
                gicv3_data.nr_spis = 1020;

        /*
         * Initialize only the distributor for now.
         * The redistributor and CPU interfaces are initialized
         * later for every PE.
         */
        gicv3_dist_init();
}

const struct gic_common_ops gicv3_ops = {
        .gic_init = gicv3_init,
        .gic_cpu_init = gicv3_cpu_init,
        .gic_irq_enable = gicv3_irq_enable,
        .gic_irq_disable = gicv3_irq_disable,
        .gic_read_iar = gicv3_read_iar,
        .gic_write_eoir = gicv3_write_eoir,
        .gic_write_dir = gicv3_write_dir,
        .gic_set_priority_mask = gicv3_set_priority_mask,
        .gic_set_eoi_split = gicv3_set_eoi_split,
        .gic_set_priority = gicv3_set_priority,
        .gic_irq_set_active = gicv3_irq_set_active,
        .gic_irq_clear_active = gicv3_irq_clear_active,
        .gic_irq_get_active = gicv3_irq_get_active,
        .gic_irq_set_pending = gicv3_irq_set_pending,
        .gic_irq_clear_pending = gicv3_irq_clear_pending,
        .gic_irq_get_pending = gicv3_irq_get_pending,
        .gic_irq_set_config = gicv3_irq_set_config,
        .gic_irq_set_group = gicv3_set_group,
};

void gic_rdist_enable_lpis(gpa_t cfg_table, size_t cfg_table_size,
                           gpa_t pend_table)
{
        volatile void *rdist_base = gicr_base_cpu(guest_get_vcpuid());

        u32 ctlr;
        u64 val;

        val = (cfg_table |
               GICR_PROPBASER_InnerShareable |
               GICR_PROPBASER_RaWaWb |
               ((ilog2(cfg_table_size) - 1) & GICR_PROPBASER_IDBITS_MASK));
        writeq_relaxed(val, rdist_base + GICR_PROPBASER);

        val = (pend_table |
               GICR_PENDBASER_InnerShareable |
               GICR_PENDBASER_RaWaWb);
        writeq_relaxed(val, rdist_base + GICR_PENDBASER);

        ctlr = readl_relaxed(rdist_base + GICR_CTLR);
        ctlr |= GICR_CTLR_ENABLE_LPIS;
        writel_relaxed(ctlr, rdist_base + GICR_CTLR);
}