arch/riscv/kvm/aia_device.c

root/arch/riscv/kvm/aia_device.c
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
 * Copyright (C) 2022 Ventana Micro Systems Inc.
 *
 * Authors:
 *      Anup Patel <apatel@ventanamicro.com>
 */

#include <linux/bits.h>
#include <linux/irqchip/riscv-imsic.h>
#include <linux/kvm_host.h>
#include <linux/uaccess.h>
#include <linux/cpufeature.h>

static int aia_create(struct kvm_device *dev, u32 type)
{
        int ret;
        unsigned long i;
        struct kvm *kvm = dev->kvm;
        struct kvm_vcpu *vcpu;

        if (irqchip_in_kernel(kvm))
                return -EEXIST;

        if (!riscv_isa_extension_available(NULL, SSAIA))
                return -ENODEV;

        ret = -EBUSY;
        if (kvm_trylock_all_vcpus(kvm))
                return ret;

        kvm_for_each_vcpu(i, vcpu, kvm) {
                if (vcpu->arch.ran_atleast_once)
                        goto out_unlock;
        }
        ret = 0;

        kvm->arch.aia.in_kernel = true;

out_unlock:
        kvm_unlock_all_vcpus(kvm);
        return ret;
}

static void aia_destroy(struct kvm_device *dev)
{
        kfree(dev);
}

static int aia_config(struct kvm *kvm, unsigned long type,
                      u32 *nr, bool write)
{
        struct kvm_aia *aia = &kvm->arch.aia;

        /* Writes can only be done before irqchip is initialized */
        if (write && kvm_riscv_aia_initialized(kvm))
                return -EBUSY;

        switch (type) {
        case KVM_DEV_RISCV_AIA_CONFIG_MODE:
                if (write) {
                        switch (*nr) {
                        case KVM_DEV_RISCV_AIA_MODE_EMUL:
                                break;
                        case KVM_DEV_RISCV_AIA_MODE_HWACCEL:
                        case KVM_DEV_RISCV_AIA_MODE_AUTO:
                                /*
                                 * HW Acceleration and Auto modes only
                                 * supported on host with non-zero guest
                                 * external interrupts (i.e. non-zero
                                 * VS-level IMSIC pages).
                                 */
                                if (!kvm_riscv_aia_nr_hgei)
                                        return -EINVAL;
                                break;
                        default:
                                return -EINVAL;
                        }
                        aia->mode = *nr;
                } else
                        *nr = aia->mode;
                break;
        case KVM_DEV_RISCV_AIA_CONFIG_IDS:
                if (write) {
                        if ((*nr < KVM_DEV_RISCV_AIA_IDS_MIN) ||
                            (*nr >= KVM_DEV_RISCV_AIA_IDS_MAX) ||
                            ((*nr & KVM_DEV_RISCV_AIA_IDS_MIN) !=
                             KVM_DEV_RISCV_AIA_IDS_MIN) ||
                            (kvm_riscv_aia_max_ids <= *nr))
                                return -EINVAL;
                        aia->nr_ids = *nr;
                } else
                        *nr = aia->nr_ids;
                break;
        case KVM_DEV_RISCV_AIA_CONFIG_SRCS:
                if (write) {
                        if ((*nr >= KVM_DEV_RISCV_AIA_SRCS_MAX) ||
                            (*nr >= kvm_riscv_aia_max_ids))
                                return -EINVAL;
                        aia->nr_sources = *nr;
                } else
                        *nr = aia->nr_sources;
                break;
        case KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS:
                if (write) {
                        if (*nr >= KVM_DEV_RISCV_AIA_GROUP_BITS_MAX)
                                return -EINVAL;
                        aia->nr_group_bits = *nr;
                } else
                        *nr = aia->nr_group_bits;
                break;
        case KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT:
                if (write) {
                        if ((*nr < KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN) ||
                            (*nr >= KVM_DEV_RISCV_AIA_GROUP_SHIFT_MAX))
                                return -EINVAL;
                        aia->nr_group_shift = *nr;
                } else
                        *nr = aia->nr_group_shift;
                break;
        case KVM_DEV_RISCV_AIA_CONFIG_HART_BITS:
                if (write) {
                        if (*nr >= KVM_DEV_RISCV_AIA_HART_BITS_MAX)
                                return -EINVAL;
                        aia->nr_hart_bits = *nr;
                } else
                        *nr = aia->nr_hart_bits;
                break;
        case KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS:
                if (write) {
                        if (*nr >= KVM_DEV_RISCV_AIA_GUEST_BITS_MAX)
                                return -EINVAL;
                        aia->nr_guest_bits = *nr;
                } else
                        *nr = aia->nr_guest_bits;
                break;
        default:
                return -ENXIO;
        }

        return 0;
}

static int aia_aplic_addr(struct kvm *kvm, u64 *addr, bool write)
{
        struct kvm_aia *aia = &kvm->arch.aia;

        if (write) {
                /* Writes can only be done before irqchip is initialized */
                if (kvm_riscv_aia_initialized(kvm))
                        return -EBUSY;

                if (*addr & (KVM_DEV_RISCV_APLIC_ALIGN - 1))
                        return -EINVAL;

                aia->aplic_addr = *addr;
        } else
                *addr = aia->aplic_addr;

        return 0;
}

static int aia_imsic_addr(struct kvm *kvm, u64 *addr,
                          unsigned long vcpu_idx, bool write)
{
        struct kvm_vcpu *vcpu;
        struct kvm_vcpu_aia *vcpu_aia;

        vcpu = kvm_get_vcpu(kvm, vcpu_idx);
        if (!vcpu)
                return -EINVAL;
        vcpu_aia = &vcpu->arch.aia_context;

        if (write) {
                /* Writes can only be done before irqchip is initialized */
                if (kvm_riscv_aia_initialized(kvm))
                        return -EBUSY;

                if (*addr & (KVM_DEV_RISCV_IMSIC_ALIGN - 1))
                        return -EINVAL;
        }

        mutex_lock(&vcpu->mutex);
        if (write)
                vcpu_aia->imsic_addr = *addr;
        else
                *addr = vcpu_aia->imsic_addr;
        mutex_unlock(&vcpu->mutex);

        return 0;
}

static gpa_t aia_imsic_ppn(struct kvm_aia *aia, gpa_t addr)
{
        u32 h, l;
        gpa_t mask = 0;

        h = aia->nr_hart_bits + aia->nr_guest_bits +
            IMSIC_MMIO_PAGE_SHIFT - 1;
        mask = GENMASK_ULL(h, 0);

        if (aia->nr_group_bits) {
                h = aia->nr_group_bits + aia->nr_group_shift - 1;
                l = aia->nr_group_shift;
                mask |= GENMASK_ULL(h, l);
        }

        return (addr & ~mask) >> IMSIC_MMIO_PAGE_SHIFT;
}

static u32 aia_imsic_hart_index(struct kvm_aia *aia, gpa_t addr)
{
        u32 hart = 0, group = 0;

        if (aia->nr_hart_bits)
                hart = (addr >> (aia->nr_guest_bits + IMSIC_MMIO_PAGE_SHIFT)) &
                       GENMASK_ULL(aia->nr_hart_bits - 1, 0);
        if (aia->nr_group_bits)
                group = (addr >> aia->nr_group_shift) &
                        GENMASK_ULL(aia->nr_group_bits - 1, 0);

        return (group << aia->nr_hart_bits) | hart;
}

static int aia_init(struct kvm *kvm)
{
        int ret, i;
        unsigned long idx;
        struct kvm_vcpu *vcpu;
        struct kvm_vcpu_aia *vaia;
        struct kvm_aia *aia = &kvm->arch.aia;
        gpa_t base_ppn = KVM_RISCV_AIA_UNDEF_ADDR;

        /* Irqchip can be initialized only once */
        if (kvm_riscv_aia_initialized(kvm))
                return -EBUSY;

        /* We might be in the middle of creating a VCPU? */
        if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
                return -EBUSY;

        /* Number of sources should be less than or equals number of IDs */
        if (aia->nr_ids < aia->nr_sources)
                return -EINVAL;

        /* APLIC base is required for non-zero number of sources */
        if (aia->nr_sources && aia->aplic_addr == KVM_RISCV_AIA_UNDEF_ADDR)
                return -EINVAL;

        /* Initialize APLIC */
        ret = kvm_riscv_aia_aplic_init(kvm);
        if (ret)
                return ret;

        /* Iterate over each VCPU */
        kvm_for_each_vcpu(idx, vcpu, kvm) {
                vaia = &vcpu->arch.aia_context;

                /* IMSIC base is required */
                if (vaia->imsic_addr == KVM_RISCV_AIA_UNDEF_ADDR) {
                        ret = -EINVAL;
                        goto fail_cleanup_imsics;
                }

                /* All IMSICs should have matching base PPN */
                if (base_ppn == KVM_RISCV_AIA_UNDEF_ADDR)
                        base_ppn = aia_imsic_ppn(aia, vaia->imsic_addr);
                if (base_ppn != aia_imsic_ppn(aia, vaia->imsic_addr)) {
                        ret = -EINVAL;
                        goto fail_cleanup_imsics;
                }

                /* Update HART index of the IMSIC based on IMSIC base */
                vaia->hart_index = aia_imsic_hart_index(aia,
                                                        vaia->imsic_addr);

                /* Initialize IMSIC for this VCPU */
                ret = kvm_riscv_vcpu_aia_imsic_init(vcpu);
                if (ret)
                        goto fail_cleanup_imsics;
        }

        /* Set the initialized flag */
        kvm->arch.aia.initialized = true;

        return 0;

fail_cleanup_imsics:
        for (i = idx - 1; i >= 0; i--) {
                vcpu = kvm_get_vcpu(kvm, i);
                if (!vcpu)
                        continue;
                kvm_riscv_vcpu_aia_imsic_cleanup(vcpu);
        }
        kvm_riscv_aia_aplic_cleanup(kvm);
        return ret;
}

static int aia_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
{
        u32 nr;
        u64 addr;
        int nr_vcpus, r = -ENXIO;
        unsigned long v, type = (unsigned long)attr->attr;
        void __user *uaddr = (void __user *)(long)attr->addr;

        switch (attr->group) {
        case KVM_DEV_RISCV_AIA_GRP_CONFIG:
                if (copy_from_user(&nr, uaddr, sizeof(nr)))
                        return -EFAULT;

                mutex_lock(&dev->kvm->lock);
                r = aia_config(dev->kvm, type, &nr, true);
                mutex_unlock(&dev->kvm->lock);

                break;

        case KVM_DEV_RISCV_AIA_GRP_ADDR:
                if (copy_from_user(&addr, uaddr, sizeof(addr)))
                        return -EFAULT;

                nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
                mutex_lock(&dev->kvm->lock);
                if (type == KVM_DEV_RISCV_AIA_ADDR_APLIC)
                        r = aia_aplic_addr(dev->kvm, &addr, true);
                else if (type < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus))
                        r = aia_imsic_addr(dev->kvm, &addr,
                            type - KVM_DEV_RISCV_AIA_ADDR_IMSIC(0), true);
                mutex_unlock(&dev->kvm->lock);

                break;

        case KVM_DEV_RISCV_AIA_GRP_CTRL:
                switch (type) {
                case KVM_DEV_RISCV_AIA_CTRL_INIT:
                        mutex_lock(&dev->kvm->lock);
                        r = aia_init(dev->kvm);
                        mutex_unlock(&dev->kvm->lock);
                        break;
                }

                break;
        case KVM_DEV_RISCV_AIA_GRP_APLIC:
                if (copy_from_user(&nr, uaddr, sizeof(nr)))
                        return -EFAULT;

                mutex_lock(&dev->kvm->lock);
                r = kvm_riscv_aia_aplic_set_attr(dev->kvm, type, nr);
                mutex_unlock(&dev->kvm->lock);

                break;
        case KVM_DEV_RISCV_AIA_GRP_IMSIC:
                if (copy_from_user(&v, uaddr, sizeof(v)))
                        return -EFAULT;

                mutex_lock(&dev->kvm->lock);
                r = kvm_riscv_aia_imsic_rw_attr(dev->kvm, type, true, &v);
                mutex_unlock(&dev->kvm->lock);

                break;
        }

        return r;
}

static int aia_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
{
        u32 nr;
        u64 addr;
        int nr_vcpus, r = -ENXIO;
        void __user *uaddr = (void __user *)(long)attr->addr;
        unsigned long v, type = (unsigned long)attr->attr;

        switch (attr->group) {
        case KVM_DEV_RISCV_AIA_GRP_CONFIG:
                if (copy_from_user(&nr, uaddr, sizeof(nr)))
                        return -EFAULT;

                mutex_lock(&dev->kvm->lock);
                r = aia_config(dev->kvm, type, &nr, false);
                mutex_unlock(&dev->kvm->lock);
                if (r)
                        return r;

                if (copy_to_user(uaddr, &nr, sizeof(nr)))
                        return -EFAULT;

                break;
        case KVM_DEV_RISCV_AIA_GRP_ADDR:
                if (copy_from_user(&addr, uaddr, sizeof(addr)))
                        return -EFAULT;

                nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
                mutex_lock(&dev->kvm->lock);
                if (type == KVM_DEV_RISCV_AIA_ADDR_APLIC)
                        r = aia_aplic_addr(dev->kvm, &addr, false);
                else if (type < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus))
                        r = aia_imsic_addr(dev->kvm, &addr,
                            type - KVM_DEV_RISCV_AIA_ADDR_IMSIC(0), false);
                mutex_unlock(&dev->kvm->lock);
                if (r)
                        return r;

                if (copy_to_user(uaddr, &addr, sizeof(addr)))
                        return -EFAULT;

                break;
        case KVM_DEV_RISCV_AIA_GRP_APLIC:
                if (copy_from_user(&nr, uaddr, sizeof(nr)))
                        return -EFAULT;

                mutex_lock(&dev->kvm->lock);
                r = kvm_riscv_aia_aplic_get_attr(dev->kvm, type, &nr);
                mutex_unlock(&dev->kvm->lock);
                if (r)
                        return r;

                if (copy_to_user(uaddr, &nr, sizeof(nr)))
                        return -EFAULT;

                break;
        case KVM_DEV_RISCV_AIA_GRP_IMSIC:
                if (copy_from_user(&v, uaddr, sizeof(v)))
                        return -EFAULT;

                mutex_lock(&dev->kvm->lock);
                r = kvm_riscv_aia_imsic_rw_attr(dev->kvm, type, false, &v);
                mutex_unlock(&dev->kvm->lock);
                if (r)
                        return r;

                if (copy_to_user(uaddr, &v, sizeof(v)))
                        return -EFAULT;

                break;
        }

        return r;
}

static int aia_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
{
        int nr_vcpus, r = -ENXIO;

        switch (attr->group) {
        case KVM_DEV_RISCV_AIA_GRP_CONFIG:
                switch (attr->attr) {
                case KVM_DEV_RISCV_AIA_CONFIG_MODE:
                case KVM_DEV_RISCV_AIA_CONFIG_IDS:
                case KVM_DEV_RISCV_AIA_CONFIG_SRCS:
                case KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS:
                case KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT:
                case KVM_DEV_RISCV_AIA_CONFIG_HART_BITS:
                case KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS:
                        return 0;
                }
                break;
        case KVM_DEV_RISCV_AIA_GRP_ADDR:
                nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
                if (attr->attr == KVM_DEV_RISCV_AIA_ADDR_APLIC)
                        return 0;
                else if (attr->attr < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus))
                        return 0;
                break;
        case KVM_DEV_RISCV_AIA_GRP_CTRL:
                switch (attr->attr) {
                case KVM_DEV_RISCV_AIA_CTRL_INIT:
                        return 0;
                }
                break;
        case KVM_DEV_RISCV_AIA_GRP_APLIC:
                mutex_lock(&dev->kvm->lock);
                r = kvm_riscv_aia_aplic_has_attr(dev->kvm, attr->attr);
                mutex_unlock(&dev->kvm->lock);
                break;
        case KVM_DEV_RISCV_AIA_GRP_IMSIC:
                mutex_lock(&dev->kvm->lock);
                r = kvm_riscv_aia_imsic_has_attr(dev->kvm, attr->attr);
                mutex_unlock(&dev->kvm->lock);
                break;
        }

        return r;
}

struct kvm_device_ops kvm_riscv_aia_device_ops = {
        .name = "kvm-riscv-aia",
        .create = aia_create,
        .destroy = aia_destroy,
        .set_attr = aia_set_attr,
        .get_attr = aia_get_attr,
        .has_attr = aia_has_attr,
};

int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu)
{
        /* Proceed only if AIA was initialized successfully */
        if (!kvm_riscv_aia_initialized(vcpu->kvm))
                return 1;

        /* Update the IMSIC HW state before entering guest mode */
        return kvm_riscv_vcpu_aia_imsic_update(vcpu);
}

void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu)
{
        struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;

        if (!kvm_riscv_aia_available())
                return;
        memset(csr, 0, sizeof(*csr));

        /* Proceed only if AIA was initialized successfully */
        if (!kvm_riscv_aia_initialized(vcpu->kvm))
                return;

        /* Reset the IMSIC context */
        kvm_riscv_vcpu_aia_imsic_reset(vcpu);
}

void kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu)
{
        struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context;

        if (!kvm_riscv_aia_available())
                return;

        /*
         * We don't do any memory allocations over here because these
         * will be done after AIA device is initialized by the user-space.
         *
         * Refer, aia_init() implementation for more details.
         */

        /* Initialize default values in AIA vcpu context */
        vaia->imsic_addr = KVM_RISCV_AIA_UNDEF_ADDR;
        vaia->hart_index = vcpu->vcpu_idx;
}

void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu)
{
        /* Proceed only if AIA was initialized successfully */
        if (!kvm_riscv_aia_initialized(vcpu->kvm))
                return;

        /* Cleanup IMSIC context */
        kvm_riscv_vcpu_aia_imsic_cleanup(vcpu);
}

int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index,
                                   u32 guest_index, u32 iid)
{
        unsigned long idx;
        struct kvm_vcpu *vcpu;

        /* Proceed only if AIA was initialized successfully */
        if (!kvm_riscv_aia_initialized(kvm))
                return -EBUSY;

        /* Inject MSI to matching VCPU */
        kvm_for_each_vcpu(idx, vcpu, kvm) {
                if (vcpu->arch.aia_context.hart_index == hart_index)
                        return kvm_riscv_vcpu_aia_imsic_inject(vcpu,
                                                               guest_index,
                                                               0, iid);
        }

        return 0;
}

int kvm_riscv_aia_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
{
        gpa_t tppn, ippn;
        unsigned long idx;
        struct kvm_vcpu *vcpu;
        u32 g, toff, iid = msi->data;
        struct kvm_aia *aia = &kvm->arch.aia;
        gpa_t target = (((gpa_t)msi->address_hi) << 32) | msi->address_lo;

        /* Proceed only if AIA was initialized successfully */
        if (!kvm_riscv_aia_initialized(kvm))
                return -EBUSY;

        /* Convert target address to target PPN */
        tppn = target >> IMSIC_MMIO_PAGE_SHIFT;

        /* Extract and clear Guest ID from target PPN */
        g = tppn & (BIT(aia->nr_guest_bits) - 1);
        tppn &= ~((gpa_t)(BIT(aia->nr_guest_bits) - 1));

        /* Inject MSI to matching VCPU */
        kvm_for_each_vcpu(idx, vcpu, kvm) {
                ippn = vcpu->arch.aia_context.imsic_addr >>
                                        IMSIC_MMIO_PAGE_SHIFT;
                if (ippn == tppn) {
                        toff = target & (IMSIC_MMIO_PAGE_SZ - 1);
                        return kvm_riscv_vcpu_aia_imsic_inject(vcpu, g,
                                                               toff, iid);
                }
        }

        return 0;
}

int kvm_riscv_aia_inject_irq(struct kvm *kvm, unsigned int irq, bool level)
{
        /* Proceed only if AIA was initialized successfully */
        if (!kvm_riscv_aia_initialized(kvm))
                return -EBUSY;

        /* Inject interrupt level change in APLIC */
        return kvm_riscv_aia_aplic_inject(kvm, irq, level);
}

void kvm_riscv_aia_init_vm(struct kvm *kvm)
{
        struct kvm_aia *aia = &kvm->arch.aia;

        if (!kvm_riscv_aia_available())
                return;

        /*
         * We don't do any memory allocations over here because these
         * will be done after AIA device is initialized by the user-space.
         *
         * Refer, aia_init() implementation for more details.
         */

        /* Initialize default values in AIA global context */
        aia->mode = (kvm_riscv_aia_nr_hgei) ?
                KVM_DEV_RISCV_AIA_MODE_AUTO : KVM_DEV_RISCV_AIA_MODE_EMUL;
        aia->nr_ids = kvm_riscv_aia_max_ids - 1;
        aia->nr_sources = 0;
        aia->nr_group_bits = 0;
        aia->nr_group_shift = KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN;
        aia->nr_hart_bits = 0;
        aia->nr_guest_bits = 0;
        aia->aplic_addr = KVM_RISCV_AIA_UNDEF_ADDR;
}

void kvm_riscv_aia_destroy_vm(struct kvm *kvm)
{
        /* Proceed only if AIA was initialized successfully */
        if (!kvm_riscv_aia_initialized(kvm))
                return;

        /* Cleanup APLIC context */
        kvm_riscv_aia_aplic_cleanup(kvm);
}
Linux
