/* $NetBSD: gicv3_its.c,v 1.42 2026/05/24 22:00:52 jmcneill Exp $ */

/*-
 * Copyright (c) 2018 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jared McNeill <jmcneill@invisible.ca>.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#define _INTR_PRIVATE

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: gicv3_its.c,v 1.42 2026/05/24 22:00:52 jmcneill Exp $");

#include <sys/param.h>
#include <sys/kmem.h>
#include <sys/bus.h>
#include <sys/cpu.h>
#include <sys/bitops.h>

#include <uvm/uvm.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>

#include <machine/cpufunc.h>

#include <arm/pic/picvar.h>
#include <arm/cortex/gicv3_its.h>

#ifdef ITS_DEBUG
#define DPRINTF(x)      printf x
#else
#define DPRINTF(x)      __nothing
#endif

/*
 * ITS translation table sizes
 */
#define GITS_COMMANDS_SIZE      0x1000
#define GITS_COMMANDS_ALIGN     0x10000

#define GITS_ITT_ALIGN          0x100

#define GITS_INDIRECT_ENTRY_SIZE        8

/*
 * IIDR values used for errata
 */
#define GITS_IIDR_PID_CAVIUM_THUNDERX   0xa1
#define GITS_IIDR_IMP_CAVIUM            0x34c
#define GITS_IIDR_CAVIUM_ERRATA_MASK    (GITS_IIDR_Implementor|GITS_IIDR_ProductID|GITS_IIDR_Variant)
#define GITS_IIDR_CAVIUM_ERRATA_VALUE                                                   \
                (__SHIFTIN(GITS_IIDR_IMP_CAVIUM, GITS_IIDR_Implementor) |               \
                 __SHIFTIN(GITS_IIDR_PID_CAVIUM_THUNDERX, GITS_IIDR_ProductID) |        \
                 __SHIFTIN(0, GITS_IIDR_Variant))

static const char * gits_cache_type[] = {
        [GITS_Cache_DEVICE_nGnRnE]      = "Device-nGnRnE",
        [GITS_Cache_NORMAL_NC]          = "Non-cacheable",
        [GITS_Cache_NORMAL_RA_WT]       = "Cacheable RA WT",
        [GITS_Cache_NORMAL_RA_WB]       = "Cacheable RA WB",
        [GITS_Cache_NORMAL_WA_WT]       = "Cacheable WA WT",
        [GITS_Cache_NORMAL_WA_WB]       = "Cacheable WA WB",
        [GITS_Cache_NORMAL_RA_WA_WT]    = "Cacheable RA WA WT",
        [GITS_Cache_NORMAL_RA_WA_WB]    = "Cacheable RA WA WB",
};

static const char * gits_share_type[] = {
        [GITS_Shareability_NS]          = "Non-shareable",
        [GITS_Shareability_IS]          = "Inner shareable",
        [GITS_Shareability_OS]          = "Outer shareable",
        [3]                             = "(Reserved)",
};

static inline uint32_t
gits_read_4(struct gicv3_its *its, bus_size_t reg)
{
        return bus_space_read_4(its->its_bst, its->its_bsh, reg);
}

static inline void
gits_write_4(struct gicv3_its *its, bus_size_t reg, uint32_t val)
{
        bus_space_write_4(its->its_bst, its->its_bsh, reg, val);
}

static inline uint64_t
gits_read_8(struct gicv3_its *its, bus_size_t reg)
{
        return bus_space_read_8(its->its_bst, its->its_bsh, reg);
}

static inline void
gits_write_8(struct gicv3_its *its, bus_size_t reg, uint64_t val)
{
        bus_space_write_8(its->its_bst, its->its_bsh, reg, val);
}

static int
gits_command(struct gicv3_its *its, const struct gicv3_its_command *cmd)
{
        uint64_t cwriter, creadr;
        u_int woff;

        creadr = gits_read_8(its, GITS_CREADR);
        if (ISSET(creadr, GITS_CREADR_Stalled)) {
                DPRINTF(("ITS: stalled! GITS_CREADR = 0x%lx\n", creadr));
                return EIO;
        }

        cwriter = gits_read_8(its, GITS_CWRITER);
        woff = cwriter & GITS_CWRITER_Offset;

        uint64_t *dw = (uint64_t *)(its->its_cmd.base + woff);
        for (int i = 0; i < __arraycount(cmd->dw); i++) {
                dw[i] = htole64(cmd->dw[i]);
                DPRINTF(("ITS:     dw[%u] = 0x%016lx\n", i, cmd->dw[i]));
        }

        if (its->its_cmd_flush) {
                cpu_dcache_wb_range((vaddr_t)dw, sizeof(cmd->dw));
        }
        dsb(sy);

        woff += sizeof(cmd->dw);
        if (woff == its->its_cmd.len)
                woff = 0;

        gits_write_8(its, GITS_CWRITER, woff);

        return 0;
}

static int
gits_command_mapc(struct gicv3_its *its, uint16_t icid, uint64_t rdbase, bool v)
{
        struct gicv3_its_command cmd;

        KASSERT((rdbase & 0xffff) == 0);

        /*
         * Map a collection table entry (ICID) to the target redistributor (RDbase).
         */
        memset(&cmd, 0, sizeof(cmd));
        cmd.dw[0] = GITS_CMD_MAPC;
        cmd.dw[2] = icid;
        if (v) {
                cmd.dw[2] |= rdbase;
                cmd.dw[2] |= __BIT(63);
        }

        DPRINTF(("ITS #%u: MAPC icid 0x%x rdbase 0x%lx valid %u\n",
            its->its_id, icid, rdbase, v));

        return gits_command(its, &cmd);
}

static int
gits_command_mapd(struct gicv3_its *its, uint32_t deviceid, uint64_t itt_addr, u_int size, bool v)
{
        struct gicv3_its_command cmd;

        KASSERT((itt_addr & 0xff) == 0);

        /*
         * Map a device table entry (DeviceID) to its associated ITT (ITT_addr).
         */
        memset(&cmd, 0, sizeof(cmd));
        cmd.dw[0] = GITS_CMD_MAPD | ((uint64_t)deviceid << 32);
        if (v) {
                cmd.dw[1] = size;
                cmd.dw[2] = itt_addr | __BIT(63);
        }

        DPRINTF(("ITS #%u: MAPD deviceid 0x%x itt_addr 0x%lx size %u valid %u\n",
            its->its_id, deviceid, itt_addr, size, v));

        return gits_command(its, &cmd);
}

static int
gits_command_mapti(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid, uint32_t pintid, uint16_t icid)
{
        struct gicv3_its_command cmd;

        /*
         * Map the event defined by EventID and DeviceID to its associated ITE, defined by ICID and pINTID
         * in the ITT associated with DeviceID.
         */
        memset(&cmd, 0, sizeof(cmd));
        cmd.dw[0] = GITS_CMD_MAPTI | ((uint64_t)deviceid << 32);
        cmd.dw[1] = eventid | ((uint64_t)pintid << 32);
        cmd.dw[2] = icid;

        DPRINTF(("ITS #%u: MAPTI deviceid 0x%x eventid 0x%x pintid 0x%x icid 0x%x\n",
            its->its_id, deviceid, eventid, pintid, icid));

        return gits_command(its, &cmd);
}

static int
gits_command_movi(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid, uint16_t icid)
{
        struct gicv3_its_command cmd;

        /*
         * Update the ICID field in the ITT entry for the event defined by DeviceID and
         * EventID.
         */
        memset(&cmd, 0, sizeof(cmd));
        cmd.dw[0] = GITS_CMD_MOVI | ((uint64_t)deviceid << 32);
        cmd.dw[1] = eventid;
        cmd.dw[2] = icid;

        DPRINTF(("ITS #%u: MOVI deviceid 0x%x eventid 0x%x icid 0x%x\n",
            its->its_id, deviceid, eventid, icid));

        return gits_command(its, &cmd);
}

static int
gits_command_inv(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid)
{
        struct gicv3_its_command cmd;

        /*
         * Ensure any caching in the redistributors associated with the specified
         * EventID is consistent with the LPI configuration tables.
         */
        memset(&cmd, 0, sizeof(cmd));
        cmd.dw[0] = GITS_CMD_INV | ((uint64_t)deviceid << 32);
        cmd.dw[1] = eventid;

        DPRINTF(("ITS #%u: INV deviceid 0x%x eventid 0x%x\n",
            its->its_id, deviceid, eventid));

        return gits_command(its, &cmd);
}

static int
gits_command_invall(struct gicv3_its *its, uint16_t icid)
{
        struct gicv3_its_command cmd;

        /*
         * Ensure any caching associated with this ICID is consistent with LPI
         * configuration tables for all redistributors.
         */
        memset(&cmd, 0, sizeof(cmd));
        cmd.dw[0] = GITS_CMD_INVALL;
        cmd.dw[2] = icid;

        DPRINTF(("ITS #%u: INVALL icid 0x%x\n", its->its_id, icid));

        return gits_command(its, &cmd);
}

static int
gits_command_sync(struct gicv3_its *its, uint64_t rdbase)
{
        struct gicv3_its_command cmd;

        KASSERT((rdbase & 0xffff) == 0);

        /*
         * Ensure all outstanding ITS operations associated with physical interrupts
         * for the specified redistributor (RDbase) are globally observed before
         * further ITS commands are executed.
         */
        memset(&cmd, 0, sizeof(cmd));
        cmd.dw[0] = GITS_CMD_SYNC;
        cmd.dw[2] = rdbase;

        DPRINTF(("ITS #%u: SYNC rdbase 0x%lx\n", its->its_id, rdbase));

        return gits_command(its, &cmd);
}

#if 0
static int
gits_command_int(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid)
{
        struct gicv3_its_command cmd;

        /*
         * Translate the deviceid and eventid into an icid and pintid through
         * the device table and ITT. Mark the pintid as pending
         * on the redistributor.
         * If the interrupt is not configured the command queue stalls.
         */
        memset(&cmd, 0, sizeof(cmd));
        cmd.dw[0] = GITS_CMD_INT | ((uint64_t)deviceid << 32);
        cmd.dw[1] = eventid;

        DPRINTF(("ITS #%u: INT deviceid 0x%x eventid 0x%x\n",
            its->its_id, deviceid, eventid));

        return gits_command(its, &cmd);
}
#endif

static int
gits_wait(struct gicv3_its *its)
{
        u_int woff, roff;
        int retry = 100000;

        /*
         * The ITS command queue is empty when CWRITER and CREADR specify the
         * same base address offset value.
         */
        for (retry = 1000; retry > 0; retry--) {
                woff = gits_read_8(its, GITS_CWRITER) & GITS_CWRITER_Offset;
                roff = gits_read_8(its, GITS_CREADR) & GITS_CREADR_Offset;
                if (woff == roff)
                        break;
                delay(100);
        }
        if (retry == 0) {
                device_printf(its->its_gic->sc_dev,
                    "ITS command queue timeout! CREADR=0x%lx CWRITER=0x%lx\n",
                    gits_read_8(its, GITS_CREADR), gits_read_8(its, GITS_CWRITER));
                return ETIMEDOUT;
        }

        return 0;
}

static int
gicv3_its_msi_alloc_lpi(struct gicv3_its *its,
    const struct pci_attach_args *pa)
{
        struct pci_attach_args *new_pa;
        vmem_addr_t n;

        KASSERT(its->its_gic->sc_lpi_pool != NULL);

        if (vmem_alloc(its->its_gic->sc_lpi_pool, 1, VM_INSTANTFIT|VM_SLEEP, &n) != 0)
                return -1;

        KASSERT(its->its_pa[n] == NULL);

        new_pa = kmem_alloc(sizeof(*new_pa), KM_SLEEP);
        memcpy(new_pa, pa, sizeof(*new_pa));
        its->its_pa[n] = new_pa;
        return n + its->its_pic->pic_irqbase;
}

static void
gicv3_its_msi_free_lpi(struct gicv3_its *its, int lpi)
{
        struct pci_attach_args *pa;

        KASSERT(its->its_gic->sc_lpi_pool != NULL);
        KASSERT(lpi >= its->its_pic->pic_irqbase);

        pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
        its->its_pa[lpi - its->its_pic->pic_irqbase] = NULL;
        kmem_free(pa, sizeof(*pa));

        vmem_free(its->its_gic->sc_lpi_pool, lpi - its->its_pic->pic_irqbase, 1);
}

static uint32_t
gicv3_its_alloc_eventid(struct gicv3_its_device *dev, unsigned count)
{
        uint32_t eventid = ITS_INVALID_EVENTID;
        unsigned index, bit, n;
        unsigned nalloc = 0;

        KASSERT(count > 0);

        for (index = 0; index < __arraycount(dev->dev_eventid); index++) {
                for (bit = 0; bit < 32; bit++) {
                        if ((dev->dev_eventid[index] & __BIT(bit)) == 0) {
                                if (nalloc++ == 0) {
                                        eventid = index * 32 + bit;
                                }
                                if (nalloc == count) {
                                        break;  
                                }
                        } else if (nalloc != 0) {
                                nalloc = 0;
                                eventid = ITS_INVALID_EVENTID;
                        }
                }
                if (nalloc == count) {
                        break;
                }
        }
        if (nalloc != count) {
                eventid = ITS_INVALID_EVENTID;
        } else {
                for (n = 0; n < count; n++) {
                        index = (eventid + n) / 32;
                        bit = (eventid + n) % 32;

                        KASSERT((dev->dev_eventid[index] & __BIT(bit)) == 0);
                        dev->dev_eventid[index] |= __BIT(bit);
                }
        }

        if (__predict_false(eventid == ITS_INVALID_EVENTID)) {
                panic("failed to allocate %u event ID(s) for device ID 0x%x; "
                      "increase ITS_MAX_EVENTS", count, dev->dev_id);
        }

        return eventid;
}

static void
gicv3_its_free_eventid(struct gicv3_its_device *dev, uint32_t eventid)
{
        unsigned index, bit;

        KASSERT(eventid < ITS_MAX_EVENTS);

        index = eventid / 32;
        bit = eventid % 32;

        KASSERT((dev->dev_eventid[index] & __BIT(bit)) != 0);
        dev->dev_eventid[index] &= ~__BIT(bit);
}

static uint32_t
gicv3_its_devid(pci_chipset_tag_t pc, pcitag_t tag)
{
        uint32_t devid;
        int b, d, f;

        pci_decompose_tag(pc, tag, &b, &d, &f);

        devid = (b << 8) | (d << 3) | f;

        return pci_get_devid(pc, devid);
}

static struct gicv3_its_device *
gicv3_its_device_map(struct gicv3_its *its, uint32_t devid, u_int count)
{
        struct gicv3_its_device *dev;
        struct gicv3_its_table *itstab = &its->its_tab_device;
        u_int vectors;
        int error;

        vectors = MAX(2, count);
        while (!powerof2(vectors))
                vectors++;

        const uint64_t typer = gits_read_8(its, GITS_TYPER);
        const u_int itt_entry_size = __SHIFTOUT(typer, GITS_TYPER_ITT_entry_size) + 1;

        /*
         * Compute the size of the ITT. To simplify things, allocate enough
         * space for ITS_MAX_EVENTS even if it's more than supported by
         * hardware. Then use the smaller of the two to determin the
         * effective supported ID bits for this device.
         */
        const u_int itt_size = roundup(ITS_MAX_EVENTS * itt_entry_size, GITS_ITT_ALIGN);
        const u_int id_bits = uimin(31 - __builtin_clz(ITS_MAX_EVENTS) - 1,
                                    __SHIFTOUT(typer, GITS_TYPER_ID_bits));

        LIST_FOREACH(dev, &its->its_devices, dev_list)
                if (dev->dev_id == devid) {
                        return itt_size <= dev->dev_size ? dev : NULL;
                }

        if (itstab->tab_indirect) {
                uint64_t *l1_tab = itstab->tab_l1;
                const u_int index = devid / itstab->tab_l2_num_ids;

                if ((l1_tab[index] & GITS_BASER_Valid) == 0) {
                        /* Need to allocate the L2 table. */
                        struct gicv3_its_page_table *pt;

                        pt = kmem_alloc(sizeof(*pt), KM_SLEEP);
                        pt->pt_index = index;
                        gicv3_dma_alloc(its->its_gic, &pt->pt_dma, itstab->tab_l2_entry_size,
                            itstab->tab_page_size);
                        LIST_INSERT_HEAD(&itstab->tab_pt, pt, pt_list);

                        if (!itstab->tab_shareable) {
                                cpu_dcache_wb_range((vaddr_t)pt->pt_dma.base,
                                    itstab->tab_l2_entry_size);
                        }
                        l1_tab[index] = pt->pt_dma.segs[0].ds_addr | GITS_BASER_Valid;
                        if (!itstab->tab_shareable) {
                                cpu_dcache_wb_range((vaddr_t)&l1_tab[index],
                                    sizeof(l1_tab[index]));
                        }
                        dsb(sy);

                        DPRINTF(("ITS: Allocated L2 entry at index %u\n", index));
                }
        }

        dev = kmem_alloc(sizeof(*dev), KM_SLEEP);
        dev->dev_id = devid;
        dev->dev_size = itt_size;
        memset(dev->dev_eventid, 0, sizeof(dev->dev_eventid));
        gicv3_dma_alloc(its->its_gic, &dev->dev_itt, itt_size, GITS_ITT_ALIGN);
        LIST_INSERT_HEAD(&its->its_devices, dev, dev_list);

        if (its->its_cmd_flush) {
                cpu_dcache_wb_range((vaddr_t)dev->dev_itt.base, itt_size);
        }
        dsb(sy);

        /*
         * Map the device to the ITT
         */
        mutex_enter(its->its_lock);
        error = gits_command_mapd(its, devid, dev->dev_itt.segs[0].ds_addr, id_bits, true);
        if (error == 0) {
                error = gits_wait(its);
        }
        mutex_exit(its->its_lock);

        return error == 0 ? dev : NULL;
}

static void
gicv3_its_msi_enable(struct gicv3_its *its, int lpi, uint32_t data, int count)
{
        const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
        pci_chipset_tag_t pc = pa->pa_pc;
        pcitag_t tag = pa->pa_tag;
        pcireg_t ctl;
        int off;

        if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL))
                panic("gicv3_its_msi_enable: device is not MSI-capable");

        ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
        ctl &= ~PCI_MSI_CTL_MME_MASK;
        ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK);
        pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);

        const uint64_t addr = its->its_base + GITS_TRANSLATER;
        ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
        if (ctl & PCI_MSI_CTL_64BIT_ADDR) {
                pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO,
                    addr & 0xffffffff);
                pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI,
                    (addr >> 32) & 0xffffffff);
                pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data);
        } else {
                KASSERT((addr >> 32) == 0);
                pci_conf_write(pc, tag, off + PCI_MSI_MADDR,
                    addr & 0xffffffff);
                pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data);
        }
        ctl |= PCI_MSI_CTL_MSI_ENABLE;
        pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
}

static void
gicv3_its_msi_disable(struct gicv3_its *its, int lpi)
{
        const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
        pci_chipset_tag_t pc = pa->pa_pc;
        pcitag_t tag = pa->pa_tag;
        pcireg_t ctl;
        int off;

        if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL))
                panic("gicv3_its_msi_enable: device is not MSI-capable");

        ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
        ctl &= ~PCI_MSI_CTL_MSI_ENABLE;
        pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
}

static void
gicv3_its_msix_enable(struct gicv3_its *its, int lpi, int msix_vec,
    uint32_t data, bus_space_tag_t bst, bus_space_handle_t bsh)
{
        const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
        pci_chipset_tag_t pc = pa->pa_pc;
        pcitag_t tag = pa->pa_tag;
        pcireg_t ctl;
        uint32_t val;
        int off;

        if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL))
                panic("gicv3_its_msix_enable: device is not MSI-X-capable");

        const uint64_t addr = its->its_base + GITS_TRANSLATER;
        const uint64_t entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;
        bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO,
            (uint32_t)addr);
        bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI,
            (uint32_t)(addr >> 32));
        bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_DATA,
            data);
        val = bus_space_read_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL);
        val &= ~PCI_MSIX_VECTCTL_MASK;
        bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, val);

        ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
        ctl |= PCI_MSIX_CTL_ENABLE;
        pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
}

static void
gicv3_its_msix_disable(struct gicv3_its *its, int lpi)
{
        const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
        pci_chipset_tag_t pc = pa->pa_pc;
        pcitag_t tag = pa->pa_tag;
        pcireg_t ctl;
        int off;

        if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL))
                panic("gicv3_its_msix_disable: device is not MSI-X-capable");

        ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
        ctl &= ~PCI_MSIX_CTL_ENABLE;
        pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
}

static pci_intr_handle_t *
gicv3_its_msi_alloc(struct arm_pci_msi *msi, int *count,
    const struct pci_attach_args *pa, bool exact)
{
        struct gicv3_its * const its = msi->msi_priv;
        struct gicv3_its_device *dev;
        struct cpu_info * const ci = cpu_lookup(0);
        pci_intr_handle_t *vectors;
        int n, off, error;
        uint32_t eventid;

        if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSI, &off, NULL))
                return NULL;

        const uint64_t typer = gits_read_8(its, GITS_TYPER);
        const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1;
        if (*count == 0 || *count > (1 << id_bits))
                return NULL;

        const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag);

        dev = gicv3_its_device_map(its, devid, *count);
        if (dev == NULL)
                return NULL;

        vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP);
        mutex_enter(its->its_lock);
        eventid = gicv3_its_alloc_eventid(dev, *count);
        KASSERT(eventid != ITS_INVALID_EVENTID);
        for (n = 0; n < *count; n++) {
                const int lpi = gicv3_its_msi_alloc_lpi(its, pa);
                KASSERT(lpi >= 0);
                vectors[n] = ARM_PCI_INTR_MSI |
                    __SHIFTIN(lpi, ARM_PCI_INTR_IRQ) |
                    __SHIFTIN(eventid + n, ARM_PCI_INTR_MSI_VEC) |
                    __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME);

                if (n == 0)
                        gicv3_its_msi_enable(its, lpi, eventid + n, *count);

                /*
                 * Record devid, eventid, dev, and target PE
                 */
                its->its_devid[lpi - its->its_pic->pic_irqbase] = devid;
                its->its_eventid[lpi - its->its_pic->pic_irqbase] = eventid + n;
                its->its_lpitodev[lpi - its->its_pic->pic_irqbase] = dev;
                its->its_targets[lpi - its->its_pic->pic_irqbase] = ci;

                /*
                 * Map event
                 */
                gits_command_mapti(its, devid, eventid + n, lpi, cpu_index(ci));
                gits_command_sync(its, its->its_rdbase[cpu_index(ci)]);
        }
        error = gits_wait(its);
        mutex_exit(its->its_lock);

        if (error != 0) {
                kmem_free(vectors, sizeof(*vectors) * *count);
                vectors = NULL;
        }

        return vectors;
}

static pci_intr_handle_t *
gicv3_its_msix_alloc(struct arm_pci_msi *msi, u_int *table_indexes, int *count,
    const struct pci_attach_args *pa, bool exact)
{
        struct gicv3_its * const its = msi->msi_priv;
        struct gicv3_its_device *dev;
        struct cpu_info *ci = cpu_lookup(0);
        pci_intr_handle_t *vectors;
        bus_space_tag_t bst;
        bus_space_handle_t bsh;
        bus_size_t bsz;
        uint32_t table_offset, table_size;
        uint32_t eventid;
        int n, off, bar, error;
        pcireg_t tbl;

        if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &off, NULL))
                return NULL;

        const uint64_t typer = gits_read_8(its, GITS_TYPER);
        const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1;
        if (*count == 0 || *count > (1 << id_bits))
                return NULL;

        tbl = pci_conf_read(pa->pa_pc, pa->pa_tag, off + PCI_MSIX_TBLOFFSET);
        bar = PCI_BAR0 + (4 * (tbl & PCI_MSIX_TBLBIR_MASK));
        table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK;
        table_size = pci_msix_count(pa->pa_pc, pa->pa_tag) * PCI_MSIX_TABLE_ENTRY_SIZE;
        if (table_size == 0)
                return NULL;

        error = pci_mapreg_submap(pa, bar, pci_mapreg_type(pa->pa_pc, pa->pa_tag, bar),
            BUS_SPACE_MAP_LINEAR, roundup(table_size, PAGE_SIZE), table_offset,
            &bst, &bsh, NULL, &bsz);
        if (error)
                return NULL;

        const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag);

        dev = gicv3_its_device_map(its, devid, *count);
        if (dev == NULL) {
                bus_space_unmap(bst, bsh, bsz);
                return NULL;
        }

        vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP);
        mutex_enter(its->its_lock);
        eventid = gicv3_its_alloc_eventid(dev, *count);
        KASSERT(eventid != ITS_INVALID_EVENTID);
        for (n = 0; n < *count; n++) {
                const int lpi = gicv3_its_msi_alloc_lpi(its, pa);
                KASSERT(lpi >= 0);
                const int msix_vec = table_indexes ? table_indexes[n] : n;
                vectors[msix_vec] = ARM_PCI_INTR_MSIX |
                    __SHIFTIN(lpi, ARM_PCI_INTR_IRQ) |
                    __SHIFTIN(eventid + n, ARM_PCI_INTR_MSI_VEC) |
                    __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME);

                gicv3_its_msix_enable(its, lpi, msix_vec, eventid + n, bst, bsh);

                /*
                 * Record devid, eventid, dev, and target PE
                 */
                its->its_devid[lpi - its->its_pic->pic_irqbase] = devid;
                its->its_eventid[lpi - its->its_pic->pic_irqbase] = eventid + n;
                its->its_lpitodev[lpi - its->its_pic->pic_irqbase] = dev;
                its->its_targets[lpi - its->its_pic->pic_irqbase] = ci;

                /*
                 * Map event
                 */
                gits_command_mapti(its, devid, eventid + n, lpi, cpu_index(ci));
                gits_command_sync(its, its->its_rdbase[cpu_index(ci)]);
        }
        gits_wait(its);
        mutex_exit(its->its_lock);

        bus_space_unmap(bst, bsh, bsz);

        return vectors;
}

static void *
gicv3_its_msi_intr_establish(struct arm_pci_msi *msi,
    pci_intr_handle_t ih, int ipl, int (*func)(void *), void *arg, const char *xname)
{
        struct gicv3_its * const its = msi->msi_priv;
        void *intrh;

        const int lpi = __SHIFTOUT(ih, ARM_PCI_INTR_IRQ);
        const uint32_t eventid = __SHIFTOUT(ih, ARM_PCI_INTR_MSI_VEC);
        const int mpsafe = (ih & ARM_PCI_INTR_MPSAFE) ? IST_MPSAFE : 0;

        intrh = pic_establish_intr(its->its_pic, lpi - its->its_pic->pic_irqbase, ipl,
            IST_EDGE | mpsafe, func, arg, xname);
        if (intrh == NULL)
                return NULL;

        /* Invalidate LPI configuration tables */
        KASSERT(its->its_pa[lpi - its->its_pic->pic_irqbase] != NULL);
        const uint32_t devid = its->its_devid[lpi - its->its_pic->pic_irqbase];
        gits_command_inv(its, devid, eventid);

        return intrh;
}

static void
gicv3_its_msi_intr_release(struct arm_pci_msi *msi, pci_intr_handle_t *pih,
    int count)
{
        struct gicv3_its * const its = msi->msi_priv;
        int n;

        for (n = 0; n < count; n++) {
                const int lpi = __SHIFTOUT(pih[n], ARM_PCI_INTR_IRQ);
                const uint32_t eventid = __SHIFTOUT(pih[n], ARM_PCI_INTR_MSI_VEC);
                KASSERT(lpi >= its->its_pic->pic_irqbase);
                struct gicv3_its_device *dev =
                    its->its_lpitodev[lpi - its->its_pic->pic_irqbase];
                KASSERT(dev != NULL);
                if (pih[n] & ARM_PCI_INTR_MSIX)
                        gicv3_its_msix_disable(its, lpi);
                if (pih[n] & ARM_PCI_INTR_MSI)
                        gicv3_its_msi_disable(its, lpi);
                gicv3_its_free_eventid(dev, eventid);
                gicv3_its_msi_free_lpi(its, lpi);
                its->its_targets[lpi - its->its_pic->pic_irqbase] = NULL;
                its->its_devid[lpi - its->its_pic->pic_irqbase] = 0;
                its->its_eventid[lpi - its->its_pic->pic_irqbase] = ITS_INVALID_EVENTID;
                its->its_lpitodev[lpi - its->its_pic->pic_irqbase] = NULL;
                struct intrsource * const is =
                    its->its_pic->pic_sources[lpi - its->its_pic->pic_irqbase];
                if (is != NULL)
                        pic_disestablish_source(is);
        }
}

static void
gicv3_its_command_init(struct gicv3_softc *sc, struct gicv3_its *its)
{
        uint64_t cbaser, tmp;

        gicv3_dma_alloc(sc, &its->its_cmd, GITS_COMMANDS_SIZE, GITS_COMMANDS_ALIGN);
        if (its->its_cmd_flush) {
                cpu_dcache_wb_range((vaddr_t)its->its_cmd.base, GITS_COMMANDS_SIZE);
        }
        dsb(sy);

        KASSERT((gits_read_4(its, GITS_CTLR) & GITS_CTLR_Enabled) == 0);
        KASSERT((gits_read_4(its, GITS_CTLR) & GITS_CTLR_Quiescent) != 0);

        cbaser = its->its_cmd.segs[0].ds_addr;
        cbaser |= __SHIFTIN((its->its_cmd.len / 4096) - 1, GITS_CBASER_Size);
        cbaser |= GITS_CBASER_Valid;

        cbaser |= __SHIFTIN(GITS_Cache_NORMAL_WA_WB, GITS_CBASER_InnerCache);
        cbaser |= __SHIFTIN(GITS_Shareability_IS, GITS_CBASER_Shareability);
        gits_write_8(its, GITS_CBASER, cbaser);

        tmp = gits_read_8(its, GITS_CBASER);
        if (__SHIFTOUT(tmp, GITS_CBASER_Shareability) != GITS_Shareability_IS) {
                if (__SHIFTOUT(tmp, GITS_CBASER_InnerCache) == GITS_Shareability_NS) {
                        cbaser &= ~GITS_CBASER_InnerCache;
                        cbaser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_CBASER_InnerCache);
                        cbaser &= ~GITS_CBASER_Shareability;
                        cbaser |= __SHIFTIN(GITS_Shareability_NS, GITS_CBASER_Shareability);
                        gits_write_8(its, GITS_CBASER, cbaser);
                }

                its->its_cmd_flush = true;
        }
        aprint_normal_dev(sc->sc_dev, "ITS command table @ %#lx/%#lx, %s, %s\n",
            its->its_cmd.segs[0].ds_addr, its->its_cmd.len,
            gits_cache_type[__SHIFTOUT(cbaser, GITS_BASER_InnerCache)],
            gits_share_type[__SHIFTOUT(cbaser, GITS_BASER_Shareability)]);

        gits_write_8(its, GITS_CWRITER, 0);
}

static void
gicv3_its_table_params(struct gicv3_softc *sc, struct gicv3_its *its,
    u_int *devbits, u_int *innercache, u_int *share)
{

        const uint64_t typer = gits_read_8(its, GITS_TYPER);
        const uint32_t iidr = gits_read_4(its, GITS_IIDR);

        /* Default values */
        *devbits = __SHIFTOUT(typer, GITS_TYPER_Devbits) + 1;
        *innercache = GITS_Cache_NORMAL_WA_WB;
        *share = GITS_Shareability_IS;

        /* Cavium ThunderX errata */
        if ((iidr & GITS_IIDR_CAVIUM_ERRATA_MASK) == GITS_IIDR_CAVIUM_ERRATA_VALUE) {
                *devbits = 20;          /* 8Mb */
                *innercache = GITS_Cache_DEVICE_nGnRnE;
                aprint_normal_dev(sc->sc_dev, "Cavium ThunderX errata detected\n");
        }
}

static bool
gicv3_its_table_probe_indirect(struct gicv3_its *its, int tab)
{
        uint64_t baser;

        baser = gits_read_8(its, GITS_BASERn(tab));
        baser |= GITS_BASER_Indirect;
        gits_write_8(its, GITS_BASERn(tab), baser);

        baser = gits_read_8(its, GITS_BASERn(tab));

        return (baser & GITS_BASER_Indirect) != 0;
}

static void
gicv3_its_table_init(struct gicv3_softc *sc, struct gicv3_its *its)
{
        u_int page_size, table_align;
        u_int devbits, innercache, share;
        const char *table_type;
        uint64_t baser;
        int tab;

        gicv3_its_table_params(sc, its, &devbits, &innercache, &share);

        DPRINTF(("ITS: devbits = %u\n", devbits));

        for (tab = 0; tab < 8; tab++) {
                struct gicv3_its_table *itstab;
                bool indirect = false;
                uint64_t l1_entry_size, l2_entry_size;
                uint64_t l1_num_ids, l2_num_ids;
                uint64_t table_size;

                baser = gits_read_8(its, GITS_BASERn(tab));

                l1_entry_size = __SHIFTOUT(baser, GITS_BASER_Entry_Size) + 1;
                l2_entry_size = 0;
                l2_num_ids = 0;

                switch (__SHIFTOUT(baser, GITS_BASER_Page_Size)) {
                case GITS_Page_Size_64KB:
                        page_size = 65536;
                        break;
                case GITS_Page_Size_16KB:
                        page_size = 16384;
                        break;
                case GITS_Page_Size_4KB:
                default:
                        page_size = 4096;
                }
                table_align = page_size;

                switch (__SHIFTOUT(baser, GITS_BASER_Type)) {
                case GITS_Type_Devices:
                        /*
                         * Table size scales with the width of the DeviceID.
                         */
                        l1_num_ids = 1ULL << devbits;
                        DPRINTF(("ITS: l1_num_ids = %lu\n", l1_num_ids));
                        indirect =
                            gicv3_its_table_probe_indirect(its, tab);
                        if (indirect) {
                                DPRINTF(("ITS: indirect\n"));
                                l2_entry_size = l1_entry_size;
                                l2_num_ids = page_size / l2_entry_size;
                                l1_num_ids = l1_num_ids / l2_num_ids;
                                l1_entry_size = GITS_INDIRECT_ENTRY_SIZE;
                        }
                        table_size = roundup2(l1_entry_size * l1_num_ids, page_size);
                        if (howmany(table_size, page_size) > GITS_BASER_Size + 1) {
                                DPRINTF(("ITS: clamp table size 0x%lx -> ", table_size));
                                table_size = (GITS_BASER_Size + 1) * page_size;
                                DPRINTF(("0x%lx\n", table_size));
                        }
                        table_type = "Devices";

                        DPRINTF(("ITS: table_size is 0x%lx\n", table_size));

                        itstab = &its->its_tab_device;
                        itstab->tab_page_size = page_size;
                        itstab->tab_l1_entry_size = l1_entry_size;
                        itstab->tab_l1_num_ids = l1_num_ids;
                        itstab->tab_l2_entry_size = l2_entry_size;
                        itstab->tab_l2_num_ids = l2_num_ids;
                        itstab->tab_indirect = indirect;
                        LIST_INIT(&itstab->tab_pt);
                        break;
                case GITS_Type_InterruptCollections:
                        /*
                         * Allocate space for one interrupt collection per CPU.
                         */
                        table_size = roundup(l1_entry_size * ncpu, page_size);
                        table_type = "Collections";
                        break;
                default:
                        table_size = 0;
                        break;
                }

                if (table_size == 0)
                        continue;

                gicv3_dma_alloc(sc, &its->its_tab[tab], table_size, table_align);
                if (its->its_cmd_flush) {
                        cpu_dcache_wb_range((vaddr_t)its->its_tab[tab].base, table_size);
                }
                dsb(sy);

                baser &= ~GITS_BASER_Size;
                baser |= __SHIFTIN(howmany(table_size, page_size) - 1, GITS_BASER_Size);
                baser &= ~GITS_BASER_Physical_Address;
                baser |= its->its_tab[tab].segs[0].ds_addr;
                baser &= ~GITS_BASER_InnerCache;
                baser |= __SHIFTIN(innercache, GITS_BASER_InnerCache);
                baser &= ~GITS_BASER_Shareability;
                baser |= __SHIFTIN(share, GITS_BASER_Shareability);
                baser |= GITS_BASER_Valid;
                if (indirect) {
                        baser |= GITS_BASER_Indirect;
                } else {
                        baser &= ~GITS_BASER_Indirect;
                }

                gits_write_8(its, GITS_BASERn(tab), baser);

                baser = gits_read_8(its, GITS_BASERn(tab));
                if (__SHIFTOUT(baser, GITS_BASER_Shareability) == GITS_Shareability_NS) {
                        baser &= ~GITS_BASER_InnerCache;
                        baser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_BASER_InnerCache);

                        gits_write_8(its, GITS_BASERn(tab), baser);
                }

                baser = gits_read_8(its, GITS_BASERn(tab));
                aprint_normal_dev(sc->sc_dev, "ITS [#%d] %s table @ %#lx/%#lx, %s, %s%s\n",
                    tab, table_type, its->its_tab[tab].segs[0].ds_addr, table_size,
                    gits_cache_type[__SHIFTOUT(baser, GITS_BASER_InnerCache)],
                    gits_share_type[__SHIFTOUT(baser, GITS_BASER_Shareability)],
                    indirect ? ", indirect" : "");

                if (__SHIFTOUT(baser, GITS_BASER_Type) == GITS_Type_Devices) {
                        its->its_tab_device.tab_l1 = its->its_tab[tab].base;
                        its->its_tab_device.tab_shareable =
                            __SHIFTOUT(baser, GITS_BASER_Shareability) != GITS_Shareability_NS;
                }

        }
}

static void
gicv3_its_enable(struct gicv3_softc *sc, struct gicv3_its *its)
{
        uint32_t ctlr;

        ctlr = gits_read_4(its, GITS_CTLR);
        ctlr |= GITS_CTLR_Enabled;
        gits_write_4(its, GITS_CTLR, ctlr);
}

static void
gicv3_its_cpu_init(void *priv, struct cpu_info *ci)
{
        struct gicv3_its * const its = priv;
        struct gicv3_softc * const sc = its->its_gic;
        uint64_t rdbase;
        size_t irq;

        const uint64_t typer = bus_space_read_8(sc->sc_bst, its->its_bsh, GITS_TYPER);
        if (typer & GITS_TYPER_PTA) {
                void *va = bus_space_vaddr(sc->sc_bst, sc->sc_bsh_r[ci->ci_gic_redist]);
                rdbase = vtophys((vaddr_t)va);
        } else {
                rdbase = (uint64_t)sc->sc_processor_id[cpu_index(ci)] << 16;
        }
        its->its_rdbase[cpu_index(ci)] = rdbase;

        /*
         * Map collection ID of this CPU's index to this CPU's redistributor.
         */
        mutex_enter(its->its_lock);
        gits_command_mapc(its, cpu_index(ci), rdbase, true);
        gits_command_invall(its, cpu_index(ci));
        gits_wait(its);

        /*
         * Update routing for LPIs targetting this CPU
         */
        for (irq = 0; irq < its->its_pic->pic_maxsources; irq++) {
                if (its->its_targets[irq] != ci)
                        continue;
                KASSERT(its->its_pa[irq] != NULL);

                const uint32_t devid = its->its_devid[irq];
                const uint32_t eventid = its->its_eventid[irq];
                KASSERT(eventid != ITS_INVALID_EVENTID);
                gits_command_movi(its, devid, eventid, cpu_index(ci));
                gits_command_sync(its, its->its_rdbase[cpu_index(ci)]);
        }
        gits_wait(its);
        mutex_exit(its->its_lock);

        its->its_cpuonline[cpu_index(ci)] = true;
}

static void
gicv3_its_get_affinity(void *priv, size_t irq, kcpuset_t *affinity)
{
        struct gicv3_its * const its = priv;
        struct cpu_info *ci;

        ci = its->its_targets[irq];
        if (ci)
                kcpuset_set(affinity, cpu_index(ci));
}

static int
gicv3_its_set_affinity(void *priv, size_t irq, const kcpuset_t *affinity)
{
        struct gicv3_its * const its = priv;
        const struct pci_attach_args *pa;
        struct cpu_info *ci;

        const int set = kcpuset_countset(affinity);
        if (set != 1)
                return EINVAL;

        pa = its->its_pa[irq];
        if (pa == NULL)
                return EPASSTHROUGH;

        ci = cpu_lookup(kcpuset_ffs(affinity) - 1);
        its->its_targets[irq] = ci;

        if (its->its_cpuonline[cpu_index(ci)] == true) {
                const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag);
                const uint32_t eventid = its->its_eventid[irq];
                KASSERT(eventid != ITS_INVALID_EVENTID);
                mutex_enter(its->its_lock);
                gits_command_movi(its, devid, eventid, cpu_index(ci));
                gits_command_sync(its, its->its_rdbase[cpu_index(ci)]);
                mutex_exit(its->its_lock);
        }

        return 0;
}

int
gicv3_its_init(struct gicv3_softc *sc, bus_space_handle_t bsh,
    uint64_t its_base, uint32_t its_id)
{
        struct gicv3_its *its;
        struct arm_pci_msi *msi;

        const uint64_t typer = bus_space_read_8(sc->sc_bst, bsh, GITS_TYPER);
        if ((typer & GITS_TYPER_Physical) == 0)
                return ENXIO;

        its = kmem_zalloc(sizeof(*its), KM_SLEEP);
        its->its_id = its_id;
        its->its_bst = sc->sc_bst;
        its->its_bsh = bsh;
        its->its_dmat = sc->sc_dmat;
        its->its_base = its_base;
        its->its_pic = &sc->sc_lpi;
        snprintf(its->its_pic->pic_name, sizeof(its->its_pic->pic_name), "gicv3-its");
        KASSERT(its->its_pic->pic_maxsources > 0);
        its->its_pa = kmem_zalloc(sizeof(struct pci_attach_args *) * its->its_pic->pic_maxsources, KM_SLEEP);
        its->its_targets = kmem_zalloc(sizeof(struct cpu_info *) * its->its_pic->pic_maxsources, KM_SLEEP);
        its->its_devid = kmem_zalloc(sizeof(uint32_t) * its->its_pic->pic_maxsources, KM_SLEEP);
        its->its_eventid = kmem_alloc(sizeof(uint32_t) * its->its_pic->pic_maxsources, KM_SLEEP);
        memset(its->its_eventid, 0xff, sizeof(uint32_t) * its->its_pic->pic_maxsources);
        its->its_lpitodev = kmem_alloc(sizeof(struct gicv3_its_device *) * its->its_pic->pic_maxsources, KM_SLEEP);
        its->its_gic = sc;
        its->its_rdbase = kmem_zalloc(sizeof(*its->its_rdbase) * ncpu, KM_SLEEP);
        its->its_cpuonline = kmem_zalloc(sizeof(*its->its_cpuonline) * ncpu, KM_SLEEP);
        its->its_cb.cpu_init = gicv3_its_cpu_init;
        its->its_cb.get_affinity = gicv3_its_get_affinity;
        its->its_cb.set_affinity = gicv3_its_set_affinity;
        its->its_cb.priv = its;
        LIST_INIT(&its->its_devices);
        LIST_INSERT_HEAD(&sc->sc_lpi_callbacks, &its->its_cb, list);
        its->its_lock = mutex_obj_alloc(MUTEX_SPIN, IPL_NONE);

        gicv3_its_command_init(sc, its);
        gicv3_its_table_init(sc, its);

        gicv3_its_enable(sc, its);

        gicv3_its_cpu_init(its, curcpu());

        msi = &its->its_msi;
        msi->msi_id = its_id;
        msi->msi_dev = sc->sc_dev;
        msi->msi_priv = its;
        msi->msi_alloc = gicv3_its_msi_alloc;
        msi->msix_alloc = gicv3_its_msix_alloc;
        msi->msi_intr_establish = gicv3_its_msi_intr_establish;
        msi->msi_intr_release = gicv3_its_msi_intr_release;

        return arm_pci_msi_add(msi);
}