/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * PX Interrupt Block implementation
 */

#include <sys/types.h>
#include <sys/kmem.h>
#include <sys/async.h>
#include <sys/systm.h>          /* panicstr */
#include <sys/spl.h>
#include <sys/sunddi.h>
#include <sys/machsystm.h>      /* intr_dist_add */
#include <sys/ddi_impldefs.h>
#include <sys/cpuvar.h>
#include <sys/time.h>
#include "px_obj.h"

/*LINTLIBRARY*/

static void px_ib_intr_redist(void *arg, int32_t weight_max, int32_t weight);
static void px_ib_cpu_ticks_to_ih_nsec(px_ib_t *ib_p, px_ih_t *ih_p,
    uint32_t cpu_id);
static uint_t px_ib_intr_reset(void *arg);
static void px_fill_in_intr_devs(pcitool_intr_dev_t *dev, char *driver_name,
    char *path_name, int instance);

extern uint64_t xc_tick_jump_limit;

int
px_ib_attach(px_t *px_p)
{
        dev_info_t      *dip = px_p->px_dip;
        px_ib_t         *ib_p;
        sysino_t        sysino;
        px_fault_t      *fault_p = &px_p->px_fault;

        DBG(DBG_IB, dip, "px_ib_attach\n");

        if (px_lib_intr_devino_to_sysino(px_p->px_dip,
            px_p->px_inos[PX_INTR_PEC], &sysino) != DDI_SUCCESS)
                return (DDI_FAILURE);

        /*
         * Allocate interrupt block state structure and link it to
         * the px state structure.
         */
        ib_p = kmem_zalloc(sizeof (px_ib_t), KM_SLEEP);
        px_p->px_ib_p = ib_p;
        ib_p->ib_px_p = px_p;
        ib_p->ib_ino_lst = (px_ino_t *)NULL;

        mutex_init(&ib_p->ib_intr_lock, NULL, MUTEX_DRIVER, NULL);
        mutex_init(&ib_p->ib_ino_lst_mutex, NULL, MUTEX_DRIVER, NULL);

        bus_func_register(BF_TYPE_RESINTR, px_ib_intr_reset, ib_p);

        intr_dist_add_weighted(px_ib_intr_redist, ib_p);

        /*
         * Initialize PEC fault data structure
         */
        fault_p->px_fh_dip = dip;
        fault_p->px_fh_sysino = sysino;
        fault_p->px_err_func = px_err_dmc_pec_intr;
        fault_p->px_intr_ino = px_p->px_inos[PX_INTR_PEC];

        return (DDI_SUCCESS);
}

void
px_ib_detach(px_t *px_p)
{
        px_ib_t         *ib_p = px_p->px_ib_p;
        dev_info_t      *dip = px_p->px_dip;

        DBG(DBG_IB, dip, "px_ib_detach\n");

        bus_func_unregister(BF_TYPE_RESINTR, px_ib_intr_reset, ib_p);
        intr_dist_rem_weighted(px_ib_intr_redist, ib_p);

        mutex_destroy(&ib_p->ib_ino_lst_mutex);
        mutex_destroy(&ib_p->ib_intr_lock);

        px_ib_free_ino_all(ib_p);

        px_p->px_ib_p = NULL;
        kmem_free(ib_p, sizeof (px_ib_t));
}

void
px_ib_intr_enable(px_t *px_p, cpuid_t cpu_id, devino_t ino)
{
        px_ib_t         *ib_p = px_p->px_ib_p;
        sysino_t        sysino;

        /*
         * Determine the cpu for the interrupt
         */
        mutex_enter(&ib_p->ib_intr_lock);

        DBG(DBG_IB, px_p->px_dip,
            "px_ib_intr_enable: ino=%x cpu_id=%x\n", ino, cpu_id);

        if (px_lib_intr_devino_to_sysino(px_p->px_dip, ino,
            &sysino) != DDI_SUCCESS) {
                DBG(DBG_IB, px_p->px_dip,
                    "px_ib_intr_enable: px_intr_devino_to_sysino() failed\n");

                mutex_exit(&ib_p->ib_intr_lock);
                return;
        }

        PX_INTR_ENABLE(px_p->px_dip, sysino, cpu_id);
        px_lib_intr_setstate(px_p->px_dip, sysino, INTR_IDLE_STATE);

        mutex_exit(&ib_p->ib_intr_lock);
}

/*ARGSUSED*/
void
px_ib_intr_disable(px_ib_t *ib_p, devino_t ino, int wait)
{
        sysino_t        sysino;

        mutex_enter(&ib_p->ib_intr_lock);

        DBG(DBG_IB, ib_p->ib_px_p->px_dip, "px_ib_intr_disable: ino=%x\n", ino);

        /* Disable the interrupt */
        if (px_lib_intr_devino_to_sysino(ib_p->ib_px_p->px_dip, ino,
            &sysino) != DDI_SUCCESS) {
                DBG(DBG_IB, ib_p->ib_px_p->px_dip,
                    "px_ib_intr_disable: px_intr_devino_to_sysino() failed\n");

                mutex_exit(&ib_p->ib_intr_lock);
                return;
        }

        PX_INTR_DISABLE(ib_p->ib_px_p->px_dip, sysino);

        mutex_exit(&ib_p->ib_intr_lock);
}

int
px_ib_intr_pend(dev_info_t *dip, sysino_t sysino)
{
        int             ret = DDI_SUCCESS;
        hrtime_t        start_time, prev, curr, interval, jump;
        hrtime_t        intr_timeout;
        intr_state_t    intr_state;

        /* Disable the interrupt */
        PX_INTR_DISABLE(dip, sysino);

        intr_timeout = px_intrpend_timeout;
        jump = TICK_TO_NSEC(xc_tick_jump_limit);

        /* Busy wait on pending interrupt */
        for (curr = start_time = gethrtime(); !panicstr &&
            ((ret = px_lib_intr_getstate(dip, sysino,
            &intr_state)) == DDI_SUCCESS) &&
            (intr_state == INTR_DELIVERED_STATE); /* */) {
                /*
                 * If we have a really large jump in hrtime, it is most
                 * probably because we entered the debugger (or OBP,
                 * in general). So, we adjust the timeout accordingly
                 * to prevent declaring an interrupt timeout. The
                 * master-interrupt mechanism in OBP should deliver
                 * the interrupts properly.
                 */
                prev = curr;
                curr = gethrtime();
                interval = curr - prev;
                if (interval > jump)
                        intr_timeout += interval;
                if (curr - start_time > intr_timeout) {
                        ret = DDI_FAILURE;
                        break;
                }
        }
        return (ret);
}

void
px_ib_intr_dist_en(dev_info_t *dip, cpuid_t cpu_id, devino_t ino,
    boolean_t wait_flag)
{
        uint32_t        old_cpu_id;
        sysino_t        sysino;
        intr_valid_state_t      enabled = 0;

        DBG(DBG_IB, dip, "px_ib_intr_dist_en: ino=0x%x\n", ino);

        if (px_lib_intr_devino_to_sysino(dip, ino, &sysino) != DDI_SUCCESS) {
                DBG(DBG_IB, dip, "px_ib_intr_dist_en: "
                    "px_intr_devino_to_sysino() failed, ino 0x%x\n", ino);
                return;
        }

        /* Skip enabling disabled interrupts */
        if (px_lib_intr_getvalid(dip, sysino, &enabled) != DDI_SUCCESS) {
                DBG(DBG_IB, dip, "px_ib_intr_dist_en: px_intr_getvalid() "
                    "failed, sysino 0x%x\n", sysino);
                return;
        }
        if (!enabled)
                return;

        /* Done if redistributed onto the same cpuid */
        if (px_lib_intr_gettarget(dip, sysino, &old_cpu_id) != DDI_SUCCESS) {
                DBG(DBG_IB, dip, "px_ib_intr_dist_en: "
                    "px_intr_gettarget() failed\n");
                return;
        }
        if (cpu_id == old_cpu_id)
                return;

        /* Wait on pending interrupts */
        if (wait_flag != 0 && px_ib_intr_pend(dip, sysino) != DDI_SUCCESS) {
                cmn_err(CE_WARN,
                    "%s%d: px_ib_intr_dist_en: sysino 0x%lx(ino 0x%x) "
                    "from cpu id 0x%x to 0x%x timeout",
                    ddi_driver_name(dip), ddi_get_instance(dip),
                    sysino, ino, old_cpu_id, cpu_id);

                DBG(DBG_IB, dip, "px_ib_intr_dist_en: failed, "
                    "ino 0x%x sysino 0x%x\n", ino, sysino);
        }

        PX_INTR_ENABLE(dip, sysino, cpu_id);
}

static void
px_ib_cpu_ticks_to_ih_nsec(px_ib_t *ib_p, px_ih_t *ih_p, uint32_t cpu_id)
{
        extern kmutex_t pxintr_ks_template_lock;
        hrtime_t ticks;

        /*
         * Because we are updating two fields in ih_t we must lock
         * pxintr_ks_template_lock to prevent someone from reading the
         * kstats after we set ih_ticks to 0 and before we increment
         * ih_nsec to compensate.
         *
         * We must also protect against the interrupt arriving and incrementing
         * ih_ticks between the time we read it and when we reset it to 0.
         * To do this we use atomic_swap.
         */

        ASSERT(MUTEX_HELD(&ib_p->ib_ino_lst_mutex));

        mutex_enter(&pxintr_ks_template_lock);
        ticks = atomic_swap_64(&ih_p->ih_ticks, 0);
        ih_p->ih_nsec += (uint64_t)tick2ns(ticks, cpu_id);
        mutex_exit(&pxintr_ks_template_lock);
}


/*
 * Redistribute interrupts of the specified weight. The first call has a weight
 * of weight_max, which can be used to trigger initialization for
 * redistribution. The inos with weight [weight_max, inf.) should be processed
 * on the "weight == weight_max" call.  This first call is followed by calls
 * of decreasing weights, inos of that weight should be processed.  The final
 * call specifies a weight of zero, this can be used to trigger processing of
 * stragglers.
 */
static void
px_ib_intr_redist(void *arg, int32_t weight_max, int32_t weight)
{
        px_ib_t         *ib_p = (px_ib_t *)arg;
        px_t            *px_p = ib_p->ib_px_p;
        dev_info_t      *dip = px_p->px_dip;
        px_ino_t        *ino_p;
        px_ino_pil_t    *ipil_p;
        px_ih_t         *ih_lst;
        int32_t         dweight = 0;
        int             i;

        /* Redistribute internal interrupts */
        if (weight == 0) {
                mutex_enter(&ib_p->ib_intr_lock);
                px_ib_intr_dist_en(dip, intr_dist_cpuid(),
                    px_p->px_inos[PX_INTR_PEC], B_FALSE);
                mutex_exit(&ib_p->ib_intr_lock);

                px_hp_intr_redist(px_p);
        }

        /* Redistribute device interrupts */
        mutex_enter(&ib_p->ib_ino_lst_mutex);
        px_msiq_redist(px_p);

        for (ino_p = ib_p->ib_ino_lst; ino_p; ino_p = ino_p->ino_next_p) {
                /*
                 * Recomputes the sum of interrupt weights of devices that
                 * share the same ino upon first call marked by
                 * (weight == weight_max).
                 */
                if (weight == weight_max) {
                        ino_p->ino_intr_weight = 0;

                        for (ipil_p = ino_p->ino_ipil_p; ipil_p;
                            ipil_p = ipil_p->ipil_next_p) {
                                for (i = 0, ih_lst = ipil_p->ipil_ih_head;
                                    i < ipil_p->ipil_ih_size; i++,
                                    ih_lst = ih_lst->ih_next) {
                                        dweight = i_ddi_get_intr_weight(
                                            ih_lst->ih_dip);
                                        if (dweight > 0)
                                                ino_p->ino_intr_weight +=
                                                    dweight;
                                }
                        }
                }

                /*
                 * As part of redistributing weighted interrupts over cpus,
                 * nexus redistributes device interrupts and updates
                 * cpu weight. The purpose is for the most light weighted
                 * cpu to take the next interrupt and gain weight, therefore
                 * attention demanding device gains more cpu attention by
                 * making itself heavy.
                 */
                if ((weight == ino_p->ino_intr_weight) ||
                    ((weight >= weight_max) &&
                    (ino_p->ino_intr_weight >= weight_max))) {
                        uint32_t orig_cpuid = ino_p->ino_cpuid;

                        if (cpu[orig_cpuid] == NULL)
                                orig_cpuid = CPU->cpu_id;

                        DBG(DBG_IB, dip, "px_ib_intr_redist: sysino 0x%llx "
                            "current cpuid 0x%x current default cpuid 0x%x\n",
                            ino_p->ino_sysino, ino_p->ino_cpuid,
                            ino_p->ino_default_cpuid);

                        /* select target cpuid and mark ino established */
                        if (ino_p->ino_default_cpuid == -1)
                                ino_p->ino_cpuid = ino_p->ino_default_cpuid =
                                    intr_dist_cpuid();
                        else if ((ino_p->ino_cpuid !=
                            ino_p->ino_default_cpuid) &&
                            cpu[ino_p->ino_default_cpuid] &&
                            cpu_intr_on(cpu[ino_p->ino_default_cpuid]))
                                ino_p->ino_cpuid = ino_p->ino_default_cpuid;
                        else if (!cpu_intr_on(cpu[ino_p->ino_cpuid]))
                                ino_p->ino_cpuid = intr_dist_cpuid();

                        DBG(DBG_IB, dip, "px_ib_intr_redist: sysino 0x%llx "
                            "new cpuid 0x%x new default cpuid 0x%x\n",
                            ino_p->ino_sysino, ino_p->ino_cpuid,
                            ino_p->ino_default_cpuid);

                        /* Add device weight to targeted cpu. */
                        for (ipil_p = ino_p->ino_ipil_p; ipil_p;
                            ipil_p = ipil_p->ipil_next_p) {
                                for (i = 0, ih_lst = ipil_p->ipil_ih_head;
                                    i < ipil_p->ipil_ih_size; i++,
                                    ih_lst = ih_lst->ih_next) {

                                        dweight = i_ddi_get_intr_weight(
                                            ih_lst->ih_dip);
                                        intr_dist_cpuid_add_device_weight(
                                            ino_p->ino_cpuid, ih_lst->ih_dip,
                                            dweight);

                                        /*
                                         * Different cpus may have different
                                         * clock speeds. to account for this,
                                         * whenever an interrupt is moved to a
                                         * new CPU, we convert the accumulated
                                         * ticks into nsec, based upon the clock
                                         * rate of the prior CPU.
                                         *
                                         * It is possible that the prior CPU no
                                         * longer exists. In this case, fall
                                         * back to using this CPU's clock rate.
                                         *
                                         * Note that the value in ih_ticks has
                                         * already been corrected for any power
                                         * savings mode which might have been
                                         * in effect.
                                         */
                                        px_ib_cpu_ticks_to_ih_nsec(ib_p, ih_lst,
                                            orig_cpuid);
                                }
                        }

                        /* enable interrupt on new targeted cpu */
                        px_ib_intr_dist_en(dip, ino_p->ino_cpuid,
                            ino_p->ino_ino, B_TRUE);
                }
        }
        mutex_exit(&ib_p->ib_ino_lst_mutex);
}

/*
 * Reset interrupts to IDLE.  This function is called during
 * panic handling after redistributing interrupts; it's needed to
 * support dumping to network devices after 'sync' from OBP.
 *
 * N.B.  This routine runs in a context where all other threads
 * are permanently suspended.
 */
static uint_t
px_ib_intr_reset(void *arg)
{
        px_ib_t         *ib_p = (px_ib_t *)arg;

        DBG(DBG_IB, ib_p->ib_px_p->px_dip, "px_ib_intr_reset\n");

        if (px_lib_intr_reset(ib_p->ib_px_p->px_dip) != DDI_SUCCESS)
                return (BF_FATAL);

        return (BF_NONE);
}

/*
 * Locate px_ino_t structure on ib_p->ib_ino_lst according to ino#
 * returns NULL if not found.
 */
px_ino_t *
px_ib_locate_ino(px_ib_t *ib_p, devino_t ino_num)
{
        px_ino_t        *ino_p = ib_p->ib_ino_lst;

        ASSERT(MUTEX_HELD(&ib_p->ib_ino_lst_mutex));

        for (; ino_p && ino_p->ino_ino != ino_num; ino_p = ino_p->ino_next_p)
                ;

        return (ino_p);
}

px_ino_t *
px_ib_alloc_ino(px_ib_t *ib_p, devino_t ino_num)
{
        sysino_t        sysino;
        px_ino_t        *ino_p;

        if (px_lib_intr_devino_to_sysino(ib_p->ib_px_p->px_dip,
            ino_num, &sysino) != DDI_SUCCESS)
                return (NULL);

        ino_p = kmem_zalloc(sizeof (px_ino_t), KM_SLEEP);

        ino_p->ino_next_p = ib_p->ib_ino_lst;
        ib_p->ib_ino_lst = ino_p;

        ino_p->ino_ino = ino_num;
        ino_p->ino_sysino = sysino;
        ino_p->ino_ib_p = ib_p;
        ino_p->ino_unclaimed_intrs = 0;
        ino_p->ino_lopil = 0;
        ino_p->ino_cpuid = ino_p->ino_default_cpuid = (cpuid_t)-1;

        return (ino_p);
}

px_ino_pil_t *
px_ib_new_ino_pil(px_ib_t *ib_p, devino_t ino_num, uint_t pil, px_ih_t *ih_p)
{
        px_ino_pil_t    *ipil_p = kmem_zalloc(sizeof (px_ino_pil_t), KM_SLEEP);
        px_ino_t        *ino_p;

        if ((ino_p = px_ib_locate_ino(ib_p, ino_num)) == NULL)
                ino_p = px_ib_alloc_ino(ib_p, ino_num);

        ASSERT(ino_p != NULL);

        ih_p->ih_next = ih_p;
        ipil_p->ipil_pil = pil;
        ipil_p->ipil_ih_head = ih_p;
        ipil_p->ipil_ih_tail = ih_p;
        ipil_p->ipil_ih_start = ih_p;
        ipil_p->ipil_ih_size = 1;
        ipil_p->ipil_ino_p = ino_p;

        ipil_p->ipil_next_p = ino_p->ino_ipil_p;
        ino_p->ino_ipil_p = ipil_p;
        ino_p->ino_ipil_size++;

        if ((ino_p->ino_lopil == 0) || (ino_p->ino_lopil > pil))
                ino_p->ino_lopil = pil;

        return (ipil_p);
}

void
px_ib_delete_ino_pil(px_ib_t *ib_p, px_ino_pil_t *ipil_p)
{
        px_ino_t        *ino_p = ipil_p->ipil_ino_p;
        ushort_t        pil = ipil_p->ipil_pil;
        px_ino_pil_t    *prev, *next;

        ASSERT(MUTEX_HELD(&ib_p->ib_ino_lst_mutex));

        if (ino_p->ino_ipil_p == ipil_p)
                ino_p->ino_ipil_p = ipil_p->ipil_next_p;
        else {
                for (prev = next = ino_p->ino_ipil_p; next != ipil_p;
                    prev = next, next = next->ipil_next_p)
                        ;

                if (prev)
                        prev->ipil_next_p = ipil_p->ipil_next_p;
        }

        kmem_free(ipil_p, sizeof (px_ino_pil_t));

        if ((--ino_p->ino_ipil_size) && (ino_p->ino_lopil == pil)) {
                for (next = ino_p->ino_ipil_p, pil = next->ipil_pil;
                    next; next = next->ipil_next_p) {

                        if (pil > next->ipil_pil)
                                pil = next->ipil_pil;
                }

                /*
                 * Value stored in pil should be the lowest pil.
                 */
                ino_p->ino_lopil = pil;
        }

        if (ino_p->ino_ipil_size)
                return;

        ino_p->ino_lopil = 0;

        if (ino_p->ino_msiq_p)
                return;

        if (ib_p->ib_ino_lst == ino_p)
                ib_p->ib_ino_lst = ino_p->ino_next_p;
        else {
                px_ino_t        *list = ib_p->ib_ino_lst;

                for (; list->ino_next_p != ino_p; list = list->ino_next_p)
                        ;
                list->ino_next_p = ino_p->ino_next_p;
        }
}

/*
 * Free all ino when we are detaching.
 */
void
px_ib_free_ino_all(px_ib_t *ib_p)
{
        px_ino_t        *ino_p = ib_p->ib_ino_lst;
        px_ino_t        *next = NULL;

        while (ino_p) {
                next = ino_p->ino_next_p;
                kmem_free(ino_p, sizeof (px_ino_t));
                ino_p = next;
        }
}

/*
 * Locate px_ino_pil_t structure on ino_p->ino_ipil_p according to ino#
 * returns NULL if not found.
 */
px_ino_pil_t *
px_ib_ino_locate_ipil(px_ino_t *ino_p, uint_t pil)
{
        px_ino_pil_t    *ipil_p = ino_p->ino_ipil_p;

        for (; ipil_p && ipil_p->ipil_pil != pil; ipil_p = ipil_p->ipil_next_p)
                ;

        return (ipil_p);
}

int
px_ib_ino_add_intr(px_t *px_p, px_ino_pil_t *ipil_p, px_ih_t *ih_p)
{
        px_ino_t        *ino_p = ipil_p->ipil_ino_p;
        px_ib_t         *ib_p = ino_p->ino_ib_p;
        devino_t        ino = ino_p->ino_ino;
        sysino_t        sysino = ino_p->ino_sysino;
        dev_info_t      *dip = px_p->px_dip;
        cpuid_t         curr_cpu;
        int             ret = DDI_SUCCESS;

        ASSERT(MUTEX_HELD(&ib_p->ib_ino_lst_mutex));
        ASSERT(ib_p == px_p->px_ib_p);

        DBG(DBG_IB, dip, "px_ib_ino_add_intr ino=%x\n", ino_p->ino_ino);

        /* Disable the interrupt */
        if ((ret = px_lib_intr_gettarget(dip, sysino,
            &curr_cpu)) != DDI_SUCCESS) {
                DBG(DBG_IB, dip,
                    "px_ib_ino_add_intr px_intr_gettarget() failed\n");

                return (ret);
        }

        /* Wait on pending interrupt */
        if ((ret = px_ib_intr_pend(dip, sysino)) != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s%d: px_ib_ino_add_intr: pending "
                    "sysino 0x%lx(ino 0x%x) timeout",
                    ddi_driver_name(dip), ddi_get_instance(dip),
                    sysino, ino);
        }

        /*
         * If the interrupt was previously blocked (left in pending state)
         * because of jabber we need to clear the pending state in case the
         * jabber has gone away.
         */
        if (ino_p->ino_unclaimed_intrs > px_unclaimed_intr_max) {
                cmn_err(CE_WARN,
                    "%s%d: px_ib_ino_add_intr: ino 0x%x has been unblocked",
                    ddi_driver_name(dip), ddi_get_instance(dip), ino);

                ino_p->ino_unclaimed_intrs = 0;
                ret = px_lib_intr_setstate(dip, sysino, INTR_IDLE_STATE);
        }

        if (ret != DDI_SUCCESS) {
                DBG(DBG_IB, dip, "px_ib_ino_add_intr: failed, "
                    "ino 0x%x sysino 0x%x\n", ino, sysino);

                return (ret);
        }

        /* Link up px_ih_t */
        ih_p->ih_next = ipil_p->ipil_ih_head;
        ipil_p->ipil_ih_tail->ih_next = ih_p;
        ipil_p->ipil_ih_tail = ih_p;

        ipil_p->ipil_ih_start = ipil_p->ipil_ih_head;
        ipil_p->ipil_ih_size++;

        /* Re-enable interrupt */
        PX_INTR_ENABLE(dip, sysino, curr_cpu);

        return (ret);
}

/*
 * Removes px_ih_t from the ino's link list.
 * uses hardware mutex to lock out interrupt threads.
 * Side effects: interrupt belongs to that ino is turned off on return.
 * if we are sharing PX slot with other inos, the caller needs
 * to turn it back on.
 */
int
px_ib_ino_rem_intr(px_t *px_p, px_ino_pil_t *ipil_p, px_ih_t *ih_p)
{
        px_ino_t        *ino_p = ipil_p->ipil_ino_p;
        devino_t        ino = ino_p->ino_ino;
        sysino_t        sysino = ino_p->ino_sysino;
        dev_info_t      *dip = px_p->px_dip;
        px_ih_t         *ih_lst = ipil_p->ipil_ih_head;
        int             i, ret = DDI_SUCCESS;

        ASSERT(MUTEX_HELD(&ino_p->ino_ib_p->ib_ino_lst_mutex));

        DBG(DBG_IB, px_p->px_dip, "px_ib_ino_rem_intr ino=%x\n",
            ino_p->ino_ino);

        /* Wait on pending interrupt */
        if ((ret = px_ib_intr_pend(dip, sysino)) != DDI_SUCCESS) {
                cmn_err(CE_WARN, "%s%d: px_ib_ino_rem_intr: pending "
                    "sysino 0x%lx(ino 0x%x) timeout",
                    ddi_driver_name(dip), ddi_get_instance(dip),
                    sysino, ino);
        }

        /*
         * If the interrupt was previously blocked (left in pending state)
         * because of jabber we need to clear the pending state in case the
         * jabber has gone away.
         */
        if (ret == DDI_SUCCESS &&
            ino_p->ino_unclaimed_intrs > px_unclaimed_intr_max) {
                cmn_err(CE_WARN, "%s%d: px_ib_ino_rem_intr: "
                    "ino 0x%x has been unblocked",
                    ddi_driver_name(dip), ddi_get_instance(dip), ino);

                ino_p->ino_unclaimed_intrs = 0;
                ret = px_lib_intr_setstate(dip, sysino, INTR_IDLE_STATE);
        }

        if (ret != DDI_SUCCESS) {
                DBG(DBG_IB, dip, "px_ib_ino_rem_intr: failed, "
                    "ino 0x%x sysino 0x%x\n", ino, sysino);

                return (ret);
        }

        if (ipil_p->ipil_ih_size == 1) {
                if (ih_lst != ih_p)
                        goto not_found;

                /* No need to set head/tail as ino_p will be freed */
                goto reset;
        }

        /* Search the link list for ih_p */
        for (i = 0; (i < ipil_p->ipil_ih_size) &&
            (ih_lst->ih_next != ih_p); i++, ih_lst = ih_lst->ih_next)
                ;

        if (ih_lst->ih_next != ih_p)
                goto not_found;

        /* Remove ih_p from the link list and maintain the head/tail */
        ih_lst->ih_next = ih_p->ih_next;

        if (ipil_p->ipil_ih_head == ih_p)
                ipil_p->ipil_ih_head = ih_p->ih_next;
        if (ipil_p->ipil_ih_tail == ih_p)
                ipil_p->ipil_ih_tail = ih_lst;

        ipil_p->ipil_ih_start = ipil_p->ipil_ih_head;

reset:
        if (ih_p->ih_config_handle)
                pci_config_teardown(&ih_p->ih_config_handle);
        if (ih_p->ih_ksp != NULL)
                kstat_delete(ih_p->ih_ksp);

        kmem_free(ih_p, sizeof (px_ih_t));
        ipil_p->ipil_ih_size--;

        return (ret);

not_found:
        DBG(DBG_R_INTX, ino_p->ino_ib_p->ib_px_p->px_dip,
            "ino_p=%x does not have ih_p=%x\n", ino_p, ih_p);

        return (DDI_FAILURE);
}

px_ih_t *
px_ib_intr_locate_ih(px_ino_pil_t *ipil_p, dev_info_t *rdip,
    uint32_t inum, msiq_rec_type_t rec_type, msgcode_t msg_code)
{
        px_ih_t *ih_p = ipil_p->ipil_ih_head;
        int     i;

        for (i = 0; i < ipil_p->ipil_ih_size; i++, ih_p = ih_p->ih_next) {
                if ((ih_p->ih_dip == rdip) && (ih_p->ih_inum == inum) &&
                    (ih_p->ih_rec_type == rec_type) &&
                    (ih_p->ih_msg_code == msg_code))
                        return (ih_p);
        }

        return ((px_ih_t *)NULL);
}

px_ih_t *
px_ib_alloc_ih(dev_info_t *rdip, uint32_t inum,
    uint_t (*int_handler)(caddr_t int_handler_arg1, caddr_t int_handler_arg2),
    caddr_t int_handler_arg1, caddr_t int_handler_arg2,
    msiq_rec_type_t rec_type, msgcode_t msg_code)
{
        px_ih_t *ih_p;

        ih_p = kmem_alloc(sizeof (px_ih_t), KM_SLEEP);
        ih_p->ih_dip = rdip;
        ih_p->ih_inum = inum;
        ih_p->ih_intr_state = PX_INTR_STATE_DISABLE;
        ih_p->ih_intr_flags = PX_INTR_IDLE;
        ih_p->ih_handler = int_handler;
        ih_p->ih_handler_arg1 = int_handler_arg1;
        ih_p->ih_handler_arg2 = int_handler_arg2;
        ih_p->ih_config_handle = NULL;
        ih_p->ih_rec_type = rec_type;
        ih_p->ih_msg_code = msg_code;
        ih_p->ih_nsec = 0;
        ih_p->ih_ticks = 0;
        ih_p->ih_ksp = NULL;

        return (ih_p);
}

int
px_ib_update_intr_state(px_t *px_p, dev_info_t *rdip,
    uint_t inum, devino_t ino, uint_t pil,
    uint_t new_intr_state, msiq_rec_type_t rec_type,
    msgcode_t msg_code)
{
        px_ib_t         *ib_p = px_p->px_ib_p;
        px_ino_t        *ino_p;
        px_ino_pil_t    *ipil_p;
        px_ih_t         *ih_p;
        int             ret = DDI_FAILURE;

        DBG(DBG_IB, px_p->px_dip, "px_ib_update_intr_state: %s%d "
            "inum %x devino %x pil %x state %x\n", ddi_driver_name(rdip),
            ddi_get_instance(rdip), inum, ino, pil, new_intr_state);

        mutex_enter(&ib_p->ib_ino_lst_mutex);

        ino_p = px_ib_locate_ino(ib_p, ino);
        if (ino_p && (ipil_p = px_ib_ino_locate_ipil(ino_p, pil))) {
                if (ih_p = px_ib_intr_locate_ih(ipil_p, rdip, inum, rec_type,
                    msg_code)) {
                        ih_p->ih_intr_state = new_intr_state;
                        ret = DDI_SUCCESS;
                }
        }

        mutex_exit(&ib_p->ib_ino_lst_mutex);
        return (ret);
}


/*
 * Get interrupt CPU for a given ino.
 * Return info only for inos which are already mapped to devices.
 */
/*ARGSUSED*/
int
px_ib_get_intr_target(px_t *px_p, devino_t ino, cpuid_t *cpu_id_p)
{
        dev_info_t      *dip = px_p->px_dip;
        sysino_t        sysino;
        int             ret;

        DBG(DBG_IB, px_p->px_dip, "px_ib_get_intr_target: devino %x\n", ino);

        /* Convert leaf-wide intr to system-wide intr */
        if (px_lib_intr_devino_to_sysino(dip, ino, &sysino) != DDI_SUCCESS)
                return (DDI_FAILURE);

        ret = px_lib_intr_gettarget(dip, sysino, cpu_id_p);

        DBG(DBG_IB, px_p->px_dip, "px_ib_get_intr_target: cpu_id %x\n",
            *cpu_id_p);

        return (ret);
}


/*
 * Associate a new CPU with a given ino.
 * Operate only on INOs which are already mapped to devices.
 */
int
px_ib_set_intr_target(px_t *px_p, devino_t ino, cpuid_t cpu_id)
{
        dev_info_t              *dip = px_p->px_dip;
        cpuid_t                 old_cpu_id;
        sysino_t                sysino;
        int                     ret = DDI_SUCCESS;
        extern const int        _ncpu;
        extern cpu_t            *cpu[];

        DBG(DBG_IB, px_p->px_dip, "px_ib_set_intr_target: devino %x "
            "cpu_id %x\n", ino, cpu_id);

        mutex_enter(&cpu_lock);

        /* Convert leaf-wide intr to system-wide intr */
        if (px_lib_intr_devino_to_sysino(dip, ino, &sysino) != DDI_SUCCESS) {
                ret = DDI_FAILURE;
                goto done;
        }

        if (px_lib_intr_gettarget(dip, sysino, &old_cpu_id) != DDI_SUCCESS) {
                ret = DDI_FAILURE;
                goto done;
        }

        /*
         * Get lock, validate cpu and write it.
         */
        if ((cpu_id < _ncpu) && (cpu[cpu_id] && cpu_is_online(cpu[cpu_id]))) {
                DBG(DBG_IB, dip, "px_ib_set_intr_target: Enabling CPU %d\n",
                    cpu_id);
                px_ib_intr_dist_en(dip, cpu_id, ino, B_TRUE);
                px_ib_log_new_cpu(px_p->px_ib_p, old_cpu_id, cpu_id, ino);
        } else {        /* Invalid cpu */
                DBG(DBG_IB, dip, "px_ib_set_intr_target: Invalid cpuid %x\n",
                    cpu_id);
                ret = DDI_EINVAL;
        }

done:
        mutex_exit(&cpu_lock);
        return (ret);
}

hrtime_t px_ib_msix_retarget_timeout = 120ll * NANOSEC; /* 120 seconds */

/*
 * Associate a new CPU with a given MSI/X.
 * Operate only on MSI/Xs which are already mapped to devices.
 */
int
px_ib_set_msix_target(px_t *px_p, ddi_intr_handle_impl_t *hdlp,
    msinum_t msi_num, cpuid_t cpu_id)
{
        px_ib_t                 *ib_p = px_p->px_ib_p;
        px_msi_state_t          *msi_state_p = &px_p->px_ib_p->ib_msi_state;
        dev_info_t              *dip = px_p->px_dip;
        dev_info_t              *rdip = hdlp->ih_dip;
        msiqid_t                msiq_id, old_msiq_id;
        pci_msi_state_t         msi_state;
        msiq_rec_type_t         msiq_rec_type;
        msi_type_t              msi_type;
        px_ino_t                *ino_p;
        px_ih_t                 *ih_p, *old_ih_p;
        cpuid_t                 old_cpu_id;
        hrtime_t                start_time, end_time;
        int                     ret = DDI_SUCCESS;
        extern const int        _ncpu;
        extern cpu_t            *cpu[];

        DBG(DBG_IB, dip, "px_ib_set_msix_target: msi_num %x new cpu_id %x\n",
            msi_num, cpu_id);

        mutex_enter(&cpu_lock);

        /* Check for MSI64 support */
        if ((hdlp->ih_cap & DDI_INTR_FLAG_MSI64) && msi_state_p->msi_addr64) {
                msiq_rec_type = MSI64_REC;
                msi_type = MSI64_TYPE;
        } else {
                msiq_rec_type = MSI32_REC;
                msi_type = MSI32_TYPE;
        }

        if ((ret = px_lib_msi_getmsiq(dip, msi_num,
            &old_msiq_id)) != DDI_SUCCESS) {

                mutex_exit(&cpu_lock);
                return (ret);
        }

        DBG(DBG_IB, dip, "px_ib_set_msix_target: current msiq 0x%x\n",
            old_msiq_id);

        if ((ret = px_ib_get_intr_target(px_p,
            px_msiqid_to_devino(px_p, old_msiq_id),
            &old_cpu_id)) != DDI_SUCCESS) {

                mutex_exit(&cpu_lock);
                return (ret);
        }

        DBG(DBG_IB, dip, "px_ib_set_msix_target: current cpuid 0x%x\n",
            old_cpu_id);

        if (cpu_id == old_cpu_id) {

                mutex_exit(&cpu_lock);
                return (DDI_SUCCESS);
        }

        /*
         * Get lock, validate cpu and write it.
         */
        if (!((cpu_id < _ncpu) && (cpu[cpu_id] &&
            cpu_is_online(cpu[cpu_id])))) {
                /* Invalid cpu */
                DBG(DBG_IB, dip, "px_ib_set_msix_target: Invalid cpuid %x\n",
                    cpu_id);

                mutex_exit(&cpu_lock);
                return (DDI_EINVAL);
        }

        DBG(DBG_IB, dip, "px_ib_set_msix_target: Enabling CPU %d\n", cpu_id);

        if ((ret = px_add_msiq_intr(dip, rdip, hdlp,
            msiq_rec_type, msi_num, cpu_id, &msiq_id)) != DDI_SUCCESS) {
                DBG(DBG_IB, dip, "px_ib_set_msix_target: Add MSI handler "
                    "failed, rdip 0x%p msi 0x%x\n", rdip, msi_num);

                mutex_exit(&cpu_lock);
                return (ret);
        }

        if ((ret = px_lib_msi_setmsiq(dip, msi_num,
            msiq_id, msi_type)) != DDI_SUCCESS) {
                mutex_exit(&cpu_lock);

                (void) px_rem_msiq_intr(dip, rdip,
                    hdlp, msiq_rec_type, msi_num, msiq_id);

                return (ret);
        }

        if ((ret = px_ib_update_intr_state(px_p, rdip, hdlp->ih_inum,
            px_msiqid_to_devino(px_p, msiq_id), hdlp->ih_pri,
            PX_INTR_STATE_ENABLE, msiq_rec_type, msi_num)) != DDI_SUCCESS) {
                mutex_exit(&cpu_lock);

                (void) px_rem_msiq_intr(dip, rdip,
                    hdlp, msiq_rec_type, msi_num, msiq_id);

                return (ret);
        }

        mutex_exit(&cpu_lock);

        /*
         * Remove the old handler, but first ensure it is finished.
         *
         * Each handler sets its PENDING flag before it clears the MSI state.
         * Then it clears that flag when finished.  If a re-target occurs while
         * the MSI state is DELIVERED, then it is not yet known which of the
         * two handlers will take the interrupt.  So the re-target operation
         * sets a RETARGET flag on both handlers in that case.  Monitoring both
         * flags on both handlers then determines when the old handler can be
         * be safely removed.
         */
        mutex_enter(&ib_p->ib_ino_lst_mutex);

        ino_p = px_ib_locate_ino(ib_p, px_msiqid_to_devino(px_p, old_msiq_id));
        old_ih_p = px_ib_intr_locate_ih(px_ib_ino_locate_ipil(ino_p,
            hdlp->ih_pri), rdip, hdlp->ih_inum, msiq_rec_type, msi_num);

        ino_p = px_ib_locate_ino(ib_p, px_msiqid_to_devino(px_p, msiq_id));
        ih_p = px_ib_intr_locate_ih(px_ib_ino_locate_ipil(ino_p, hdlp->ih_pri),
            rdip, hdlp->ih_inum, msiq_rec_type, msi_num);

        if ((ret = px_lib_msi_getstate(dip, msi_num,
            &msi_state)) != DDI_SUCCESS) {
                (void) px_rem_msiq_intr(dip, rdip,
                    hdlp, msiq_rec_type, msi_num, msiq_id);

                mutex_exit(&ib_p->ib_ino_lst_mutex);
                return (ret);
        }

        if (msi_state == PCI_MSI_STATE_DELIVERED) {
                ih_p->ih_intr_flags |= PX_INTR_RETARGET;
                old_ih_p->ih_intr_flags |= PX_INTR_RETARGET;
        }

        start_time = gethrtime();
        while (((ih_p->ih_intr_flags & PX_INTR_RETARGET) &&
            (old_ih_p->ih_intr_flags & PX_INTR_RETARGET)) ||
            (old_ih_p->ih_intr_flags & PX_INTR_PENDING)) {

                /* Wait for one second */
                delay(drv_usectohz(1000000));

                end_time = gethrtime() - start_time;
                if (end_time > px_ib_msix_retarget_timeout) {
                        cmn_err(CE_WARN, "MSIX retarget %x is not completed, "
                            "even after waiting %llx ticks\n",
                            msi_num, end_time);
                        break;
                }
        }

        ih_p->ih_intr_flags &= ~(PX_INTR_RETARGET);

        mutex_exit(&ib_p->ib_ino_lst_mutex);

        ret = px_rem_msiq_intr(dip, rdip,
            hdlp, msiq_rec_type, msi_num, old_msiq_id);

        return (ret);
}


static void
px_fill_in_intr_devs(pcitool_intr_dev_t *dev, char *driver_name,
    char *path_name, int instance)
{
        (void) strlcpy(dev->driver_name, driver_name, MAXMODCONFNAME);
        (void) strlcpy(dev->path, path_name, MAXPATHLEN);
        dev->dev_inst = instance;
}


/*
 * Return the dips or number of dips associated with a given interrupt block.
 * Size of dips array arg is passed in as dips_ret arg.
 * Number of dips returned is returned in dips_ret arg.
 * Array of dips gets returned in the dips argument.
 * Function returns number of dips existing for the given interrupt block.
 *
 * Note: this function assumes an enabled/valid INO, which is why it returns
 * the px node and (Internal) when it finds no other devices (and *devs_ret > 0)
 */
uint8_t
pxtool_ib_get_ino_devs(px_t *px_p, uint32_t ino, uint32_t msi_num,
    uint8_t *devs_ret, pcitool_intr_dev_t *devs)
{
        px_ib_t         *ib_p = px_p->px_ib_p;
        px_ino_t        *ino_p;
        px_ino_pil_t    *ipil_p;
        px_ih_t         *ih_p;
        uint32_t        num_devs = 0;
        char            pathname[MAXPATHLEN];
        int             i, j;

        mutex_enter(&ib_p->ib_ino_lst_mutex);
        ino_p = px_ib_locate_ino(ib_p, ino);
        if (ino_p != NULL) {
                for (j = 0, ipil_p = ino_p->ino_ipil_p; ipil_p;
                    ipil_p = ipil_p->ipil_next_p) {
                        num_devs += ipil_p->ipil_ih_size;

                        for (i = 0, ih_p = ipil_p->ipil_ih_head;
                            ((i < ipil_p->ipil_ih_size) && (i < *devs_ret));
                            i++, j++, ih_p = ih_p->ih_next) {
                                (void) ddi_pathname(ih_p->ih_dip, pathname);

                                if (ih_p->ih_msg_code == msi_num) {
                                        num_devs = *devs_ret = 1;
                                        px_fill_in_intr_devs(&devs[0],
                                            (char *)ddi_driver_name(
                                            ih_p->ih_dip), pathname,
                                            ddi_get_instance(ih_p->ih_dip));
                                        goto done;
                                }

                                px_fill_in_intr_devs(&devs[j],
                                    (char *)ddi_driver_name(ih_p->ih_dip),
                                    pathname, ddi_get_instance(ih_p->ih_dip));
                        }
                }

                *devs_ret = j;
        } else if (*devs_ret > 0) {
                (void) ddi_pathname(px_p->px_dip, pathname);
                strcat(pathname, " (Internal)");
                px_fill_in_intr_devs(&devs[0],
                    (char *)ddi_driver_name(px_p->px_dip),  pathname,
                    ddi_get_instance(px_p->px_dip));
                num_devs = *devs_ret = 1;
        }

done:
        mutex_exit(&ib_p->ib_ino_lst_mutex);

        return (num_devs);
}


int
pxtool_ib_get_msi_info(px_t *px_p, devino_t ino, msinum_t msi_num,
    ddi_intr_handle_impl_t *hdlp)
{
        px_ib_t         *ib_p = px_p->px_ib_p;
        px_ino_t        *ino_p;
        px_ino_pil_t    *ipil_p;
        px_ih_t         *ih_p;
        int             i;

        mutex_enter(&ib_p->ib_ino_lst_mutex);

        if ((ino_p = px_ib_locate_ino(ib_p, ino)) == NULL) {
                mutex_exit(&ib_p->ib_ino_lst_mutex);
                return (DDI_FAILURE);
        }

        for (ipil_p = ino_p->ino_ipil_p; ipil_p;
            ipil_p = ipil_p->ipil_next_p) {
                for (i = 0, ih_p = ipil_p->ipil_ih_head;
                    ((i < ipil_p->ipil_ih_size) && ih_p);
                    i++, ih_p = ih_p->ih_next) {

                        if (ih_p->ih_msg_code != msi_num)
                                continue;

                        hdlp->ih_dip = ih_p->ih_dip;
                        hdlp->ih_inum = ih_p->ih_inum;
                        hdlp->ih_cb_func = ih_p->ih_handler;
                        hdlp->ih_cb_arg1 = ih_p->ih_handler_arg1;
                        hdlp->ih_cb_arg2 = ih_p->ih_handler_arg2;
                        if (ih_p->ih_rec_type == MSI64_REC)
                                hdlp->ih_cap = DDI_INTR_FLAG_MSI64;
                        hdlp->ih_pri = ipil_p->ipil_pil;
                        hdlp->ih_ver = DDI_INTR_VERSION;

                        mutex_exit(&ib_p->ib_ino_lst_mutex);
                        return (DDI_SUCCESS);
                }
        }

        mutex_exit(&ib_p->ib_ino_lst_mutex);
        return (DDI_FAILURE);
}

void
px_ib_log_new_cpu(px_ib_t *ib_p, cpuid_t old_cpu_id, cpuid_t new_cpu_id,
    uint32_t ino)
{
        px_ino_t        *ino_p;
        px_ino_pil_t    *ipil_p;
        px_ih_t         *ih_p;
        int             i;

        mutex_enter(&ib_p->ib_ino_lst_mutex);

        /* Log in OS data structures the new CPU. */
        if (ino_p = px_ib_locate_ino(ib_p, ino)) {

                /* Log in OS data structures the new CPU. */
                ino_p->ino_cpuid = new_cpu_id;

                for (ipil_p = ino_p->ino_ipil_p; ipil_p;
                    ipil_p = ipil_p->ipil_next_p) {
                        for (i = 0, ih_p = ipil_p->ipil_ih_head;
                            (i < ipil_p->ipil_ih_size);
                            i++, ih_p = ih_p->ih_next) {
                                /*
                                 * Account for any residual time
                                 * to be logged for old cpu.
                                 */
                                px_ib_cpu_ticks_to_ih_nsec(ib_p,
                                    ih_p, old_cpu_id);
                        }
                }
        }

        mutex_exit(&ib_p->ib_ino_lst_mutex);
}