/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2019 Joyent, Inc.
 */

#include <sys/types.h>
#include <sys/ddi.h>
#include <sys/kmem.h>
#include <sys/sysmacros.h>
#include <sys/sunddi.h>
#include <sys/sunpm.h>
#include <sys/epm.h>
#include <sys/sunndi.h>
#include <sys/ddi_impldefs.h>
#include <sys/ddi_implfuncs.h>
#include <sys/pcie.h>
#include <sys/pcie_impl.h>
#include <sys/promif.h>         /* prom_printf */
#include <sys/pcie_pwr.h>

/*
 * This file implements the power management functionality for
 * pci express switch and pci express-to-pci/pci-x bridge. All the
 * code in this file is generic and is not specific to a particular chip.
 * The algorithm, which decides when to go to a lower power is explained
 * below:
 *
 *      1. Initially when no children are attached, the driver is idle from
 *      PM framework point of view ( PM idle/PM busy).
 *
 *      2. Driver is PM busy if either a reference count called pwr_hold is
 *      greater than zero or driver is already at the lowest possible power
 *      level. The lowest possible power level for the driver is equal to the
 *      highest power level among its children. The PM busy condition is
 *      indicated by PCIE_PM_BUSY bit. At any point, only one pm_busy_component
 *      call is made for a nexus driver instance.
 *
 *      3. Driver is PM idle if the pwr_hold is zero and the lowest
 *      possible power level is less than the driver's current power level.
 *      At any point, only one pm_idle_component call is made for a nexus
 *      driver instance.
 *
 *      4. For any events like child attach, it increments pwr_hold and marks
 *      itslef busy, if it is not already done so. This temporary hold is
 *      removed when the event is complete.
 *
 *      5. Any child's power change requires the parent (this driver) to be
 *      full power. So it raises its power and increments pwr_hold. It also
 *      marks itself temporarily busy, if it is not already done. This hold
 *      is removed when the child power change is complete.
 *
 *      6. After each child power change, it evaluates what is the lowest
 *      possible power level. If the lowest possible power level is less than
 *      the current power level and pwr_hold is zero, then it marks itself
 *      idle. The lowest power level is equal or greater than the highest level
 *      among the children. It keeps track of children's power level by
 *      using counters.
 *
 *      7. Any code e.g., which is accessing the driver's own registers should
 *      place a temporary hold using pcie_pm_hold.
 */

static int pcie_pwr_change(dev_info_t *dip, pcie_pwr_t *pwr_p, int new);
static void pwr_update_counters(int *countersp, int olevel, int nlevel);
static int pwr_level_allowed(pcie_pwr_t *pwr_p);
static void pcie_add_comps(dev_info_t *dip, dev_info_t *cdip,
    pcie_pwr_t *pwr_p);
static void pcie_remove_comps(dev_info_t *dip, dev_info_t *cdip,
    pcie_pwr_t *pwr_p);
static void pcie_pm_subrelease(dev_info_t *dip, pcie_pwr_t *pwr_p);
static boolean_t pcie_is_pcie(dev_info_t *dip);
#ifdef DEBUG
static char *pcie_decode_pwr_op(pm_bus_power_op_t op);
#else
#define pcie_decode_pwr_op
#endif

/*
 * power entry point.
 *
 * This function decides whether the PM request is honorable.
 * If yes, it then does what's necessary for switch or
 *    bridge to change its power.
 */
/* ARGSUSED */
int
pcie_power(dev_info_t *dip, int component, int level)
{
        pcie_pwr_t *pwr_p = PCIE_NEXUS_PMINFO(dip);
        int *counters = pwr_p->pwr_counters;
        int pmcaps = pwr_p->pwr_pmcaps;
        int ret = DDI_FAILURE;

#if defined(__x86)
        if (dip)
                return (DDI_SUCCESS);
#endif /* defined(__x86) */

        ASSERT(level != PM_LEVEL_UNKNOWN);
        /* PM should not asking for a level, which is unsupported */
        ASSERT(level == PM_LEVEL_D0 || level == PM_LEVEL_D3 ||
            (level == PM_LEVEL_D1 && (pmcaps & PCIE_SUPPORTS_D1)) ||
            (level == PM_LEVEL_D2 && (pmcaps & PCIE_SUPPORTS_D2)));

        mutex_enter(&pwr_p->pwr_lock);
        PCIE_DBG("%s(%d): pcie_power: change from %d to %d\n",
            ddi_driver_name(dip), ddi_get_instance(dip), pwr_p->pwr_func_lvl,
            level);
        if (pwr_p->pwr_func_lvl == level) {
                PCIE_DBG("%s(%d): pcie_power: already at %d\n",
                    ddi_driver_name(dip), ddi_get_instance(dip), level);
                ret = DDI_SUCCESS;
                goto pcie_pwr_done;
        }

        if (level < pwr_p->pwr_func_lvl) {
                /*
                 * Going to lower power. Reject this if we are either busy
                 * or there is a hold.
                 */
                if (pwr_p->pwr_flags & PCIE_PM_BUSY) {
                        PCIE_DBG("%s(%d): pcie_power: rejecting change to %d "
                            "as busy\n", ddi_driver_name(dip),
                            ddi_get_instance(dip), level);
                        goto pcie_pwr_done;
                }

                /*
                 * Now we know that we are neither busy nor there is a hold.
                 * At this point none of the children should be at full power.
                 * Reject the request if level reqested is lower than the level
                 * possible.
                 */
                ASSERT(!counters[PCIE_D0_INDEX] &&
                    !counters[PCIE_UNKNOWN_INDEX]);
                if (level < pwr_level_allowed(pwr_p)) {
                        PCIE_DBG("%s(%d): pcie_power: rejecting level %d as"
                            " %d is the lowest possible\n",
                            ddi_driver_name(dip), ddi_get_instance(dip), level,
                            pwr_level_allowed(pwr_p));
                        goto pcie_pwr_done;
                }
        }

        if (pcie_pwr_change(dip, pwr_p, level) != DDI_SUCCESS) {
                PCIE_DBG("%s(%d): pcie_power: attempt to change to %d "
                    " failed \n", ddi_driver_name(dip), ddi_get_instance(dip),
                    level);
                goto pcie_pwr_done;
        }
        pwr_p->pwr_func_lvl = level;
        PCIE_DBG("%s(%d): pcie_power: level changed to %d \n",
            ddi_driver_name(dip), ddi_get_instance(dip), level);
        ret = DDI_SUCCESS;

pcie_pwr_done:
        mutex_exit(&pwr_p->pwr_lock);
        return (ret);
}

/*
 * Called by pcie_power() only. Caller holds the pwr_lock.
 *
 * dip - dev_info pointer
 * pwr_p - pm info for the node.
 * new     - new level
 */
static int
pcie_pwr_change(dev_info_t *dip, pcie_pwr_t *pwr_p, int new)
{
        uint16_t pmcsr;

        ASSERT(MUTEX_HELD(&pwr_p->pwr_lock));
        ASSERT(new != pwr_p->pwr_func_lvl);
        pmcsr = pci_config_get16(pwr_p->pwr_conf_hdl, pwr_p->pwr_pmcsr_offset);
        pmcsr &= ~PCI_PMCSR_STATE_MASK;
        switch (new) {
        case PM_LEVEL_D0:
                pmcsr |= PCI_PMCSR_D0;
                break;

        case PM_LEVEL_D1:
                pmcsr |= PCI_PMCSR_D1;
                break;

        case PM_LEVEL_D2:
                pmcsr |= PCI_PMCSR_D2;
                break;

        case PM_LEVEL_D3:
                pmcsr |= PCI_PMCSR_D3HOT;
                break;

        default:
                ASSERT(0);
                break;
        }
        /* Save config space, if going to D3 */
        if (new == PM_LEVEL_D3) {
                PCIE_DBG("%s(%d): pwr_change: saving config space regs\n",
                    ddi_driver_name(dip), ddi_get_instance(dip));
                if (pci_save_config_regs(dip) != DDI_SUCCESS) {
                        PCIE_DBG("%s(%d): pcie_pwr_change: failed to save "
                            "config space regs\n", ddi_driver_name(dip),
                            ddi_get_instance(dip));
                        return (DDI_FAILURE);
                }
        }

        pci_config_put16(pwr_p->pwr_conf_hdl, pwr_p->pwr_pmcsr_offset, pmcsr);

        /*
         * TBD: Taken from pci_pci driver. Is this required?
         * No bus transactions should occur without waiting for
         * settle time specified in PCI PM spec rev 2.1 sec 5.6.1
         * To make things simple, just use the max time specified for
         * all state transitions.
         */
        delay(drv_usectohz(PCI_CLK_SETTLE_TIME));

        /*
         * Restore config space if coming out of D3
         */
        if (pwr_p->pwr_func_lvl == PM_LEVEL_D3) {
                PCIE_DBG("%s(%d): pcie_pwr_change: restoring config space\n",
                    ddi_driver_name(dip), ddi_get_instance(dip));
                if (pci_restore_config_regs(dip) != DDI_SUCCESS) {
                        PCIE_DBG("%s(%d): pcie_pwr_change: failed to restore "
                            "config space regs\n", ddi_driver_name(dip),
                            ddi_get_instance(dip));
                        return (DDI_FAILURE);
                }
        }
        return (DDI_SUCCESS);
}

/*
 * bus_ctlops.bus_power function.
 *
 * This function handles PRE_ POST_ change notifications, sent by
 * PM framework related to child's power level change. It marks itself
 * idle or busy based on the children's power level.
 */
int
pcie_bus_power(dev_info_t *dip, void *impl_arg, pm_bus_power_op_t op,
    void *arg, void *result)
{
        pcie_pwr_t *pwr_p = PCIE_NEXUS_PMINFO(dip);
        int *counters = pwr_p->pwr_counters; /* nexus counters */
        int *child_counters; /* per child dip counters */
        pm_bp_child_pwrchg_t *bpc;
        pm_bp_has_changed_t *bphc;
        dev_info_t *cdip;
        int new_level;
        int old_level;
        int rv = DDI_SUCCESS;
        int level_allowed, comp;

#if defined(__x86)
        if (dip)
                return (DDI_SUCCESS);
#endif /* defined(__x86) */

        switch (op) {
        case BUS_POWER_PRE_NOTIFICATION:
        case BUS_POWER_POST_NOTIFICATION:
                bpc = (pm_bp_child_pwrchg_t *)arg;
                cdip = bpc->bpc_dip;
                new_level = bpc->bpc_nlevel;
                old_level = bpc->bpc_olevel;
                comp = bpc->bpc_comp;
                break;

        case BUS_POWER_HAS_CHANGED:
                bphc = (pm_bp_has_changed_t *)arg;
                cdip = bphc->bphc_dip;
                new_level = bphc->bphc_nlevel;
                old_level = bphc->bphc_olevel;
                comp = bphc->bphc_comp;
                break;

        default:
                break;

        }

        ASSERT(pwr_p);
        mutex_enter(&pwr_p->pwr_lock);
        switch (op) {
        case BUS_POWER_PRE_NOTIFICATION:
                PCIE_DBG("%s(%d): pcie_bus_power: %s@%d op %s %d->%d\n",
                    ddi_driver_name(dip), ddi_get_instance(dip),
                    ddi_driver_name(cdip), ddi_get_instance(cdip),
                    pcie_decode_pwr_op(op), old_level, new_level);
                /*
                 * If the nexus doesn't want the child to go into
                 * non-D0 state, mark the child busy. This way PM
                 * framework will never try to lower the child's power.
                 * In case of pm_lower_power, marking busy won't help.
                 * So we need to specifically reject the attempt to
                 * go to non-D0 state.
                 */
                if (pwr_p->pwr_flags & PCIE_NO_CHILD_PM) {
                        if (!PCIE_IS_COMPS_COUNTED(cdip)) {
                                PCIE_DBG("%s(%d): pcie_bus_power: marking "
                                    "child busy to disable pm \n",
                                    ddi_driver_name(dip),
                                    ddi_get_instance(dip));
                                (void) pm_busy_component(cdip, 0);
                        }
                        if (new_level < PM_LEVEL_D0 && !comp) {
                                PCIE_DBG("%s(%d): pcie_bus_power: rejecting "
                                    "child's attempt to go to %d\n",
                                    ddi_driver_name(dip), ddi_get_instance(dip),
                                    new_level);
                                rv = DDI_FAILURE;
                        }
                }
                mutex_exit(&pwr_p->pwr_lock);
                if (rv == DDI_SUCCESS)
                        rv = pcie_pm_hold(dip);
                return (rv);

        case BUS_POWER_HAS_CHANGED:
        case BUS_POWER_POST_NOTIFICATION:
                PCIE_DBG("%s(%d): pcie_bus_power: %s@%d op %s %d->%d\n",
                    ddi_driver_name(dip), ddi_get_instance(dip),
                    ddi_driver_name(cdip), ddi_get_instance(cdip),
                    pcie_decode_pwr_op(op), old_level, new_level);
                /*
                 * Child device power changed
                 * If pm components of this child aren't accounted for
                 * then add the components to the counters. This can't
                 * be done in POST_ATTACH ctlop as pm info isn't created
                 * by then. Also because a driver can make a pm call during
                 * the attach.
                 */
                if (!PCIE_IS_COMPS_COUNTED(cdip)) {
                        (void) pcie_pm_add_child(dip, cdip);
                        if ((pwr_p->pwr_flags & PCIE_NO_CHILD_PM) &&
                            (op == BUS_POWER_HAS_CHANGED)) {
                                PCIE_DBG("%s(%d): pcie_bus_power: marking "
                                    "child busy to disable pm \n",
                                    ddi_driver_name(dip),
                                    ddi_get_instance(dip));
                                (void) pm_busy_component(cdip, 0);
                                /*
                                 * If the driver has already changed to lower
                                 * power(pm_power_has_changed) on its own,
                                 * there is nothing we can do other than
                                 * logging the warning message on the console.
                                 */
                                if (new_level < PM_LEVEL_D0)
                                        cmn_err(CE_WARN, "!Downstream device "
                                            "%s@%d went to non-D0 state: "
                                            "possible loss of link\n",
                                            ddi_driver_name(cdip),
                                            ddi_get_instance(cdip));
                        }
                }


                /*
                 * If it is POST and device PM is supported, release the
                 * hold done in PRE.
                 */
                if (op == BUS_POWER_POST_NOTIFICATION &&
                    PCIE_SUPPORTS_DEVICE_PM(dip)) {
                        pcie_pm_subrelease(dip, pwr_p);
                }

                if (*((int *)result) == DDI_FAILURE) {
                        PCIE_DBG("%s(%d): pcie_bus_power: change for %s%d "
                            "failed\n", ddi_driver_name(dip),
                            ddi_get_instance(dip), ddi_driver_name(cdip),
                            ddi_get_instance(cdip));
                        break;
                }
                /* Modify counters appropriately */
                pwr_update_counters(counters, old_level, new_level);

                child_counters = PCIE_CHILD_COUNTERS(cdip);
                pwr_update_counters(child_counters, old_level, new_level);

                /* If no device PM, return */
                if (!PCIE_SUPPORTS_DEVICE_PM(dip))
                        break;

                level_allowed = pwr_level_allowed(pwr_p);
                /*
                 * Check conditions for marking busy
                 * Check the flag to set this busy only once for multiple
                 * busy conditions. Mark busy if our current lowest possible
                 * is equal or greater to the current level.
                 */
                if (level_allowed >= pwr_p->pwr_func_lvl &&
                    !(pwr_p->pwr_flags & PCIE_PM_BUSY)) {
                        PCIE_DBG("%s(%d): pcie_bus_power: marking busy\n",
                            ddi_driver_name(dip), ddi_get_instance(dip));
                        (void) pm_busy_component(dip, 0);
                        pwr_p->pwr_flags |= PCIE_PM_BUSY;
                        break;
                }
                /*
                 * Check conditions for marking idle.
                 * If our lowest possible level is less than our current
                 * level mark idle. Mark idle only if it is not already done.
                 */
                if ((level_allowed < pwr_p->pwr_func_lvl) &&
                    (pwr_p->pwr_hold == 0) &&
                    (pwr_p->pwr_flags & PCIE_PM_BUSY)) {
                        /*
                         * For pci express, we should check here whether
                         * the link is in L1 state or not.
                         */
                        PCIE_DBG("%s(%d): pcie_bus_power: marking idle\n",
                            ddi_driver_name(dip), ddi_get_instance(dip));
                        (void) pm_idle_component(dip, 0);
                        pwr_p->pwr_flags &= ~PCIE_PM_BUSY;
                        break;
                }
                break;

        default:
                mutex_exit(&pwr_p->pwr_lock);
                return (pm_busop_bus_power(dip, impl_arg, op, arg, result));
        }
        mutex_exit(&pwr_p->pwr_lock);
        return (rv);
}

/*
 * Decrement the count of children at olevel by one and increment
 * count of children at nlevel by one.
 */
static void
pwr_update_counters(int *countersp, int olevel, int nlevel)
{
        uint32_t        index;

        ASSERT(olevel >= PM_LEVEL_UNKNOWN && olevel <= PM_LEVEL_D0);
        ASSERT(nlevel >= PM_LEVEL_UNKNOWN && nlevel <= PM_LEVEL_D0);

        index = (olevel == PM_LEVEL_UNKNOWN ? PCIE_UNKNOWN_INDEX : olevel);
        countersp[index]--;
        index = (nlevel == PM_LEVEL_UNKNOWN ? PCIE_UNKNOWN_INDEX : nlevel);
        countersp[index]++;
}

/*
 * Returns the lowest possible power level allowed for nexus
 * based on children's power level. Lowest possible level is
 * equal to the highest level among the children. It also checks
 * for the supported level
 * UNKNOWN = D0 > D1 > D2 > D3
 */
static int
pwr_level_allowed(pcie_pwr_t *pwr_p)
{
        int *counters = pwr_p->pwr_counters;
        int i, j;

        ASSERT(MUTEX_HELD(&pwr_p->pwr_lock));
        /*
         * Search from UNKNOWN to D2. unknown is same as D0.
         * find the highest level among the children. If that
         * level is supported, return that level. If not,
         * find the next higher supported level and return that
         * level. For example, if the D1 is the highest among
         * children and if D1 isn't supported return D0 as the
         * lowest possible level. We don't need to look at D3
         * as that is the default lowest level and it is always
         * supported.
         */
        for (i = PCIE_UNKNOWN_INDEX; i > 0; i--) {
                if (counters[i]) {
                        if (i == PCIE_UNKNOWN_INDEX)
                                return (PM_LEVEL_D0);
                        /*
                         * i is the highest level among children. If this is
                         * supported, return i.
                         */
                        if (PCIE_LEVEL_SUPPORTED(pwr_p->pwr_pmcaps, i))
                                return (i);
                        /* find the next higher supported level */
                        for (j = i + 1; j <= PCIE_D0_INDEX; j++) {
                                if (PCIE_LEVEL_SUPPORTED(pwr_p->pwr_pmcaps, j))
                                        return (j);
                        }
                }
        }

        return (PM_LEVEL_D3);
}

/*
 * Update the counters with number pm components of the child
 * all components are assumed to be at UNKNOWN level.
 */
static void
pcie_add_comps(dev_info_t *dip, dev_info_t *cdip, pcie_pwr_t *pwr_p)
{
        int comps = PM_NUMCMPTS(cdip);
        pcie_pm_t *pcie_pm_p;
        pcie_pwr_child_t *cpwr_p;

        ASSERT(MUTEX_HELD(&pwr_p->pwr_lock));
        if (!comps)
                return;

        PCIE_DBG("%s(%d): pcie_add_comps: unknown level counter incremented "
            "from %d by %d because of %s@%d\n",
            ddi_driver_name(dip), ddi_get_instance(dip),
            (pwr_p->pwr_counters)[PCIE_UNKNOWN_INDEX], comps,
            ddi_driver_name(cdip), ddi_get_instance(cdip));
        (pwr_p->pwr_counters)[PCIE_UNKNOWN_INDEX] += comps;
        /*
         * Allocate counters per child. This is a part of pcie
         * pm info. If there is no pcie pm info, allocate it here.
         * pcie pm info might already be there for pci express nexus
         * driver e.g. pcieb. For all leaf nodes, it is allocated here.
         */
        if ((pcie_pm_p = PCIE_PMINFO(cdip)) == NULL) {
                pcie_pm_p = (pcie_pm_t *)kmem_zalloc(
                    sizeof (pcie_pm_t), KM_SLEEP);
                PCIE_SET_PMINFO(cdip, pcie_pm_p);
        }
        cpwr_p = (pcie_pwr_child_t *)kmem_zalloc(sizeof (pcie_pwr_child_t),
            KM_SLEEP);
        pcie_pm_p->pcie_par_pminfo = cpwr_p;
        (cpwr_p->pwr_child_counters)[PCIE_UNKNOWN_INDEX] += comps;
}

/*
 * Remove the pm components of a child from our counters.
 */
static void
pcie_remove_comps(dev_info_t *dip, dev_info_t *cdip, pcie_pwr_t *pwr_p)
{
        int i;
        int *child_counters;

        ASSERT(MUTEX_HELD(&pwr_p->pwr_lock));
        if (!(PCIE_PMINFO(cdip)) || !PCIE_PAR_PMINFO(cdip)) {
                if (PCIE_SUPPORTS_DEVICE_PM(dip)) {
                        /*
                         * Driver never made a PM call and we didn't create
                         * any counters for this device. This also means that
                         * hold made at the PRE_ATTACH time, still remains.
                         * Remove the hold now. The correct thing to do is to
                         * stay at full power when a child is at full power
                         * whether a driver is there or not. This will be
                         * implemented in the future.
                         */
                        pcie_pm_subrelease(dip, pwr_p);
                }
                return;
        }
        PCIE_DBG("%s(%d): pcie_remove_comps:counters decremented because of "
            "%s@%d\n", ddi_driver_name(dip), ddi_get_instance(dip),
            ddi_driver_name(cdip), ddi_get_instance(cdip));
        child_counters = PCIE_CHILD_COUNTERS(cdip);
        /*
         * Adjust the nexus counters. No need to adjust per child dip
         * counters as we are freeing the per child dip info.
         */
        for (i = 0; i < PCIE_MAX_PWR_LEVELS; i++) {
                ASSERT((pwr_p->pwr_counters)[i] >= child_counters[i]);
                (pwr_p->pwr_counters)[i] -= child_counters[i];
        }
        /* remove both parent pm info and pcie pminfo itself */
        kmem_free(PCIE_PAR_PMINFO(cdip), sizeof (pcie_pwr_child_t));
        kmem_free(PCIE_PMINFO(cdip), sizeof (pcie_pm_t));
        PCIE_RESET_PMINFO(cdip);
}

/*
 * Power management related initialization common to px and pcieb
 */
int
pwr_common_setup(dev_info_t *dip)
{
        pcie_pm_t               *pcie_pm_p;
        pcie_pwr_t              *pwr_p;
        int                     pminfo_created = 0;

        /* Create pminfo, if it doesn't exist already */
        if ((pcie_pm_p = PCIE_PMINFO(dip)) == NULL) {
                pcie_pm_p = (pcie_pm_t *)kmem_zalloc(
                    sizeof (pcie_pm_t), KM_SLEEP);
                PCIE_SET_PMINFO(dip, pcie_pm_p);
                pminfo_created = 1;
        }
        pwr_p = (pcie_pwr_t *)kmem_zalloc(sizeof (pcie_pwr_t), KM_SLEEP);
        mutex_init(&pwr_p->pwr_lock, NULL, MUTEX_DRIVER, NULL);
        /* Initialize the power level and default level support */
        pwr_p->pwr_func_lvl = PM_LEVEL_UNKNOWN;
        pwr_p->pwr_pmcaps = PCIE_DEFAULT_LEVEL_SUPPORTED;

        if (pcie_plat_pwr_setup(dip) != DDI_SUCCESS)
                goto pwr_common_err;

        pcie_pm_p->pcie_pwr_p = pwr_p;
        return (DDI_SUCCESS);

pwr_common_err:
        mutex_destroy(&pwr_p->pwr_lock);
        kmem_free(pwr_p, sizeof (pcie_pwr_t));
        if (pminfo_created) {
                PCIE_RESET_PMINFO(dip);
                kmem_free(pcie_pm_p, sizeof (pcie_pm_t));
        }
        return (DDI_FAILURE);

}

/*
 * Undo whatever is done in pwr_common_setup. Called by px_detach or pxb_detach
 */
void
pwr_common_teardown(dev_info_t *dip)
{
        pcie_pm_t *pcie_pm_p = PCIE_PMINFO(dip);
        pcie_pwr_t *pwr_p;

        if (!pcie_pm_p || !(pwr_p = PCIE_NEXUS_PMINFO(dip)))
                return;

        pcie_plat_pwr_teardown(dip);
        mutex_destroy(&pwr_p->pwr_lock);
        pcie_pm_p->pcie_pwr_p = NULL;
        kmem_free(pwr_p, sizeof (pcie_pwr_t));
        /*
         * If the parent didn't store have any pm info about
         * this node, that means parent doesn't need pminfo when it handles
         * POST_DETACH for this node. For example, if dip is the dip of
         * root complex, then there is no parent pm info.
         */
        if (!PCIE_PAR_PMINFO(dip)) {
                kmem_free(pcie_pm_p, sizeof (pcie_pm_t));
                PCIE_RESET_PMINFO(dip);
        }
}

/*
 * Raises the power and marks itself busy.
 */
int
pcie_pm_hold(dev_info_t *dip)
{
        pcie_pwr_t *pwr_p;

        /* If no PM info or no device PM, return */
        if (!PCIE_PMINFO(dip) || !(pwr_p = PCIE_NEXUS_PMINFO(dip)) ||
            !(PCIE_SUPPORTS_DEVICE_PM(dip)))
                return (DDI_SUCCESS);

        /*
         * If we are not at full power, then powerup.
         * Need to be at full power so that link can be
         * at L0. Similarly for PCI/PCI-X bus, it should be
         * at full power.
         */
        mutex_enter(&pwr_p->pwr_lock);
        ASSERT(pwr_p->pwr_hold >= 0);
        PCIE_DBG("%s(%d): pm_hold: incrementing hold \n",
            ddi_driver_name(dip), ddi_get_instance(dip));
        pwr_p->pwr_hold++;
        /* Mark itself busy, if it is not done already */
        if (!(pwr_p->pwr_flags & PCIE_PM_BUSY)) {
                PCIE_DBG("%s(%d): pm_hold: marking busy\n",
                    ddi_driver_name(dip), ddi_get_instance(dip));
                pwr_p->pwr_flags |= PCIE_PM_BUSY;
                (void) pm_busy_component(dip, 0);
        }
        if (pwr_p->pwr_func_lvl == PM_LEVEL_D0) {
                mutex_exit(&pwr_p->pwr_lock);
                return (DDI_SUCCESS);
        }
        mutex_exit(&pwr_p->pwr_lock);
        if (pm_raise_power(dip, 0, PM_LEVEL_D0) != DDI_SUCCESS) {
                PCIE_DBG("%s(%d): pm_hold: attempt to raise power "
                    "from %d to %d failed\n", ddi_driver_name(dip),
                    ddi_get_instance(dip), pwr_p->pwr_func_lvl,
                    PM_LEVEL_D0);
                pcie_pm_release(dip);
                return (DDI_FAILURE);
        }
        return (DDI_SUCCESS);
}

/*
 * Reverse the things done in pcie_pm_hold
 */
void
pcie_pm_release(dev_info_t *dip)
{
        pcie_pwr_t *pwr_p;

        /* If no PM info or no device PM, return */
        if (!PCIE_PMINFO(dip) || !(pwr_p = PCIE_NEXUS_PMINFO(dip)) ||
            !(PCIE_SUPPORTS_DEVICE_PM(dip)))
                return;

        mutex_enter(&pwr_p->pwr_lock);
        pcie_pm_subrelease(dip, pwr_p);
        mutex_exit(&pwr_p->pwr_lock);
}

static void
pcie_pm_subrelease(dev_info_t *dip, pcie_pwr_t *pwr_p)
{
        int level;

        ASSERT(MUTEX_HELD(&pwr_p->pwr_lock));
        ASSERT(pwr_p->pwr_hold > 0);
        PCIE_DBG("%s(%d): pm_subrelease: decrementing hold \n",
            ddi_driver_name(dip), ddi_get_instance(dip));
        pwr_p->pwr_hold--;
        ASSERT(pwr_p->pwr_hold >= 0);
        ASSERT(pwr_p->pwr_flags & PCIE_PM_BUSY);
        level = pwr_level_allowed(pwr_p);
        if (pwr_p->pwr_hold == 0 && level < pwr_p->pwr_func_lvl) {
                PCIE_DBG("%s(%d): pm_subrelease: marking idle \n",
                    ddi_driver_name(dip), ddi_get_instance(dip));
                (void) pm_idle_component(dip, 0);
                pwr_p->pwr_flags &= ~PCIE_PM_BUSY;
        }
}

/*
 * Called when the child makes the first power management call.
 * sets up the counters. All the components of the child device are
 * assumed to be at unknown level. It also releases the power hold
 *      pwr_p - parent's pwr_t
 *      cdip   - child's dip
 */
int
pcie_pm_add_child(dev_info_t *dip, dev_info_t *cdip)
{
        pcie_pwr_t *pwr_p;

        /* If no PM info, return */
        if (!PCIE_PMINFO(dip) || !(pwr_p = PCIE_NEXUS_PMINFO(dip)))
                return (DDI_SUCCESS);

        ASSERT(MUTEX_HELD(&pwr_p->pwr_lock));
        ASSERT(pwr_p->pwr_func_lvl == PM_LEVEL_D0);
        pcie_add_comps(dip, cdip, pwr_p);

        /* If no device power management then return */
        if (!PCIE_SUPPORTS_DEVICE_PM(dip))
                return (DDI_SUCCESS);

        /*
         * We have informed PM that we are busy at PRE_ATTACH time for
         * this child. Release the hold and but don't clear the busy bit.
         * If a device never changes power, hold will not be released
         * and we stay at full power.
         */
        ASSERT(pwr_p->pwr_hold > 0);
        PCIE_DBG("%s(%d): pm_add_child: decrementing hold \n",
            ddi_driver_name(dip), ddi_get_instance(dip));
        pwr_p->pwr_hold--;
        /*
         * We must have made sure that busy bit
         * is set when we put the hold
         */
        ASSERT(pwr_p->pwr_flags & PCIE_PM_BUSY);
        return (DDI_SUCCESS);
}

/*
 * Adjust the counters when a child detaches
 * Marks itself idle if the idle conditions are met.
 * Called at POST_DETACH time
 */
int
pcie_pm_remove_child(dev_info_t *dip, dev_info_t *cdip)
{
        int *counters;
        int total;
        pcie_pwr_t *pwr_p;

        /* If no PM info, return */
        if (!PCIE_PMINFO(dip) || !(pwr_p = PCIE_NEXUS_PMINFO(dip)))
                return (DDI_SUCCESS);

        counters = pwr_p->pwr_counters;
        mutex_enter(&pwr_p->pwr_lock);
        pcie_remove_comps(dip, cdip, pwr_p);
        /* If no device power management then return */
        if (!PCIE_SUPPORTS_DEVICE_PM(dip)) {
                mutex_exit(&pwr_p->pwr_lock);
                return (DDI_SUCCESS);
        }
        total = (counters[PCIE_D0_INDEX] + counters[PCIE_UNKNOWN_INDEX] +
            counters[PCIE_D1_INDEX] + counters[PCIE_D2_INDEX] +
            counters[PCIE_D3_INDEX]);
        /*
         * Mark idle if either there are no children or our lowest
         * possible level is less than the current level. Mark idle
         * only if it is not already done.
         */
        if ((pwr_p->pwr_hold == 0) &&
            (!total || (pwr_level_allowed(pwr_p) < pwr_p->pwr_func_lvl))) {
                if (pwr_p->pwr_flags & PCIE_PM_BUSY) {
                        PCIE_DBG("%s(%d): pcie_bus_power: marking idle\n",
                            ddi_driver_name(dip), ddi_get_instance(dip));
                        (void) pm_idle_component(dip, 0);
                        pwr_p->pwr_flags &= ~PCIE_PM_BUSY;
                }
        }
        mutex_exit(&pwr_p->pwr_lock);
        return (DDI_SUCCESS);
}

boolean_t
pcie_is_pcie(dev_info_t *dip)
{
        pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
        ASSERT(bus_p);
        return (bus_p->bus_pcie_off != 0);
}

/*
 * Called by px_attach or pcieb_attach:: DDI_RESUME
 */
int
pcie_pwr_resume(dev_info_t *dip)
{
        dev_info_t *cdip;
        pcie_pwr_t *pwr_p = NULL;

#if defined(__x86)
        if (dip)
                return (DDI_SUCCESS);
#endif /* defined(__x86) */

        if (PCIE_PMINFO(dip))
                pwr_p = PCIE_NEXUS_PMINFO(dip);

        if (pwr_p) {
                /* Inform the PM framework that dip is at full power */
                if (PCIE_SUPPORTS_DEVICE_PM(dip)) {
                        ASSERT(pwr_p->pwr_func_lvl == PM_LEVEL_D0);
                        (void) pm_raise_power(dip, 0,
                            pwr_p->pwr_func_lvl);
                }
        }

        /*
         * Code taken from pci driver.
         * Restore config registers for children that did not save
         * their own registers.  Children pwr states are UNKNOWN after
         * a resume since it is possible for the PM framework to call
         * resume without an actual power cycle. (ie if suspend fails).
         */
        for (cdip = ddi_get_child(dip); cdip != NULL;
            cdip = ddi_get_next_sibling(cdip)) {
                boolean_t       is_pcie;

                /*
                 * Not interested in children who are not already
                 * init'ed.  They will be set up by init_child().
                 */
                if (i_ddi_node_state(cdip) < DS_INITIALIZED) {
                        PCIE_DBG("%s(%d): "
                            "DDI_RESUME: skipping %s%d not in CF1\n",
                            ddi_driver_name(dip), ddi_get_instance(dip),
                            ddi_driver_name(cdip), ddi_get_instance(cdip));
                        continue;
                }

                /*
                 * Only restore config registers if saved by nexus.
                 */
                if (ddi_prop_exists(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
                    "nexus-saved-config-regs") != 1)
                        continue;

                PCIE_DBG("%s(%d): "
                    "DDI_RESUME: nexus restoring %s%d config regs\n",
                    ddi_driver_name(dip), ddi_get_instance(dip),
                    ddi_driver_name(cdip), ddi_get_instance(cdip));

                /* clear errors left by OBP scrubbing */
                pcie_clear_errors(cdip);

                /* PCIe workaround: disable errors during 4K config resore */
                is_pcie = pcie_is_pcie(cdip);
                if (is_pcie)
                        pcie_disable_errors(cdip);
                (void) pci_restore_config_regs(cdip);
                if (is_pcie) {
                        pcie_enable_errors(cdip);
                        (void) pcie_enable_ce(cdip);
                }

                if (ndi_prop_remove(DDI_DEV_T_NONE, cdip,
                    "nexus-saved-config-regs") != DDI_PROP_SUCCESS) {
                        PCIE_DBG("%s(%d): %s%d can't remove prop %s",
                            ddi_driver_name(dip), ddi_get_instance(dip),
                            ddi_driver_name(cdip), ddi_get_instance(cdip),
                            "nexus-saved-config-regs");
                }
        }
        return (DDI_SUCCESS);
}

/*
 * Called by pcie_detach or pcieb_detach:: DDI_SUSPEND
 */
int
pcie_pwr_suspend(dev_info_t *dip)
{
        dev_info_t *cdip;
        int i, *counters; /* per nexus counters */
        int *child_counters = NULL; /* per child dip counters */
        pcie_pwr_t *pwr_p = NULL;

#if defined(__x86)
        if (dip)
                return (DDI_SUCCESS);
#endif /* defined(__x86) */

        if (PCIE_PMINFO(dip))
                pwr_p = PCIE_NEXUS_PMINFO(dip);

        /*
         * Mark all children to be unknown and bring our power level
         * to full, if required. This is to avoid any panics while
         * accessing the child's config space.
         */
        if (pwr_p) {
                mutex_enter(&pwr_p->pwr_lock);
                if (PCIE_SUPPORTS_DEVICE_PM(dip) &&
                    pwr_p->pwr_func_lvl != PM_LEVEL_D0) {
                        mutex_exit(&pwr_p->pwr_lock);
                        if (pm_raise_power(dip, 0, PM_LEVEL_D0) !=
                            DDI_SUCCESS) {
                                PCIE_DBG("%s(%d): pwr_suspend: attempt "
                                    "to raise power from %d to %d "
                                    "failed\n", ddi_driver_name(dip),
                                    ddi_get_instance(dip), pwr_p->pwr_func_lvl,
                                    PM_LEVEL_D0);
                                return (DDI_FAILURE);
                        }
                        mutex_enter(&pwr_p->pwr_lock);
                }
                counters = pwr_p->pwr_counters;
                /*
                 * Update the nexus counters. At the resume time all
                 * components are considered to be at unknown level. Use the
                 * fact that counters for unknown level are at the end.
                 */
                for (i = 0; i < PCIE_UNKNOWN_INDEX; i++) {
                        counters[PCIE_UNKNOWN_INDEX] += counters[i];
                        counters[i] = 0;
                }
                mutex_exit(&pwr_p->pwr_lock);
        }

        /*
         * Code taken from pci driver.
         * Save the state of the configuration headers of child
         * nodes.
         */
        for (cdip = ddi_get_child(dip); cdip != NULL;
            cdip = ddi_get_next_sibling(cdip)) {
                boolean_t       is_pcie;

                /*
                 * Not interested in children who are not already
                 * init'ed.  They will be set up in init_child().
                 */
                if (i_ddi_node_state(cdip) < DS_INITIALIZED) {
                        PCIE_DBG("%s(%d): DDI_SUSPEND: skipping "
                            "%s%d not in CF1\n", ddi_driver_name(dip),
                            ddi_get_instance(dip), ddi_driver_name(cdip),
                            ddi_get_instance(cdip));
                        continue;
                }
                /*
                 * Update per child dip counters, if any. Counters
                 * will not exist if the child is not power manageable
                 * or if its power entry is never invoked.
                 */
                if (PCIE_PMINFO(cdip) && PCIE_PAR_PMINFO(cdip))
                        child_counters = PCIE_CHILD_COUNTERS(cdip);
                if (child_counters && pwr_p) {
                        mutex_enter(&pwr_p->pwr_lock);
                        for (i = 0; i < PCIE_UNKNOWN_INDEX; i++) {
                                child_counters[PCIE_UNKNOWN_INDEX] +=
                                    child_counters[i];
                                child_counters[i] = 0;
                        }
                        mutex_exit(&pwr_p->pwr_lock);
                }

                /*
                 * Only save config registers if not already saved by child.
                 */
                if (ddi_prop_exists(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
                    SAVED_CONFIG_REGS) == 1) {
                        continue;
                }

                /*
                 * The nexus needs to save config registers.  Create a property
                 * so it knows to restore on resume.
                 */
                if (ndi_prop_create_boolean(DDI_DEV_T_NONE, cdip,
                    "nexus-saved-config-regs") != DDI_PROP_SUCCESS) {
                        PCIE_DBG("%s(%d): %s%d can't update prop %s",
                            ddi_driver_name(dip), ddi_get_instance(dip),
                            ddi_driver_name(cdip), ddi_get_instance(cdip),
                            "nexus-saved-config-regs");
                }
                PCIE_DBG("%s(%d): DDI_SUSPEND: saving config space for"
                    " %s%d\n", ddi_driver_name(dip), ddi_get_instance(dip),
                    ddi_driver_name(cdip), ddi_get_instance(cdip));

                /* PCIe workaround: disable errors during 4K config save */
                is_pcie = pcie_is_pcie(cdip);
                if (is_pcie)
                        pcie_disable_errors(cdip);
                (void) pci_save_config_regs(cdip);
                if (is_pcie) {
                        pcie_enable_errors(cdip);
                        (void) pcie_enable_ce(cdip);
                }
        }
        return (DDI_SUCCESS);
}

#ifdef DEBUG
/*
 * Description of bus_power_op.
 */
typedef struct pcie_buspwr_desc {
        pm_bus_power_op_t pwr_op;
        char *pwr_desc;
} pcie_buspwr_desc_t;

static pcie_buspwr_desc_t pcie_buspwr_desc[] = {
        {BUS_POWER_CHILD_PWRCHG, "CHILD_PWRCHG"},
        {BUS_POWER_NEXUS_PWRUP, "NEXUS_PWRUP"},
        {BUS_POWER_PRE_NOTIFICATION, "PRE_NOTIFICATION"},
        {BUS_POWER_POST_NOTIFICATION, "POST_NOTIFICATION"},
        {BUS_POWER_HAS_CHANGED, "HAS_CHANGED"},
        {BUS_POWER_NOINVOL, "NOINVOL"},
        {-1, NULL}
};

/*
 * Returns description of the bus_power_op.
 */
static char *
pcie_decode_pwr_op(pm_bus_power_op_t op)
{
        pcie_buspwr_desc_t *descp = pcie_buspwr_desc;

        for (; descp->pwr_desc; descp++) {
                if (op == descp->pwr_op)
                        return (descp->pwr_desc);
        }
        return ("UNKNOWN OP");
}
#endif