/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2009, Intel Corporation.
 * All rights reserved.
 */

#include <sys/atomic.h>
#include <sys/cpuvar.h>
#include <sys/cpu.h>
#include <sys/cpu_event.h>
#include <sys/cmn_err.h>
#include <sys/ddi.h>
#include <sys/kmem.h>
#include <sys/kstat.h>
#include <sys/pci.h>
#include <sys/sunddi.h>
#include <sys/sunndi.h>
#include <sys/synch.h>
#include <sys/sysmacros.h>
#include <sys/fipe.h>
#include <vm/hat.h>

/* Current PM policy, configurable through /etc/system and fipe.conf. */
fipe_pm_policy_t fipe_pm_policy = FIPE_PM_POLICY_BALANCE;
int fipe_pm_throttle_level = 1;

/* Enable kstat support. */
#define FIPE_KSTAT_SUPPORT              1

/* Enable performance relative statistics. */
#define FIPE_KSTAT_DETAIL               1

/* Enable builtin IOAT driver if no IOAT driver is available. */
#define FIPE_IOAT_BUILTIN               0
#if defined(FIPE_IOAT_BUILTIN) && (FIPE_IOAT_BUILTIN == 0)
#undef  FIPE_IOAT_BUILTIN
#endif

#ifdef  FIPE_IOAT_BUILTIN
/* Use IOAT channel 3 to generate memory transactions. */
#define FIPE_IOAT_CHAN_CTRL             0x200
#define FIPE_IOAT_CHAN_STS_LO           0x204
#define FIPE_IOAT_CHAN_STS_HI           0x208
#define FIPE_IOAT_CHAN_ADDR_LO          0x20C
#define FIPE_IOAT_CHAN_ADDR_HI          0x210
#define FIPE_IOAT_CHAN_CMD              0x214
#define FIPE_IOAT_CHAN_ERR              0x228
#else   /* FIPE_IOAT_BUILTIN */
#include <sys/dcopy.h>
#endif  /* FIPE_IOAT_BUILTIN */

/* Memory controller relative PCI configuration constants. */
#define FIPE_MC_GBLACT                  0x60
#define FIPE_MC_THRTLOW                 0x64
#define FIPE_MC_THRTCTRL                0x67
#define FIPE_MC_THRTCTRL_HUNT           0x1

/* Hardware recommended values. */
#define FIPE_MC_MEMORY_OFFSET           1024
#define FIPE_MC_MEMORY_SIZE             128

/* Number of IOAT commands posted when entering idle. */
#define FIPE_IOAT_CMD_NUM               2

/* Resource allocation retry interval in microsecond. */
#define FIPE_IOAT_RETRY_INTERVAL        (15 * 1000 * 1000)

/* Statistics update interval in nanosecond. */
#define FIPE_STAT_INTERVAL              (10 * 1000 * 1000)

/* Configuration profile support. */
#define FIPE_PROFILE_FIELD(field)       (fipe_profile_curr->field)
#define FIPE_PROF_IDLE_COUNT            FIPE_PROFILE_FIELD(idle_count)
#define FIPE_PROF_BUSY_THRESHOLD        FIPE_PROFILE_FIELD(busy_threshold)
#define FIPE_PROF_INTR_THRESHOLD        FIPE_PROFILE_FIELD(intr_threshold)
#define FIPE_PROF_INTR_BUSY_THRESHOLD   FIPE_PROFILE_FIELD(intr_busy_threshold)
#define FIPE_PROF_INTR_BUSY_THROTTLE    FIPE_PROFILE_FIELD(intr_busy_throttle)

/* Priority assigned to FIPE memory power management driver on x86. */
#define CPU_IDLE_CB_PRIO_FIPE           (CPU_IDLE_CB_PRIO_LOW_BASE + 0x4000000)

/* Structure to support power management profile. */
#pragma align CPU_CACHE_COHERENCE_SIZE(fipe_profiles)
static struct fipe_profile {
        uint32_t                        idle_count;
        uint32_t                        busy_threshold;
        uint32_t                        intr_threshold;
        uint32_t                        intr_busy_threshold;
        uint32_t                        intr_busy_throttle;
} fipe_profiles[FIPE_PM_POLICY_MAX] = {
        { 0,    0,      0,      0,      0 },
        { 5,    30,     20,     50,     5 },
        { 10,   40,     40,     75,     4 },
        { 15,   50,     60,     100,    2 },
};

/* Structure to store memory controller relative data. */
#pragma align CPU_CACHE_COHERENCE_SIZE(fipe_mc_ctrl)
static struct fipe_mc_ctrl {
        ddi_acc_handle_t                mc_pci_hdl;
        unsigned char                   mc_thrtctrl;
        unsigned char                   mc_thrtlow;
        unsigned char                   mc_gblact;
        dev_info_t                      *mc_dip;
        boolean_t                       mc_initialized;
} fipe_mc_ctrl;

/* Structure to store IOAT relative information. */
#pragma align CPU_CACHE_COHERENCE_SIZE(fipe_ioat_ctrl)
static struct fipe_ioat_control {
        kmutex_t                        ioat_lock;
        boolean_t                       ioat_ready;
#ifdef  FIPE_IOAT_BUILTIN
        boolean_t                       ioat_reg_mapped;
        ddi_acc_handle_t                ioat_reg_handle;
        uint8_t                         *ioat_reg_addr;
        uint64_t                        ioat_cmd_physaddr;
#else   /* FIPE_IOAT_BUILTIN */
        dcopy_cmd_t                     ioat_cmds[FIPE_IOAT_CMD_NUM + 1];
        dcopy_handle_t                  ioat_handle;
#endif  /* FIPE_IOAT_BUILTIN */
        dev_info_t                      *ioat_dev_info;
        uint64_t                        ioat_buf_physaddr;
        char                            *ioat_buf_virtaddr;
        char                            *ioat_buf_start;
        size_t                          ioat_buf_size;
        timeout_id_t                    ioat_timerid;
        boolean_t                       ioat_failed;
        boolean_t                       ioat_cancel;
        boolean_t                       ioat_try_alloc;
} fipe_ioat_ctrl;

#pragma align CPU_CACHE_COHERENCE_SIZE(fipe_idle_ctrl)
static struct fipe_idle_ctrl {
        boolean_t                       idle_ready;
        cpu_idle_callback_handle_t      cb_handle;
        cpu_idle_prop_handle_t          prop_enter;
        cpu_idle_prop_handle_t          prop_exit;
        cpu_idle_prop_handle_t          prop_busy;
        cpu_idle_prop_handle_t          prop_idle;
        cpu_idle_prop_handle_t          prop_intr;

        /* Put here for cache efficiency, it should be in fipe_global_ctrl. */
        hrtime_t                        tick_interval;
} fipe_idle_ctrl;

/*
 * Global control structure.
 * Solaris idle thread has no reentrance issue, so it's enough to count CPUs
 * in idle state. Otherwise cpuset_t bitmap should be used to track idle CPUs.
 */
#pragma align CPU_CACHE_COHERENCE_SIZE(fipe_gbl_ctrl)
static struct fipe_global_ctrl {
        kmutex_t                        lock;
        boolean_t                       pm_enabled;
        volatile boolean_t              pm_active;
        volatile uint32_t               cpu_count;
        volatile uint64_t               io_waiters;
        hrtime_t                        enter_ts;
        hrtime_t                        time_in_pm;
        size_t                          state_size;
        char                            *state_buf;
#ifdef  FIPE_KSTAT_SUPPORT
        kstat_t                         *fipe_kstat;
#endif  /* FIPE_KSTAT_SUPPORT */
} fipe_gbl_ctrl;

#define FIPE_CPU_STATE_PAD              (128 - \
        2 * sizeof (boolean_t) -  4 * sizeof (hrtime_t) - \
        2 * sizeof (uint64_t) - 2 * sizeof (uint32_t))

/* Per-CPU status. */
#pragma pack(1)
typedef struct fipe_cpu_state {
        boolean_t                       cond_ready;
        boolean_t                       state_ready;
        uint32_t                        idle_count;
        uint32_t                        throttle_cnt;
        hrtime_t                        throttle_ts;
        hrtime_t                        next_ts;
        hrtime_t                        last_busy;
        hrtime_t                        last_idle;
        uint64_t                        last_intr;
        uint64_t                        last_iowait;
        char                            pad1[FIPE_CPU_STATE_PAD];
} fipe_cpu_state_t;
#pragma pack()

#ifdef  FIPE_KSTAT_SUPPORT
#pragma align CPU_CACHE_COHERENCE_SIZE(fipe_kstat)
static struct fipe_kstat_s {
        kstat_named_t           fipe_enabled;
        kstat_named_t           fipe_policy;
        kstat_named_t           fipe_pm_time;
#ifdef  FIPE_KSTAT_DETAIL
        kstat_named_t           ioat_ready;
        kstat_named_t           pm_tryenter_cnt;
        kstat_named_t           pm_success_cnt;
        kstat_named_t           pm_race_cnt;
        kstat_named_t           cpu_loop_cnt;
        kstat_named_t           cpu_busy_cnt;
        kstat_named_t           cpu_idle_cnt;
        kstat_named_t           cpu_intr_busy_cnt;
        kstat_named_t           cpu_intr_throttle_cnt;
        kstat_named_t           bio_busy_cnt;
        kstat_named_t           ioat_start_fail_cnt;
        kstat_named_t           ioat_stop_fail_cnt;
#endif  /* FIPE_KSTAT_DETAIL */
} fipe_kstat = {
        { "fipe_enabled",       KSTAT_DATA_INT32 },
        { "fipe_policy",        KSTAT_DATA_INT32 },
        { "fipe_pm_time",       KSTAT_DATA_UINT64 },
#ifdef  FIPE_KSTAT_DETAIL
        { "ioat_ready",         KSTAT_DATA_INT32 },
        { "pm_tryenter_cnt",    KSTAT_DATA_UINT64 },
        { "pm_success_cnt",     KSTAT_DATA_UINT64 },
        { "pm_race_cnt",        KSTAT_DATA_UINT64 },
        { "cpu_loop_cnt",       KSTAT_DATA_UINT64 },
        { "cpu_busy_cnt",       KSTAT_DATA_UINT64 },
        { "cpu_idle_cnt",       KSTAT_DATA_UINT64 },
        { "cpu_intr_busy_cnt",  KSTAT_DATA_UINT64 },
        { "cpu_intr_thrt_cnt",  KSTAT_DATA_UINT64 },
        { "bio_busy_cnt",       KSTAT_DATA_UINT64 },
        { "ioat_start_fail_cnt", KSTAT_DATA_UINT64 },
        { "ioat_stop_fail_cnt", KSTAT_DATA_UINT64 }
#endif  /* FIPE_KSTAT_DETAIL */
};

#define FIPE_KSTAT_INC(v)               \
        atomic_inc_64(&fipe_kstat.v.value.ui64)
#ifdef  FIPE_KSTAT_DETAIL
#define FIPE_KSTAT_DETAIL_INC(v)        \
        atomic_inc_64(&fipe_kstat.v.value.ui64)
#else   /* FIPE_KSTAT_DETAIL */
#define FIPE_KSTAT_DETAIL_INC(v)
#endif  /* FIPE_KSTAT_DETAIL */

#else   /* FIPE_KSTAT_SUPPORT */

#define FIPE_KSTAT_INC(v)
#define FIPE_KSTAT_DETAIL_INC(v)

#endif  /* FIPE_KSTAT_SUPPORT */

/* Save current power management profile during suspend/resume. */
static fipe_pm_policy_t fipe_pm_policy_saved = FIPE_PM_POLICY_BALANCE;
static fipe_cpu_state_t *fipe_cpu_states = NULL;

/*
 * There is no lock to protect fipe_profile_curr, so fipe_profile_curr
 * could change on threads in fipe_idle_enter.  This is not an issue,
 * as it always points to a valid profile, and though it might make
 * an incorrect choice for the new profile, it will still be a valid
 * selection, and would do the correct operation for the new profile on
 * next cpu_idle_enter cycle.  Since the selections would always be
 * valid for some profile, the overhead for the lock is not wasted.
 */
static struct fipe_profile *fipe_profile_curr = NULL;

static void fipe_idle_enter(void *arg, cpu_idle_callback_context_t ctx,
    cpu_idle_check_wakeup_t check_func, void* check_arg);
static void fipe_idle_exit(void* arg, cpu_idle_callback_context_t ctx,
    int flags);
static cpu_idle_callback_t fipe_idle_cb = {
        CPU_IDLE_CALLBACK_VER0,
        fipe_idle_enter,
        fipe_idle_exit,
};

/*
 * Configure memory controller into power saving mode:
 * 1) OLTT activation limit is set to unlimited
 * 2) MC works in S-CLTT mode
 */
static int
fipe_mc_change(int throttle)
{
        /* Enable OLTT/disable S-CLTT mode */
        pci_config_put8(fipe_mc_ctrl.mc_pci_hdl, FIPE_MC_THRTCTRL,
            fipe_mc_ctrl.mc_thrtctrl & ~FIPE_MC_THRTCTRL_HUNT);
        /* Set OLTT activation limit to unlimited */
        pci_config_put8(fipe_mc_ctrl.mc_pci_hdl, FIPE_MC_GBLACT, 0);
        /*
         * Set S-CLTT low throttling to desired value. The lower value,
         * the more power saving and the less available memory bandwidth.
         */
        pci_config_put8(fipe_mc_ctrl.mc_pci_hdl, FIPE_MC_THRTLOW, throttle);
        /* Enable S-CLTT/disable OLTT mode */
        pci_config_put8(fipe_mc_ctrl.mc_pci_hdl, FIPE_MC_THRTCTRL,
            fipe_mc_ctrl.mc_thrtctrl | FIPE_MC_THRTCTRL_HUNT);

        return (0);
}

/*
 * Restore memory controller's original configuration.
 */
static void
fipe_mc_restore(void)
{
        pci_config_put8(fipe_mc_ctrl.mc_pci_hdl, FIPE_MC_THRTCTRL,
            fipe_mc_ctrl.mc_thrtctrl & ~FIPE_MC_THRTCTRL_HUNT);
        pci_config_put8(fipe_mc_ctrl.mc_pci_hdl, FIPE_MC_GBLACT,
            fipe_mc_ctrl.mc_gblact);
        pci_config_put8(fipe_mc_ctrl.mc_pci_hdl, FIPE_MC_THRTLOW,
            fipe_mc_ctrl.mc_thrtlow);
        pci_config_put8(fipe_mc_ctrl.mc_pci_hdl, FIPE_MC_THRTCTRL,
            fipe_mc_ctrl.mc_thrtctrl);
}

/*
 * Initialize memory controller's data structure and status.
 */
static int
fipe_mc_init(dev_info_t *dip)
{
        ddi_acc_handle_t handle;

        bzero(&fipe_mc_ctrl, sizeof (fipe_mc_ctrl));

        /* Hold one reference count and will be released in fipe_mc_fini. */
        ndi_hold_devi(dip);

        /* Setup pci configuration handler. */
        if (pci_config_setup(dip, &handle) != DDI_SUCCESS) {
                cmn_err(CE_WARN,
                    "!fipe: failed to setup pcicfg handler in mc_init.");
                ndi_rele_devi(dip);
                return (-1);
        }

        /* Save original configuration. */
        fipe_mc_ctrl.mc_thrtctrl = pci_config_get8(handle, FIPE_MC_THRTCTRL);
        fipe_mc_ctrl.mc_thrtlow = pci_config_get8(handle, FIPE_MC_THRTLOW);
        fipe_mc_ctrl.mc_gblact = pci_config_get8(handle, FIPE_MC_GBLACT);
        fipe_mc_ctrl.mc_dip = dip;
        fipe_mc_ctrl.mc_pci_hdl = handle;
        fipe_mc_ctrl.mc_initialized = B_TRUE;

        return (0);
}

/*
 * Restore memory controller's configuration and release resources.
 */
static void
fipe_mc_fini(void)
{
        if (fipe_mc_ctrl.mc_initialized) {
                fipe_mc_restore();
                pci_config_teardown(&fipe_mc_ctrl.mc_pci_hdl);
                ndi_rele_devi(fipe_mc_ctrl.mc_dip);
                fipe_mc_ctrl.mc_initialized = B_FALSE;
        }
        bzero(&fipe_mc_ctrl, sizeof (fipe_mc_ctrl));
}

/* Search device with specific pci ids. */
struct fipe_pci_ioat_id {
        uint16_t                venid;
        uint16_t                devid;
        uint16_t                subvenid;
        uint16_t                subsysid;
        char                    *unitaddr;
};

static struct fipe_pci_ioat_id fipe_pci_ioat_ids[] = {
        { 0x8086, 0x1a38, 0xffff, 0xffff, NULL },
        { 0x8086, 0x360b, 0xffff, 0xffff, NULL },
};

/*ARGSUSED*/
static int
fipe_search_ioat_dev(dev_info_t *dip, void *arg)
{
        char *unit;
        struct fipe_pci_ioat_id *id;
        int i, max, venid, devid, subvenid, subsysid;

        /* Query PCI id properties. */
        venid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
            "vendor-id", 0xffffffff);
        if (venid == 0xffffffff) {
                return (DDI_WALK_CONTINUE);
        }
        devid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
            "device-id", 0xffffffff);
        if (devid == 0xffffffff) {
                return (DDI_WALK_CONTINUE);
        }
        subvenid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
            "subsystem-vendor-id", 0xffffffff);
        if (subvenid == 0xffffffff) {
                return (DDI_WALK_CONTINUE);
        }
        subsysid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
            "subsystem-id", 0xffffffff);
        if (subvenid == 0xffffffff) {
                return (DDI_WALK_CONTINUE);
        }
        if (ddi_prop_lookup_string(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
            "unit-address", &unit) != DDI_PROP_SUCCESS) {
                return (DDI_WALK_CONTINUE);
        }

        max = sizeof (fipe_pci_ioat_ids) / sizeof (fipe_pci_ioat_ids[0]);
        for (i = 0; i < max; i++) {
                id = &fipe_pci_ioat_ids[i];
                if ((id->venid == 0xffffu || id->venid == venid) &&
                    (id->devid == 0xffffu || id->devid == devid) &&
                    (id->subvenid == 0xffffu || id->subvenid == subvenid) &&
                    (id->subsysid == 0xffffu || id->subsysid == subsysid) &&
                    (id->unitaddr == NULL || strcmp(id->unitaddr, unit) == 0)) {
                        break;
                }
        }
        ddi_prop_free(unit);
        if (i >= max) {
                return (DDI_WALK_CONTINUE);
        }

        /* Found IOAT device, hold one reference count. */
        ndi_hold_devi(dip);
        fipe_ioat_ctrl.ioat_dev_info = dip;

        return (DDI_WALK_TERMINATE);
}

/*
 * To enable FBDIMM idle power enhancement mechanism, IOAT will be used to
 * generate enough memory traffic to trigger memory controller thermal throttle
 * circuitry.
 * If dcopy/ioat is available, we will use dcopy interface to communicate
 * with IOAT. Otherwise the built-in driver will directly talk to IOAT
 * hardware.
 */
#ifdef  FIPE_IOAT_BUILTIN
static int
fipe_ioat_trigger(void)
{
        uint16_t ctrl;
        uint32_t err;
        uint8_t *addr = fipe_ioat_ctrl.ioat_reg_addr;
        ddi_acc_handle_t handle = fipe_ioat_ctrl.ioat_reg_handle;

        /* Check channel in use flag. */
        ctrl = ddi_get16(handle, (uint16_t *)(addr + FIPE_IOAT_CHAN_CTRL));
        if (ctrl & 0x100) {
                /*
                 * Channel is in use by somebody else. IOAT driver may have
                 * been loaded, forbid fipe from accessing IOAT hardware
                 * anymore.
                 */
                fipe_ioat_ctrl.ioat_ready = B_FALSE;
                fipe_ioat_ctrl.ioat_failed = B_TRUE;
                FIPE_KSTAT_INC(ioat_start_fail_cnt);
                return (-1);
        } else {
                /* Set channel in use flag. */
                ddi_put16(handle,
                    (uint16_t *)(addr + FIPE_IOAT_CHAN_CTRL), 0x100);
        }

        /* Write command address. */
        ddi_put32(handle,
            (uint32_t *)(addr + FIPE_IOAT_CHAN_ADDR_LO),
            (uint32_t)fipe_ioat_ctrl.ioat_cmd_physaddr);
        ddi_put32(handle, (uint32_t *)(addr + FIPE_IOAT_CHAN_ADDR_HI),
            (uint32_t)(fipe_ioat_ctrl.ioat_cmd_physaddr >> 32));

        /* Check and clear error flags. */
        err = ddi_get32(handle, (uint32_t *)(addr + FIPE_IOAT_CHAN_ERR));
        if (err != 0) {
                ddi_put32(handle, (uint32_t *)(addr + FIPE_IOAT_CHAN_ERR), err);
        }

        /* Start channel. */
        ddi_put8(handle, (uint8_t *)(addr + FIPE_IOAT_CHAN_CMD), 0x1);

        return (0);
}

static void
fipe_ioat_cancel(void)
{
        uint32_t status;
        uint8_t *addr = fipe_ioat_ctrl.ioat_reg_addr;
        ddi_acc_handle_t handle = fipe_ioat_ctrl.ioat_reg_handle;

        /*
         * Reset channel. Sometimes reset is not reliable,
         * so check completion or abort status after reset.
         */
        /* LINTED: constant in conditional context */
        while (1) {
                /* Issue reset channel command. */
                ddi_put8(handle, (uint8_t *)(addr + FIPE_IOAT_CHAN_CMD), 0x20);

                /* Query command status. */
                status = ddi_get32(handle,
                    (uint32_t *)(addr + FIPE_IOAT_CHAN_STS_LO));
                if (status & 0x1) {
                        /* Reset channel completed. */
                        break;
                } else {
                        SMT_PAUSE();
                }
        }

        /* Put channel into "not in use" state. */
        ddi_put16(handle, (uint16_t *)(addr + FIPE_IOAT_CHAN_CTRL), 0);
}

/*ARGSUSED*/
static void
fipe_ioat_alloc(void *arg)
{
        int rc = 0, nregs;
        dev_info_t *dip;
        ddi_device_acc_attr_t attr;
        boolean_t fatal = B_FALSE;

        mutex_enter(&fipe_ioat_ctrl.ioat_lock);
        /*
         * fipe_ioat_alloc() is called in DEVICE ATTACH context when loaded.
         * In DEVICE ATTACH context, it can't call ddi_walk_devs(), so just
         * schedule a timer and exit.
         */
        if (fipe_ioat_ctrl.ioat_try_alloc == B_FALSE) {
                fipe_ioat_ctrl.ioat_try_alloc = B_TRUE;
                goto out_error;
        }

        /* Check whether has been initialized or encountered permanent error. */
        if (fipe_ioat_ctrl.ioat_ready || fipe_ioat_ctrl.ioat_failed ||
            fipe_ioat_ctrl.ioat_cancel) {
                fipe_ioat_ctrl.ioat_timerid = 0;
                mutex_exit(&fipe_ioat_ctrl.ioat_lock);
                return;
        }

        if (fipe_ioat_ctrl.ioat_dev_info == NULL) {
                /* Find dev_info_t for IOAT engine. */
                ddi_walk_devs(ddi_root_node(), fipe_search_ioat_dev, NULL);
                if (fipe_ioat_ctrl.ioat_dev_info == NULL) {
                        cmn_err(CE_NOTE,
                            "!fipe: no IOAT hardware found, disable pm.");
                        fatal = B_TRUE;
                        goto out_error;
                }
        }

        /* Map in IOAT control register window. */
        ASSERT(fipe_ioat_ctrl.ioat_dev_info != NULL);
        ASSERT(fipe_ioat_ctrl.ioat_reg_mapped == B_FALSE);
        dip = fipe_ioat_ctrl.ioat_dev_info;
        if (ddi_dev_nregs(dip, &nregs) != DDI_SUCCESS || nregs < 2) {
                cmn_err(CE_WARN, "!fipe: ioat has not enough register bars.");
                fatal = B_TRUE;
                goto out_error;
        }
        attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
        attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
        attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
        rc = ddi_regs_map_setup(dip, 1,
            (caddr_t *)&fipe_ioat_ctrl.ioat_reg_addr,
            0, 0, &attr, &fipe_ioat_ctrl.ioat_reg_handle);
        if (rc != DDI_SUCCESS) {
                cmn_err(CE_WARN, "!fipe: failed to map IOAT registeres.");
                fatal = B_TRUE;
                goto out_error;
        }

        /* Mark IOAT status. */
        fipe_ioat_ctrl.ioat_reg_mapped = B_TRUE;
        fipe_ioat_ctrl.ioat_ready = B_TRUE;
        fipe_ioat_ctrl.ioat_failed = B_FALSE;
        fipe_ioat_ctrl.ioat_timerid = 0;
        mutex_exit(&fipe_ioat_ctrl.ioat_lock);

        return;

out_error:
        fipe_ioat_ctrl.ioat_timerid = 0;
        if (!fipe_ioat_ctrl.ioat_ready && !fipe_ioat_ctrl.ioat_cancel) {
                if (fatal) {
                        /* Mark permanent error and give up. */
                        fipe_ioat_ctrl.ioat_failed = B_TRUE;
                        /* Release reference count hold by ddi_find_devinfo. */
                        if (fipe_ioat_ctrl.ioat_dev_info != NULL) {
                                ndi_rele_devi(fipe_ioat_ctrl.ioat_dev_info);
                                fipe_ioat_ctrl.ioat_dev_info = NULL;
                        }
                } else {
                        /*
                         * Schedule another timer to keep on trying.
                         * timeout() should always succeed, no need to check
                         * return.
                         */
                        fipe_ioat_ctrl.ioat_timerid = timeout(fipe_ioat_alloc,
                            NULL, drv_usectohz(FIPE_IOAT_RETRY_INTERVAL));
                }
        }
        mutex_exit(&fipe_ioat_ctrl.ioat_lock);
}

static void
fipe_ioat_free(void)
{
        mutex_enter(&fipe_ioat_ctrl.ioat_lock);
        /* Cancel timeout to avoid race condition. */
        if (fipe_ioat_ctrl.ioat_timerid != 0) {
                fipe_ioat_ctrl.ioat_cancel = B_TRUE;
                mutex_exit(&fipe_ioat_ctrl.ioat_lock);
                (void) untimeout(fipe_ioat_ctrl.ioat_timerid);
                mutex_enter(&fipe_ioat_ctrl.ioat_lock);
                fipe_ioat_ctrl.ioat_timerid = 0;
                fipe_ioat_ctrl.ioat_cancel = B_FALSE;
        }

        if (fipe_ioat_ctrl.ioat_reg_mapped) {
                ddi_regs_map_free(&fipe_ioat_ctrl.ioat_reg_handle);
                fipe_ioat_ctrl.ioat_reg_mapped = B_FALSE;
        }

        fipe_ioat_ctrl.ioat_ready = B_FALSE;
        mutex_exit(&fipe_ioat_ctrl.ioat_lock);
}

#else   /* FIPE_IOAT_BUILTIN */

/*
 * Trigger IOAT memory copy operation when entering power saving state.
 * A group of commands will be posted to IOAT driver and those commands
 * will be placed into an IOAT ring buffer.
 */
static int
fipe_ioat_trigger(void)
{
        int idx;
        dcopy_cmd_t *cmds = fipe_ioat_ctrl.ioat_cmds;

        for (idx = FIPE_IOAT_CMD_NUM; idx > 0; idx--) {
                if (dcopy_cmd_post(cmds[idx]) == DCOPY_SUCCESS) {
                        continue;
                } else {
                        /*
                         * Don't rollback on failure, it doesn't hurt much more
                         * than some small memory copy operations.
                         */
                        FIPE_KSTAT_DETAIL_INC(ioat_start_fail_cnt);
                        return (-1);
                }
        }

        return (0);
}

/*
 * Cancel the memory copy operations posted by fipe_ioat_trigger.
 * It's achieved by posting a new command which will break the ring
 * created by fipe_ioat_trigger. If it fails, the best way to recover
 * is to just let it go. IOAT will recover when posting next command
 * on the same channel.
 */
static void
fipe_ioat_cancel(void)
{
        if (dcopy_cmd_post(fipe_ioat_ctrl.ioat_cmds[0]) != DCOPY_SUCCESS) {
                FIPE_KSTAT_DETAIL_INC(ioat_stop_fail_cnt);
        }
}

/*
 * This function will be called from allocate IOAT resources.
 * Allocation may fail due to following reasons:
 * 1) IOAT driver hasn't been loaded yet. Keep on trying in this case.
 * 2) IOAT resources are temporarily unavailable.  Keep on trying in this case.
 * 3) Other no recoverable reasons. Disable power management function.
 */
/*ARGSUSED*/
static void
fipe_ioat_alloc(void *arg)
{
        int idx, flags, rc = 0;
        uint64_t physaddr;
        boolean_t fatal = B_FALSE;
        dcopy_query_t info;
        dcopy_handle_t handle;
        dcopy_cmd_t cmds[FIPE_IOAT_CMD_NUM + 1];

        mutex_enter(&fipe_ioat_ctrl.ioat_lock);
        /*
         * fipe_ioat_alloc() is called in DEVICE ATTACH context when loaded.
         * In DEVICE ATTACH context, it can't call ddi_walk_devs(), so just
         * schedule a timer and exit.
         */
        if (fipe_ioat_ctrl.ioat_try_alloc == B_FALSE) {
                fipe_ioat_ctrl.ioat_try_alloc = B_TRUE;
                mutex_exit(&fipe_ioat_ctrl.ioat_lock);
                goto out_error;
        }

        /*
         * Check whether device has been initialized or if it encountered
         * some permanent error.
         */
        if (fipe_ioat_ctrl.ioat_ready || fipe_ioat_ctrl.ioat_failed ||
            fipe_ioat_ctrl.ioat_cancel) {
                fipe_ioat_ctrl.ioat_timerid = 0;
                mutex_exit(&fipe_ioat_ctrl.ioat_lock);
                return;
        }

        if (fipe_ioat_ctrl.ioat_dev_info == NULL) {
                /* Find dev_info_t for IOAT engine. */
                ddi_walk_devs(ddi_root_node(), fipe_search_ioat_dev, NULL);
                if (fipe_ioat_ctrl.ioat_dev_info == NULL) {
                        cmn_err(CE_NOTE,
                            "!fipe: no IOAT hardware found, disable pm.");
                        mutex_exit(&fipe_ioat_ctrl.ioat_lock);
                        fatal = B_TRUE;
                        goto out_error;
                }
        }
        mutex_exit(&fipe_ioat_ctrl.ioat_lock);

        /* Check, allocate and initialize IOAT resources with lock released. */
        dcopy_query(&info);
        if (info.dq_version < DCOPY_QUERY_V0) {
                /* Permanent error, give up. */
                cmn_err(CE_WARN, "!fipe: IOAT driver version mismatch.");
                fatal = B_TRUE;
                goto out_error;
        } else if (info.dq_num_channels == 0) {
                /* IOAT driver hasn't been loaded, keep trying. */
                goto out_error;
        }

        /* Allocate IOAT channel. */
        rc = dcopy_alloc(DCOPY_NOSLEEP, &handle);
        if (rc == DCOPY_NORESOURCES) {
                /* Resource temporarily not available, keep trying. */
                goto out_error;
        } else if (rc != DCOPY_SUCCESS) {
                /* Permanent error, give up. */
                cmn_err(CE_WARN, "!fipe: failed to allocate IOAT channel.");
                fatal = B_TRUE;
                goto out_error;
        }

        /*
         * Allocate multiple IOAT commands and organize them into a ring to
         * loop forever. Commands number is determined by IOAT descriptor size
         * and memory interleave pattern.
         * cmd[0] is used break the loop and disable IOAT operation.
         * cmd[1, FIPE_IOAT_CMD_NUM] are grouped into a ring and cmd[1] is the
         * list head.
         */
        bzero(cmds, sizeof (cmds));
        physaddr = fipe_ioat_ctrl.ioat_buf_physaddr;
        for (idx = FIPE_IOAT_CMD_NUM; idx >= 0; idx--) {
                /* Allocate IOAT commands. */
                if (idx == 0 || idx == FIPE_IOAT_CMD_NUM) {
                        flags = DCOPY_NOSLEEP;
                } else {
                        /*
                         * To link commands into a list, the initial value of
                         * cmd need to be set to next cmd on list.
                         */
                        flags = DCOPY_NOSLEEP | DCOPY_ALLOC_LINK;
                        cmds[idx] = cmds[idx + 1];
                }
                rc = dcopy_cmd_alloc(handle, flags, &cmds[idx]);
                if (rc == DCOPY_NORESOURCES) {
                        goto out_freecmd;
                } else if (rc != DCOPY_SUCCESS) {
                        /* Permanent error, give up. */
                        cmn_err(CE_WARN,
                            "!fipe: failed to allocate IOAT command.");
                        fatal = B_TRUE;
                        goto out_freecmd;
                }

                /* Disable src/dst snoop to improve CPU cache efficiency. */
                cmds[idx]->dp_flags = DCOPY_CMD_NOSRCSNP | DCOPY_CMD_NODSTSNP;
                /* Specially handle commands on the list. */
                if (idx != 0) {
                        /* Disable IOAT status. */
                        cmds[idx]->dp_flags |= DCOPY_CMD_NOSTAT;
                        /* Disable waiting for resources. */
                        cmds[idx]->dp_flags |= DCOPY_CMD_NOWAIT;
                        if (idx == 1) {
                                /* The list head, chain command into loop. */
                                cmds[idx]->dp_flags |= DCOPY_CMD_LOOP;
                        } else {
                                /* Queue all other commands except head. */
                                cmds[idx]->dp_flags |= DCOPY_CMD_QUEUE;
                        }
                }
                cmds[idx]->dp_cmd = DCOPY_CMD_COPY;
                cmds[idx]->dp.copy.cc_source = physaddr;
                cmds[idx]->dp.copy.cc_dest = physaddr + FIPE_MC_MEMORY_OFFSET;
                if (idx == 0) {
                        /*
                         * Command 0 is used to cancel memory copy by breaking
                         * the ring created in fipe_ioat_trigger().
                         * For efficiency, use the smallest memory copy size.
                         */
                        cmds[idx]->dp.copy.cc_size = 1;
                } else {
                        cmds[idx]->dp.copy.cc_size = FIPE_MC_MEMORY_SIZE;
                }
        }

        /* Update IOAT control status if it hasn't been initialized yet. */
        mutex_enter(&fipe_ioat_ctrl.ioat_lock);
        if (!fipe_ioat_ctrl.ioat_ready && !fipe_ioat_ctrl.ioat_cancel) {
                fipe_ioat_ctrl.ioat_handle = handle;
                for (idx = 0; idx <= FIPE_IOAT_CMD_NUM; idx++) {
                        fipe_ioat_ctrl.ioat_cmds[idx] = cmds[idx];
                }
                fipe_ioat_ctrl.ioat_ready = B_TRUE;
                fipe_ioat_ctrl.ioat_failed = B_FALSE;
                fipe_ioat_ctrl.ioat_timerid = 0;
                mutex_exit(&fipe_ioat_ctrl.ioat_lock);
                return;
        }
        mutex_exit(&fipe_ioat_ctrl.ioat_lock);
        /* Initialized by another thread, fall through to free resources. */

out_freecmd:
        if (cmds[0] != NULL) {
                dcopy_cmd_free(&cmds[0]);
        }
        /* Only need to free head, dcopy will free all commands on the list. */
        for (idx = 1; idx <= FIPE_IOAT_CMD_NUM; idx++) {
                if (cmds[idx] != NULL) {
                        dcopy_cmd_free(&cmds[idx]);
                        break;
                }
        }
        dcopy_free(&handle);

out_error:
        mutex_enter(&fipe_ioat_ctrl.ioat_lock);
        fipe_ioat_ctrl.ioat_timerid = 0;
        if (!fipe_ioat_ctrl.ioat_ready && !fipe_ioat_ctrl.ioat_cancel) {
                if (fatal) {
                        /* Mark permanent error and give up. */
                        fipe_ioat_ctrl.ioat_failed = B_TRUE;
                        /* Release reference count hold by ddi_find_devinfo. */
                        if (fipe_ioat_ctrl.ioat_dev_info != NULL) {
                                ndi_rele_devi(fipe_ioat_ctrl.ioat_dev_info);
                                fipe_ioat_ctrl.ioat_dev_info = NULL;
                        }
                } else {
                        /*
                         * Schedule another timer to keep on trying.
                         * timeout() should always success, no need to check.
                         */
                        fipe_ioat_ctrl.ioat_timerid = timeout(fipe_ioat_alloc,
                            NULL, drv_usectohz(FIPE_IOAT_RETRY_INTERVAL));
                }
        }
        mutex_exit(&fipe_ioat_ctrl.ioat_lock);
}

/*
 * Free resources allocated in fipe_ioat_alloc.
 */
static void
fipe_ioat_free(void)
{
        int idx = 0;
        dcopy_cmd_t *cmds = fipe_ioat_ctrl.ioat_cmds;

        mutex_enter(&fipe_ioat_ctrl.ioat_lock);

        /* Cancel timeout to avoid race condition. */
        if (fipe_ioat_ctrl.ioat_timerid != 0) {
                fipe_ioat_ctrl.ioat_cancel = B_TRUE;
                mutex_exit(&fipe_ioat_ctrl.ioat_lock);
                (void) untimeout(fipe_ioat_ctrl.ioat_timerid);
                mutex_enter(&fipe_ioat_ctrl.ioat_lock);
                fipe_ioat_ctrl.ioat_timerid = 0;
                fipe_ioat_ctrl.ioat_cancel = B_FALSE;
        }

        /* Free ioat resources. */
        if (fipe_ioat_ctrl.ioat_ready) {
                if (cmds[0] != NULL) {
                        dcopy_cmd_free(&cmds[0]);
                }
                for (idx = 1; idx <= FIPE_IOAT_CMD_NUM; idx++) {
                        if (cmds[idx] != NULL) {
                                dcopy_cmd_free(&cmds[idx]);
                                break;
                        }
                }
                bzero(fipe_ioat_ctrl.ioat_cmds,
                    sizeof (fipe_ioat_ctrl.ioat_cmds));
                dcopy_free(&fipe_ioat_ctrl.ioat_handle);
                fipe_ioat_ctrl.ioat_handle = NULL;
                fipe_ioat_ctrl.ioat_ready = B_FALSE;
        }

        /* Release reference count hold by ddi_find_devinfo. */
        if (fipe_ioat_ctrl.ioat_dev_info != NULL) {
                ndi_rele_devi(fipe_ioat_ctrl.ioat_dev_info);
                fipe_ioat_ctrl.ioat_dev_info = NULL;
        }

        mutex_exit(&fipe_ioat_ctrl.ioat_lock);
}
#endif  /* FIPE_IOAT_BUILTIN */

/*
 * Initialize IOAT relative resources.
 */
static int
fipe_ioat_init(void)
{
        char *buf;
        size_t size;

        bzero(&fipe_ioat_ctrl, sizeof (fipe_ioat_ctrl));
        mutex_init(&fipe_ioat_ctrl.ioat_lock, NULL, MUTEX_DRIVER, NULL);

        /*
         * Allocate memory for IOAT memory copy operation.
         * The allocated memory should be page aligned to achieve better power
         * savings.
         * Don't use ddi_dma_mem_alloc here to keep thing simple.  This also
         * makes quiesce easier.
         */
        size = PAGESIZE;
        buf = kmem_zalloc(size, KM_SLEEP);
        if ((intptr_t)buf & PAGEOFFSET) {
                kmem_free(buf, PAGESIZE);
                size <<= 1;
                buf = kmem_zalloc(size, KM_SLEEP);
        }
        fipe_ioat_ctrl.ioat_buf_size = size;
        fipe_ioat_ctrl.ioat_buf_start = buf;
        buf = (char *)P2ROUNDUP((intptr_t)buf, PAGESIZE);
        fipe_ioat_ctrl.ioat_buf_virtaddr = buf;
        fipe_ioat_ctrl.ioat_buf_physaddr = hat_getpfnum(kas.a_hat, buf);
        fipe_ioat_ctrl.ioat_buf_physaddr <<= PAGESHIFT;

#ifdef  FIPE_IOAT_BUILTIN
        {
                uint64_t bufpa;
                /* IOAT descriptor data structure copied from ioat.h. */
                struct fipe_ioat_cmd_desc {
                        uint32_t        dd_size;
                        uint32_t        dd_ctrl;
                        uint64_t        dd_src_paddr;
                        uint64_t        dd_dest_paddr;
                        uint64_t        dd_next_desc;
                        uint64_t        dd_res4;
                        uint64_t        dd_res5;
                        uint64_t        dd_res6;
                        uint64_t        dd_res7;
                } *desc;

                /*
                 * Build two IOAT command descriptors and chain them into ring.
                 * Control flags as below:
                 *      0x2: disable source snoop
                 *      0x4: disable destination snoop
                 *      0x0 << 24: memory copy operation
                 * The layout for command descriptors and memory buffers are
                 * organized for power saving effect, please don't change it.
                 */
                buf = fipe_ioat_ctrl.ioat_buf_virtaddr;
                bufpa = fipe_ioat_ctrl.ioat_buf_physaddr;
                fipe_ioat_ctrl.ioat_cmd_physaddr = bufpa;

                /* First command descriptor. */
                desc = (struct fipe_ioat_cmd_desc *)(buf);
                desc->dd_size = 128;
                desc->dd_ctrl = 0x6;
                desc->dd_src_paddr = bufpa + 2048;
                desc->dd_dest_paddr = bufpa + 3072;
                /* Point to second descriptor. */
                desc->dd_next_desc = bufpa + 64;

                /* Second command descriptor. */
                desc = (struct fipe_ioat_cmd_desc *)(buf + 64);
                desc->dd_size = 128;
                desc->dd_ctrl = 0x6;
                desc->dd_src_paddr = bufpa + 2048;
                desc->dd_dest_paddr = bufpa + 3072;
                /* Point to first descriptor. */
                desc->dd_next_desc = bufpa;
        }
#endif  /* FIPE_IOAT_BUILTIN */

        return (0);
}

static void
fipe_ioat_fini(void)
{
        /* Release reference count hold by ddi_find_devinfo. */
        if (fipe_ioat_ctrl.ioat_dev_info != NULL) {
                ndi_rele_devi(fipe_ioat_ctrl.ioat_dev_info);
                fipe_ioat_ctrl.ioat_dev_info = NULL;
        }

        if (fipe_ioat_ctrl.ioat_buf_start != NULL) {
                ASSERT(fipe_ioat_ctrl.ioat_buf_size != 0);
                kmem_free(fipe_ioat_ctrl.ioat_buf_start,
                    fipe_ioat_ctrl.ioat_buf_size);
        }

        mutex_destroy(&fipe_ioat_ctrl.ioat_lock);
        bzero(&fipe_ioat_ctrl, sizeof (fipe_ioat_ctrl));
}

static int
fipe_idle_start(void)
{
        int rc;

        if (fipe_idle_ctrl.idle_ready) {
                return (0);
        }

        if (cpu_idle_prop_create_handle(CPU_IDLE_PROP_ENTER_TIMESTAMP,
            &fipe_idle_ctrl.prop_enter) != 0) {
                cmn_err(CE_WARN, "!fipe: failed to get enter_ts property.");
                return (-1);
        }
        if (cpu_idle_prop_create_handle(CPU_IDLE_PROP_EXIT_TIMESTAMP,
            &fipe_idle_ctrl.prop_exit) != 0) {
                cmn_err(CE_WARN, "!fipe: failed to get exit_ts property.");
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_enter);
                return (-1);
        }
        if (cpu_idle_prop_create_handle(CPU_IDLE_PROP_TOTAL_IDLE_TIME,
            &fipe_idle_ctrl.prop_idle) != 0) {
                cmn_err(CE_WARN, "!fipe: failed to get idle_time property.");
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_exit);
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_enter);
                return (-1);
        }
        if (cpu_idle_prop_create_handle(CPU_IDLE_PROP_TOTAL_BUSY_TIME,
            &fipe_idle_ctrl.prop_busy) != 0) {
                cmn_err(CE_WARN, "!fipe: failed to get busy_time property.");
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_idle);
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_exit);
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_enter);
                return (-1);
        }
        if (cpu_idle_prop_create_handle(CPU_IDLE_PROP_INTERRUPT_COUNT,
            &fipe_idle_ctrl.prop_intr) != 0) {
                cmn_err(CE_WARN, "!fipe: failed to get intr_count property.");
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_busy);
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_idle);
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_exit);
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_enter);
                return (-1);
        }

        /* Register idle state notification callback. */
        rc = cpu_idle_register_callback(CPU_IDLE_CB_PRIO_FIPE, &fipe_idle_cb,
            NULL, &fipe_idle_ctrl.cb_handle);
        if (rc != 0) {
                cmn_err(CE_WARN, "!fipe: failed to register cpuidle callback.");
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_intr);
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_busy);
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_idle);
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_exit);
                (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_enter);
                return (-1);
        }

        fipe_idle_ctrl.idle_ready = B_TRUE;

        return (0);
}

static int
fipe_idle_stop(void)
{
        int rc;

        if (fipe_idle_ctrl.idle_ready == B_FALSE) {
                return (0);
        }

        rc = cpu_idle_unregister_callback(fipe_idle_ctrl.cb_handle);
        if (rc != 0) {
                cmn_err(CE_WARN,
                    "!fipe: failed to unregister cpuidle callback.");
                return (-1);
        }

        (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_intr);
        (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_busy);
        (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_idle);
        (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_exit);
        (void) cpu_idle_prop_destroy_handle(fipe_idle_ctrl.prop_enter);

        fipe_idle_ctrl.idle_ready = B_FALSE;

        return (0);
}

#ifdef  FIPE_KSTAT_SUPPORT
static int
fipe_kstat_update(kstat_t *ksp, int rw)
{
        struct fipe_kstat_s *sp;
        hrtime_t hrt;

        if (rw == KSTAT_WRITE) {
                return (EACCES);
        }

        sp = ksp->ks_data;
        sp->fipe_enabled.value.i32 = fipe_gbl_ctrl.pm_enabled ? 1 : 0;
        sp->fipe_policy.value.i32 = fipe_pm_policy;

        hrt = fipe_gbl_ctrl.time_in_pm;
        scalehrtime(&hrt);
        sp->fipe_pm_time.value.ui64 = (uint64_t)hrt;

#ifdef  FIPE_KSTAT_DETAIL
        sp->ioat_ready.value.i32 = fipe_ioat_ctrl.ioat_ready ? 1 : 0;
#endif  /* FIPE_KSTAT_DETAIL */

        return (0);
}
#endif  /* FIPE_KSTAT_SUPPORT */

/*
 * Initialize memory power management subsystem.
 * Note: This function should only be called from ATTACH.
 * Note: caller must ensure exclusive access to all fipe_xxx interfaces.
 */
int
fipe_init(dev_info_t *dip)
{
        size_t nsize;
        hrtime_t hrt;

        /* Initialize global control structure. */
        bzero(&fipe_gbl_ctrl, sizeof (fipe_gbl_ctrl));
        mutex_init(&fipe_gbl_ctrl.lock, NULL, MUTEX_DRIVER, NULL);

        /* Query power management policy from device property. */
        fipe_pm_policy = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
            FIPE_PROP_PM_POLICY, fipe_pm_policy);
        if (fipe_pm_policy < 0 || fipe_pm_policy >= FIPE_PM_POLICY_MAX) {
                cmn_err(CE_CONT,
                    "?fipe: invalid power management policy %d.\n",
                    fipe_pm_policy);
                fipe_pm_policy = FIPE_PM_POLICY_BALANCE;
        }
        fipe_profile_curr = &fipe_profiles[fipe_pm_policy];

        /*
         * Compute unscaled hrtime value corresponding to FIPE_STAT_INTERVAL.
         * (1 << 36) should be big enough here.
         */
        hrt = 1ULL << 36;
        scalehrtime(&hrt);
        fipe_idle_ctrl.tick_interval = FIPE_STAT_INTERVAL * (1ULL << 36) / hrt;

        if (fipe_mc_init(dip) != 0) {
                cmn_err(CE_WARN, "!fipe: failed to initialize mc state.");
                goto out_mc_error;
        }
        if (fipe_ioat_init() != 0) {
                cmn_err(CE_NOTE, "!fipe: failed to initialize ioat state.");
                goto out_ioat_error;
        }

        /* Allocate per-CPU structure. */
        nsize = max_ncpus * sizeof (fipe_cpu_state_t);
        nsize += CPU_CACHE_COHERENCE_SIZE;
        fipe_gbl_ctrl.state_buf = kmem_zalloc(nsize, KM_SLEEP);
        fipe_gbl_ctrl.state_size = nsize;
        fipe_cpu_states = (fipe_cpu_state_t *)P2ROUNDUP(
            (intptr_t)fipe_gbl_ctrl.state_buf, CPU_CACHE_COHERENCE_SIZE);

#ifdef  FIPE_KSTAT_SUPPORT
        fipe_gbl_ctrl.fipe_kstat = kstat_create("fipe", 0, "fipe-pm", "misc",
            KSTAT_TYPE_NAMED, sizeof (fipe_kstat) / sizeof (kstat_named_t),
            KSTAT_FLAG_VIRTUAL);
        if (fipe_gbl_ctrl.fipe_kstat == NULL) {
                cmn_err(CE_CONT, "?fipe: failed to create kstat object.\n");
        } else {
                fipe_gbl_ctrl.fipe_kstat->ks_lock = &fipe_gbl_ctrl.lock;
                fipe_gbl_ctrl.fipe_kstat->ks_data = &fipe_kstat;
                fipe_gbl_ctrl.fipe_kstat->ks_update = fipe_kstat_update;
                kstat_install(fipe_gbl_ctrl.fipe_kstat);
        }
#endif  /* FIPE_KSTAT_SUPPORT */

        return (0);

out_ioat_error:
        fipe_mc_fini();
out_mc_error:
        mutex_destroy(&fipe_gbl_ctrl.lock);
        bzero(&fipe_gbl_ctrl, sizeof (fipe_gbl_ctrl));

        return (-1);
}

/*
 * Destroy memory power management subsystem.
 * Note: This function should only be called from DETACH.
 * Note: caller must ensure exclusive access to all fipe_xxx interfaces.
 */
int
fipe_fini(void)
{
        if (fipe_gbl_ctrl.pm_enabled) {
                cmn_err(CE_NOTE, "!fipe: call fipe_fini without stopping PM.");
                return (EBUSY);
        }

        ASSERT(!fipe_gbl_ctrl.pm_active);
        fipe_ioat_fini();
        fipe_mc_fini();

#ifdef  FIPE_KSTAT_SUPPORT
        if (fipe_gbl_ctrl.fipe_kstat != NULL) {
                kstat_delete(fipe_gbl_ctrl.fipe_kstat);
                fipe_gbl_ctrl.fipe_kstat = NULL;
        }
#endif  /* FIPE_KSTAT_SUPPORT */

        if (fipe_gbl_ctrl.state_buf != NULL) {
                ASSERT(fipe_gbl_ctrl.state_size != 0);
                kmem_free(fipe_gbl_ctrl.state_buf, fipe_gbl_ctrl.state_size);
                fipe_cpu_states = NULL;
        }

        fipe_profile_curr = NULL;
        mutex_destroy(&fipe_gbl_ctrl.lock);
        bzero(&fipe_gbl_ctrl, sizeof (fipe_gbl_ctrl));

        return (0);
}

/*
 * Start memory power management subsystem.
 * Note: caller must ensure exclusive access to all fipe_xxx interfaces.
 */
int
fipe_start(void)
{
        if (fipe_gbl_ctrl.pm_enabled == B_TRUE) {
                return (0);
        }

        bzero(fipe_cpu_states, max_ncpus * sizeof (fipe_cpu_states[0]));
        fipe_ioat_alloc(NULL);
        if (fipe_idle_start() != 0) {
                cmn_err(CE_NOTE, "!fipe: failed to start PM subsystem.");
                fipe_ioat_free();
                return (-1);
        }

        fipe_gbl_ctrl.pm_enabled = B_TRUE;

        return (0);
}

/*
 * Stop memory power management subsystem.
 * Note: caller must ensure exclusive access to all fipe_xxx interfaces.
 */
int
fipe_stop(void)
{
        if (fipe_gbl_ctrl.pm_enabled) {
                if (fipe_idle_stop() != 0) {
                        cmn_err(CE_NOTE,
                            "!fipe: failed to stop PM subsystem.");
                        return (-1);
                }
                fipe_ioat_free();
                fipe_gbl_ctrl.pm_enabled = B_FALSE;
        }
        ASSERT(!fipe_gbl_ctrl.pm_active);

        return (0);
}

int
fipe_suspend(void)
{
        /* Save current power management policy. */
        fipe_pm_policy_saved = fipe_pm_policy;
        /* Disable PM by setting profile to FIPE_PM_POLICY_DISABLE. */
        fipe_pm_policy = FIPE_PM_POLICY_DISABLE;
        fipe_profile_curr = &fipe_profiles[fipe_pm_policy];

        return (0);
}

int
fipe_resume(void)
{
        /* Restore saved power management policy. */
        fipe_pm_policy = fipe_pm_policy_saved;
        fipe_profile_curr = &fipe_profiles[fipe_pm_policy];

        return (0);
}

fipe_pm_policy_t
fipe_get_pmpolicy(void)
{
        return (fipe_pm_policy);
}

int
fipe_set_pmpolicy(fipe_pm_policy_t policy)
{
        if (policy < 0 || policy >= FIPE_PM_POLICY_MAX) {
                return (EINVAL);
        }
        fipe_pm_policy = policy;
        fipe_profile_curr = &fipe_profiles[fipe_pm_policy];

        return (0);
}

/*
 * Check condition (fipe_gbl_ctrl.cpu_cnt == ncpus) to make sure that
 * there is other CPU trying to wake up system from memory power saving state.
 * If a CPU is waking up system, fipe_disable() will set
 * fipe_gbl_ctrl.pm_active to false as soon as possible and allow other CPU's
 * to continue, and it will take the responsibility to recover system from
 * memory power saving state.
 */
static void
fipe_enable(int throttle, cpu_idle_check_wakeup_t check_func, void* check_arg)
{
        extern void membar_sync(void);

        FIPE_KSTAT_DETAIL_INC(pm_tryenter_cnt);

        /*
         * Check CPU wakeup events.
         */
        if (check_func != NULL) {
                (*check_func)(check_arg);
        }

        /*
         * Try to acquire mutex, which also implicitly has the same effect
         * of calling membar_sync().
         * If mutex_tryenter fails, that means other CPU is waking up.
         */
        if (mutex_tryenter(&fipe_gbl_ctrl.lock) == 0) {
                FIPE_KSTAT_DETAIL_INC(pm_race_cnt);
        /*
         * Handle a special race condition for the case that a CPU wakes
         * and then enters into idle state within a short period.
         * This case can't be reliably detected by cpu_count mechanism.
         */
        } else if (fipe_gbl_ctrl.pm_active) {
                FIPE_KSTAT_DETAIL_INC(pm_race_cnt);
                mutex_exit(&fipe_gbl_ctrl.lock);
        } else {
                fipe_gbl_ctrl.pm_active = B_TRUE;
                membar_sync();
                if (fipe_gbl_ctrl.cpu_count != ncpus) {
                        FIPE_KSTAT_DETAIL_INC(pm_race_cnt);
                        fipe_gbl_ctrl.pm_active = B_FALSE;
                } else if (fipe_ioat_trigger() != 0) {
                        fipe_gbl_ctrl.pm_active = B_FALSE;
                } else if (fipe_gbl_ctrl.cpu_count != ncpus ||
                    fipe_mc_change(throttle) != 0) {
                        fipe_gbl_ctrl.pm_active = B_FALSE;
                        fipe_ioat_cancel();
                        if (fipe_gbl_ctrl.cpu_count != ncpus) {
                                FIPE_KSTAT_DETAIL_INC(pm_race_cnt);
                        }
                } else if (fipe_gbl_ctrl.cpu_count != ncpus) {
                        fipe_gbl_ctrl.pm_active = B_FALSE;
                        fipe_mc_restore();
                        fipe_ioat_cancel();
                        FIPE_KSTAT_DETAIL_INC(pm_race_cnt);
                } else {
                        FIPE_KSTAT_DETAIL_INC(pm_success_cnt);
                }
                mutex_exit(&fipe_gbl_ctrl.lock);
        }
}

static void
fipe_disable(void)
{
        /*
         * Try to acquire lock, which also implicitly has the same effect
         * of calling membar_sync().
         */
        while (mutex_tryenter(&fipe_gbl_ctrl.lock) == 0) {
                /*
                 * If power saving is inactive, just return and all dirty
                 * house-keeping work will be handled in fipe_enable().
                 */
                if (fipe_gbl_ctrl.pm_active == B_FALSE) {
                        return;
                } else {
                        (void) SMT_PAUSE();
                }
        }

        /* Disable power saving if it's active. */
        if (fipe_gbl_ctrl.pm_active) {
                /*
                 * Set pm_active to FALSE as soon as possible to prevent
                 * other CPUs from waiting on pm_active flag.
                 */
                fipe_gbl_ctrl.pm_active = B_FALSE;
                membar_producer();
                fipe_mc_restore();
                fipe_ioat_cancel();
        }

        mutex_exit(&fipe_gbl_ctrl.lock);
}

/*ARGSUSED*/
static boolean_t
fipe_check_cpu(struct fipe_cpu_state *sp, cpu_idle_callback_context_t ctx,
    hrtime_t ts)
{
        if (cpu_flagged_offline(CPU->cpu_flags)) {
                /* Treat CPU in offline state as ready. */
                sp->cond_ready = B_TRUE;
                return (B_TRUE);
        } else if (sp->next_ts <= ts) {
                uint64_t intr;
                hrtime_t idle, busy, diff;
                cpu_idle_prop_value_t val;

                /* Set default value. */
                sp->cond_ready = B_TRUE;
                sp->idle_count = 0;

                /* Calculate idle percent. */
                idle = sp->last_idle;
                sp->last_idle = cpu_idle_prop_get_hrtime(
                    fipe_idle_ctrl.prop_idle, ctx);
                idle = sp->last_idle - idle;
                busy = sp->last_busy;
                sp->last_busy = cpu_idle_prop_get_hrtime(
                    fipe_idle_ctrl.prop_busy, ctx);
                busy = sp->last_busy - busy;
                /* Check idle condition. */
                if (idle > 0 && busy > 0) {
                        if (busy * (100 - FIPE_PROF_BUSY_THRESHOLD) >
                            idle * FIPE_PROF_BUSY_THRESHOLD) {
                                FIPE_KSTAT_DETAIL_INC(cpu_busy_cnt);
                                sp->cond_ready = B_FALSE;
                        } else {
                                FIPE_KSTAT_DETAIL_INC(cpu_idle_cnt);
                        }
                } else {
                        FIPE_KSTAT_DETAIL_INC(cpu_busy_cnt);
                        sp->cond_ready = B_FALSE;
                }

                /* Calculate interrupt count. */
                diff = sp->next_ts;
                sp->next_ts = ts + fipe_idle_ctrl.tick_interval;
                diff = sp->next_ts - diff;
                intr = sp->last_intr;
                if (cpu_idle_prop_get_value(fipe_idle_ctrl.prop_intr, ctx,
                    &val) == 0) {
                        sp->last_intr = val.cipv_uint64;
                        intr = sp->last_intr - intr;
                        if (diff != 0) {
                                intr = intr * fipe_idle_ctrl.tick_interval;
                                intr /= diff;
                        } else {
                                intr = FIPE_PROF_INTR_THRESHOLD;
                        }
                } else {
                        intr = FIPE_PROF_INTR_THRESHOLD;
                }

                /*
                 * System is busy with interrupts, so disable all PM
                 * status checks for INTR_BUSY_THROTTLE ticks.
                 * Interrupts are disabled when FIPE callbacks are called,
                 * so this optimization will help to reduce interrupt
                 * latency.
                 */
                if (intr >= FIPE_PROF_INTR_BUSY_THRESHOLD) {
                        FIPE_KSTAT_DETAIL_INC(cpu_intr_busy_cnt);
                        sp->throttle_ts = ts + FIPE_PROF_INTR_BUSY_THROTTLE *
                            fipe_idle_ctrl.tick_interval;
                        sp->cond_ready = B_FALSE;
                } else if (intr >= FIPE_PROF_INTR_THRESHOLD) {
                        FIPE_KSTAT_DETAIL_INC(cpu_intr_throttle_cnt);
                        sp->cond_ready = B_FALSE;
                }
        } else if (++sp->idle_count >= FIPE_PROF_IDLE_COUNT) {
                /* Too many idle enter/exit in this tick. */
                FIPE_KSTAT_DETAIL_INC(cpu_loop_cnt);
                sp->throttle_ts = sp->next_ts + fipe_idle_ctrl.tick_interval;
                sp->idle_count = 0;
                sp->cond_ready = B_FALSE;
                return (B_FALSE);
        }

        return (sp->cond_ready);
}

/*ARGSUSED*/
static void
fipe_idle_enter(void *arg, cpu_idle_callback_context_t ctx,
    cpu_idle_check_wakeup_t check_func, void* check_arg)
{
        hrtime_t ts;
        uint32_t cnt;
        uint64_t iowait;
        cpu_t *cp = CPU;
        struct fipe_cpu_state *sp;

        sp = &fipe_cpu_states[cp->cpu_id];
        ts = cpu_idle_prop_get_hrtime(fipe_idle_ctrl.prop_enter, ctx);

        if (fipe_pm_policy != FIPE_PM_POLICY_DISABLE &&
            fipe_ioat_ctrl.ioat_ready &&
            sp->state_ready && sp->throttle_ts <= ts) {
                /* Adjust iowait count for local CPU. */
                iowait = CPU_STATS(cp, sys.iowait);
                if (iowait != sp->last_iowait) {
                        atomic_add_64(&fipe_gbl_ctrl.io_waiters,
                            iowait - sp->last_iowait);
                        sp->last_iowait = iowait;
                }

                /* Check current CPU status. */
                if (fipe_check_cpu(sp, ctx, ts)) {
                        /* Increase count of CPU ready for power saving. */
                        do {
                                cnt = fipe_gbl_ctrl.cpu_count;
                                ASSERT(cnt < ncpus);
                        } while (atomic_cas_32(&fipe_gbl_ctrl.cpu_count,
                            cnt, cnt + 1) != cnt);

                        /*
                         * Enable power saving if all CPUs are idle.
                         */
                        if (cnt + 1 == ncpus) {
                                if (fipe_gbl_ctrl.io_waiters == 0) {
                                        fipe_gbl_ctrl.enter_ts = ts;
                                        fipe_enable(fipe_pm_throttle_level,
                                            check_func, check_arg);
                                /* There are ongoing block io operations. */
                                } else {
                                        FIPE_KSTAT_DETAIL_INC(bio_busy_cnt);
                                }
                        }
                }
        } else if (fipe_pm_policy == FIPE_PM_POLICY_DISABLE ||
            fipe_ioat_ctrl.ioat_ready == B_FALSE) {
                if (sp->cond_ready == B_TRUE) {
                        sp->cond_ready = B_FALSE;
                }
        } else if (sp->state_ready == B_FALSE) {
                sp->cond_ready = B_FALSE;
                sp->state_ready = B_TRUE;
                sp->throttle_ts = 0;
                sp->next_ts = ts + fipe_idle_ctrl.tick_interval;
                sp->last_busy = cpu_idle_prop_get_hrtime(
                    fipe_idle_ctrl.prop_busy, ctx);
                sp->last_idle = cpu_idle_prop_get_hrtime(
                    fipe_idle_ctrl.prop_idle, ctx);
                sp->last_intr = cpu_idle_prop_get_hrtime(
                    fipe_idle_ctrl.prop_intr, ctx);
                sp->idle_count = 0;
        }
}

/*ARGSUSED*/
static void
fipe_idle_exit(void* arg, cpu_idle_callback_context_t ctx, int flags)
{
        uint32_t cnt;
        hrtime_t ts;
        struct fipe_cpu_state *sp;

        sp = &fipe_cpu_states[CPU->cpu_id];
        if (sp->cond_ready) {
                do {
                        cnt = fipe_gbl_ctrl.cpu_count;
                        ASSERT(cnt > 0);
                } while (atomic_cas_32(&fipe_gbl_ctrl.cpu_count,
                    cnt, cnt - 1) != cnt);

                /*
                 * Try to disable power saving state.
                 * Only the first CPU waking from idle state will try to
                 * disable power saving state, all other CPUs will just go
                 * on and not try to wait for memory to recover from power
                 * saving state.
                 * So there are possible periods during which some CPUs are in
                 * active state but memory is in power saving state.
                 * This is OK, since it is an uncommon case, and it is
                 * better for performance to let them continue as their
                 * blocking latency is smaller than a mutex, and is only
                 * hit in the uncommon condition.
                 */
                if (cnt == ncpus) {
                        fipe_disable();
                        ts = cpu_idle_prop_get_hrtime(fipe_idle_ctrl.prop_exit,
                            ctx);
                        fipe_gbl_ctrl.time_in_pm += ts - fipe_gbl_ctrl.enter_ts;
                }
        }
}