/*
 * Copyright (c) 2009 The DragonFly Project.  All rights reserved.
 * 
 * This code is derived from software contributed to The DragonFly Project
 * by Sepherosa Ziehau <sepherosa@gmail.com>
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "opt_acpi.h"

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/cpuhelper.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/queue.h>
#include <sys/rman.h>
#include <sys/sysctl.h>
#include <sys/msgport2.h>
#include <sys/cpu_topology.h>

#include "acpi.h"
#include "acpivar.h"
#include "acpi_cpu.h"
#include "acpi_cpu_pstate.h"

#define ACPI_NPSTATE_MAX        32

#define ACPI_PSS_PX_NENTRY      6

#define ACPI_PSD_COORD_SWALL    0xfc
#define ACPI_PSD_COORD_SWANY    0xfd
#define ACPI_PSD_COORD_HWALL    0xfe
#define ACPI_PSD_COORD_VALID(coord) \
        ((coord) == ACPI_PSD_COORD_SWALL || \
         (coord) == ACPI_PSD_COORD_SWANY || \
         (coord) == ACPI_PSD_COORD_HWALL)

struct acpi_pst_softc;
LIST_HEAD(acpi_pst_list, acpi_pst_softc);

struct acpi_pst_chmsg {
        struct cpuhelper_msg    ch_msg;
        const struct acpi_pst_res *ch_ctrl;
        const struct acpi_pst_res *ch_status;
};

struct acpi_pst_domain {
        uint32_t                pd_dom;
        uint32_t                pd_coord;
        uint32_t                pd_nproc;
        LIST_ENTRY(acpi_pst_domain) pd_link;

        uint32_t                pd_flags;

        struct lwkt_serialize   pd_serialize;

        int                     pd_state;
        struct acpi_pst_list    pd_pstlist;

        struct sysctl_ctx_list  pd_sysctl_ctx;
        struct sysctl_oid       *pd_sysctl_tree;
};
LIST_HEAD(acpi_pst_domlist, acpi_pst_domain);

#define ACPI_PSTDOM_FLAG_STUB   0x1     /* stub domain, no _PSD */
#define ACPI_PSTDOM_FLAG_DEAD   0x2     /* domain can't be started */
#define ACPI_PSTDOM_FLAG_INT    0x4     /* domain created from Integer _PSD */

struct acpi_pst_softc {
        device_t                pst_dev;
        struct acpi_cpu_softc   *pst_parent;
        struct acpi_pst_domain  *pst_domain;
        struct acpi_pst_res     pst_creg;
        struct acpi_pst_res     pst_sreg;

        int                     pst_state;
        int                     pst_cpuid;

        uint32_t                pst_flags;

        ACPI_HANDLE             pst_handle;

        LIST_ENTRY(acpi_pst_softc) pst_link;
};

#define ACPI_PST_FLAG_PPC       0x1
#define ACPI_PST_FLAG_PDL       0x2

static int      acpi_pst_probe(device_t dev);
static int      acpi_pst_attach(device_t dev);
static void     acpi_pst_notify(device_t dev);

static void     acpi_pst_postattach(void *);
static struct acpi_pst_domain *
                acpi_pst_domain_create_int(device_t, uint32_t);
static struct acpi_pst_domain *
                acpi_pst_domain_create_pkg(device_t, ACPI_OBJECT *);
static struct acpi_pst_domain *
                acpi_pst_domain_find(uint32_t);
static struct acpi_pst_domain *
                acpi_pst_domain_alloc(uint32_t, uint32_t, uint32_t);
static void     acpi_pst_domain_set_pstate_locked(struct acpi_pst_domain *,
                    int, int *);
static void     acpi_pst_domain_set_pstate(struct acpi_pst_domain *, int,
                    int *);
static void     acpi_pst_domain_check_nproc(device_t, struct acpi_pst_domain *);
static void     acpi_pst_global_set_pstate(int);
static void     acpi_pst_global_fixup_pstate(void);

static int      acpi_pst_check_csr(struct acpi_pst_softc *);
static int      acpi_pst_check_pstates(struct acpi_pst_softc *);
static int      acpi_pst_init(struct acpi_pst_softc *);
static int      acpi_pst_set_pstate(struct acpi_pst_softc *,
                    const struct acpi_pstate *);
static const struct acpi_pstate *
                acpi_pst_get_pstate(struct acpi_pst_softc *);
static int      acpi_pst_alloc_resource(device_t, ACPI_OBJECT *, int,
                    struct acpi_pst_res *);
static int      acpi_pst_eval_ppc(struct acpi_pst_softc *, int *);
static int      acpi_pst_eval_pdl(struct acpi_pst_softc *, int *);

static void     acpi_pst_check_csr_handler(struct cpuhelper_msg *);
static void     acpi_pst_check_pstates_handler(struct cpuhelper_msg *);
static void     acpi_pst_init_handler(struct cpuhelper_msg *);
static void     acpi_pst_set_pstate_handler(struct cpuhelper_msg *);
static void     acpi_pst_get_pstate_handler(struct cpuhelper_msg *);

static int      acpi_pst_sysctl_freqs(SYSCTL_HANDLER_ARGS);
static int      acpi_pst_sysctl_freqs_bin(SYSCTL_HANDLER_ARGS);
static int      acpi_pst_sysctl_power(SYSCTL_HANDLER_ARGS);
static int      acpi_pst_sysctl_members(SYSCTL_HANDLER_ARGS);
static int      acpi_pst_sysctl_select(SYSCTL_HANDLER_ARGS);
static int      acpi_pst_sysctl_global(SYSCTL_HANDLER_ARGS);

static struct acpi_pst_domlist  acpi_pst_domains =
        LIST_HEAD_INITIALIZER(acpi_pst_domains);
static int                      acpi_pst_domain_id;

static int                      acpi_pst_global_state;

static int                      acpi_pstate_start = -1;
static int                      acpi_pstate_count;
static int                      acpi_npstates;
static struct acpi_pstate       *acpi_pstates;

static const struct acpi_pst_md *acpi_pst_md;

static int                      acpi_pst_ppc = -1;
TUNABLE_INT("hw.acpi.cpu.pst.ppc", &acpi_pst_ppc);

static int                      acpi_pst_pdl = -1;
TUNABLE_INT("hw.acpi.cpu.pst.pdl", &acpi_pst_pdl);

static int                      acpi_pst_ht_reuse_domain = 1;
TUNABLE_INT("hw.acpi.cpu.pst.ht_reuse_domain", &acpi_pst_ht_reuse_domain);

static int                      acpi_pst_force_pkg_domain = 0;
TUNABLE_INT("hw.acpi.cpu.pst.force_pkg_domain", &acpi_pst_force_pkg_domain);

static int                      acpi_pst_handle_notify = 1;
TUNABLE_INT("hw.acpi.cpu.pst.handle_notify", &acpi_pst_handle_notify);

/*
 * Force CPU package power domain for Intel CPUs.
 *
 * As of this write (14 July 2015), all Intel CPUs only have CPU package
 * power domain.
 */
static int                      acpi_pst_intel_pkg_domain = 1;
TUNABLE_INT("hw.acpi.cpu.pst.intel_pkg_domain", &acpi_pst_intel_pkg_domain);

static device_method_t acpi_pst_methods[] = {
        /* Device interface */
        DEVMETHOD(device_probe,                 acpi_pst_probe),
        DEVMETHOD(device_attach,                acpi_pst_attach),
        DEVMETHOD(device_detach,                bus_generic_detach),
        DEVMETHOD(device_shutdown,              bus_generic_shutdown),
        DEVMETHOD(device_suspend,               bus_generic_suspend),
        DEVMETHOD(device_resume,                bus_generic_resume),

        /* Bus interface */
        DEVMETHOD(bus_add_child,                bus_generic_add_child),
        DEVMETHOD(bus_print_child,              bus_generic_print_child),
        DEVMETHOD(bus_read_ivar,                bus_generic_read_ivar),
        DEVMETHOD(bus_write_ivar,               bus_generic_write_ivar),
        DEVMETHOD(bus_get_resource_list,        bus_generic_get_resource_list),
        DEVMETHOD(bus_set_resource,             bus_generic_rl_set_resource),
        DEVMETHOD(bus_get_resource,             bus_generic_rl_get_resource),
        DEVMETHOD(bus_alloc_resource,           bus_generic_alloc_resource),
        DEVMETHOD(bus_release_resource,         bus_generic_release_resource),
        DEVMETHOD(bus_driver_added,             bus_generic_driver_added),
        DEVMETHOD(bus_activate_resource,        bus_generic_activate_resource),
        DEVMETHOD(bus_deactivate_resource,      bus_generic_deactivate_resource),
        DEVMETHOD(bus_setup_intr,               bus_generic_setup_intr),
        DEVMETHOD(bus_teardown_intr,            bus_generic_teardown_intr),

        DEVMETHOD_END
};

static driver_t acpi_pst_driver = {
        "cpu_pst",
        acpi_pst_methods,
        sizeof(struct acpi_pst_softc),
        .gpri = KOBJ_GPRI_ACPI+2
};

static devclass_t acpi_pst_devclass;
DRIVER_MODULE(cpu_pst, cpu, acpi_pst_driver, acpi_pst_devclass, NULL, NULL);
MODULE_DEPEND(cpu_pst, acpi, 1, 1, 1);

static __inline int
acpi_pst_freq2index(int freq)
{
        int i;

        for (i = 0; i < acpi_npstates; ++i) {
                if (acpi_pstates[i].st_freq == freq)
                        return i;
        }
        return -1;
}

static int
acpi_pst_probe(device_t dev)
{
        ACPI_BUFFER buf;
        ACPI_HANDLE handle;
        ACPI_STATUS status;
        ACPI_OBJECT *obj;

        if (acpi_disabled("cpu_pst") ||
            acpi_get_type(dev) != ACPI_TYPE_PROCESSOR)
                return ENXIO;

        if (acpi_pst_md == NULL)
                acpi_pst_md = acpi_pst_md_probe();

        handle = acpi_get_handle(dev);

        /*
         * Check _PSD package
         *
         * NOTE:
         * Some BIOSes do not expose _PCT for the second thread of
         * CPU cores.  In this case, _PSD should be enough to get the
         * P-state of the second thread working, since it must have
         * the same _PCT and _PSS as the first thread in the same
         * core.
         */
        buf.Pointer = NULL;
        buf.Length = ACPI_ALLOCATE_BUFFER;
        status = AcpiEvaluateObject(handle, "_PSD", NULL, &buf);
        if (!ACPI_FAILURE(status)) {
                AcpiOsFree((ACPI_OBJECT *)buf.Pointer);
                goto done;
        }

        /*
         * Check _PCT package
         */
        buf.Pointer = NULL;
        buf.Length = ACPI_ALLOCATE_BUFFER;
        status = AcpiEvaluateObject(handle, "_PCT", NULL, &buf);
        if (ACPI_FAILURE(status)) {
                if (bootverbose) {
                        device_printf(dev, "Can't get _PCT package - %s\n",
                                      AcpiFormatException(status));
                }
                return ENXIO;
        }

        obj = (ACPI_OBJECT *)buf.Pointer;
        if (!ACPI_PKG_VALID_EQ(obj, 2)) {
                device_printf(dev, "Invalid _PCT package\n");
                AcpiOsFree(obj);
                return ENXIO;
        }
        AcpiOsFree(obj);

        /*
         * Check _PSS package
         */
        buf.Pointer = NULL;
        buf.Length = ACPI_ALLOCATE_BUFFER;
        status = AcpiEvaluateObject(handle, "_PSS", NULL, &buf);
        if (ACPI_FAILURE(status)) {
                device_printf(dev, "Can't get _PSS package - %s\n",
                              AcpiFormatException(status));
                return ENXIO;
        }

        obj = (ACPI_OBJECT *)buf.Pointer;
        if (!ACPI_PKG_VALID(obj, 1)) {
                device_printf(dev, "Invalid _PSS package\n");
                AcpiOsFree(obj);
                return ENXIO;
        }
        AcpiOsFree(obj);

done:
        device_set_desc(dev, "ACPI CPU P-State");
        return 0;
}

static int
acpi_pst_attach(device_t dev)
{
        struct acpi_pst_softc *sc = device_get_softc(dev);
        struct acpi_pst_domain *dom = NULL;
        ACPI_BUFFER buf;
        ACPI_STATUS status;
        ACPI_OBJECT *obj;
        struct acpi_pstate *pstate, *p;
        int i, npstate, error, sstart, scount;

        sc->pst_dev = dev;
        sc->pst_parent = device_get_softc(device_get_parent(dev));
        sc->pst_handle = acpi_get_handle(dev);
        sc->pst_cpuid = acpi_get_magic(dev);

        /*
         * If there is a _PSD, then we create procossor domain
         * accordingly.  If there is no _PSD, we just fake a
         * default processor domain0.
         */
        buf.Pointer = NULL;
        buf.Length = ACPI_ALLOCATE_BUFFER;
        status = AcpiEvaluateObject(sc->pst_handle, "_PSD", NULL, &buf);
        if (!ACPI_FAILURE(status)) {
                obj = (ACPI_OBJECT *)buf.Pointer;

                if (acpi_pst_domain_id > 0) {
                        device_printf(dev, "Missing _PSD for certain CPUs\n");
                        AcpiOsFree(obj);
                        return ENXIO;
                }
                acpi_pst_domain_id = -1;

                if (ACPI_PKG_VALID_EQ(obj, 1)) {
                        dom = acpi_pst_domain_create_pkg(dev,
                                &obj->Package.Elements[0]);
                        if (dom == NULL) {
                                AcpiOsFree(obj);
                                return ENXIO;
                        }
                } else {
                        if (obj->Type != ACPI_TYPE_INTEGER) {
                                device_printf(dev,
                                    "Invalid _PSD package, Type 0x%x\n",
                                    obj->Type);
                                AcpiOsFree(obj);
                                return ENXIO;
                        } else {
                                device_printf(dev, "Integer _PSD %ju\n",
                                    (uintmax_t)obj->Integer.Value);
                                dom = acpi_pst_domain_create_int(dev,
                                    obj->Integer.Value);
                                if (dom == NULL) {
                                        AcpiOsFree(obj);
                                        return ENXIO;
                                }
                        }
                }

                /* Free _PSD */
                AcpiOsFree(buf.Pointer);
        } else {
                if (acpi_pst_domain_id < 0) {
                        device_printf(dev, "Missing _PSD for cpu%d\n",
                            sc->pst_cpuid);
                        return ENXIO;
                }

                /*
                 * Create a stub one processor domain for each processor
                 */
                dom = acpi_pst_domain_alloc(acpi_pst_domain_id,
                        ACPI_PSD_COORD_SWANY, 1);
                dom->pd_flags |= ACPI_PSTDOM_FLAG_STUB;

                ++acpi_pst_domain_id;
        }

        /* Make sure that adding us will not overflow our domain */
        acpi_pst_domain_check_nproc(dev, dom);

        /*
         * Get control/status registers from _PCT
         */
        buf.Pointer = NULL;
        buf.Length = ACPI_ALLOCATE_BUFFER;
        status = AcpiEvaluateObject(sc->pst_handle, "_PCT", NULL, &buf);
        if (ACPI_FAILURE(status)) {
                struct acpi_pst_softc *pst;

                /*
                 * No _PCT.  See the comment in acpi_pst_probe() near
                 * _PSD check.
                 *
                 * Use control/status registers of another CPU in the
                 * same domain, or in the same core, if the type of
                 * these registers are "Fixed Hardware", e.g. on most
                 * of the model Intel CPUs.
                 */
                pst = LIST_FIRST(&dom->pd_pstlist);
                if (pst == NULL) {
                        cpumask_t mask;

                        mask = get_cpumask_from_level(sc->pst_cpuid,
                            CORE_LEVEL);
                        if (CPUMASK_TESTNZERO(mask)) {
                                struct acpi_pst_domain *dom1;

                                LIST_FOREACH(dom1, &acpi_pst_domains, pd_link) {
                                        LIST_FOREACH(pst, &dom1->pd_pstlist,
                                            pst_link) {
                                                if (CPUMASK_TESTBIT(mask,
                                                    pst->pst_cpuid))
                                                        break;
                                        }
                                        if (pst != NULL)
                                                break;
                                }
                                if (pst != NULL && acpi_pst_ht_reuse_domain) {
                                        /*
                                         * Use the same domain for CPUs in the
                                         * same core.
                                         */
                                        device_printf(dev, "Destroy domain%u, "
                                            "reuse domain%u\n",
                                            dom->pd_dom, dom1->pd_dom);
                                        LIST_REMOVE(dom, pd_link);
                                        kfree(dom, M_DEVBUF);
                                        dom = dom1;
                                        /*
                                         * Make sure that adding us will not
                                         * overflow the domain containing
                                         * siblings in the same core.
                                         */
                                        acpi_pst_domain_check_nproc(dev, dom);
                                }
                        }
                }
                if (pst != NULL &&
                    pst->pst_creg.pr_res == NULL &&
                    pst->pst_creg.pr_rid == 0 &&
                    pst->pst_creg.pr_gas.SpaceId ==
                    ACPI_ADR_SPACE_FIXED_HARDWARE &&
                    pst->pst_sreg.pr_res == NULL &&
                    pst->pst_sreg.pr_rid == 0 &&
                    pst->pst_sreg.pr_gas.SpaceId ==
                    ACPI_ADR_SPACE_FIXED_HARDWARE) {
                        sc->pst_creg = pst->pst_creg;
                        sc->pst_sreg = pst->pst_sreg;
                        device_printf(dev,
                            "No _PCT; reuse %s control/status regs\n",
                            device_get_nameunit(pst->pst_dev));
                        goto fetch_pss;
                }
                device_printf(dev, "Can't get _PCT package - %s\n",
                              AcpiFormatException(status));
                return ENXIO;
        }

        obj = (ACPI_OBJECT *)buf.Pointer;
        if (!ACPI_PKG_VALID_EQ(obj, 2)) {
                device_printf(dev, "Invalid _PCT package\n");
                AcpiOsFree(obj);
                return ENXIO;
        }

        /* Save and try allocating control register */
        error = acpi_pst_alloc_resource(dev, obj, 0, &sc->pst_creg);
        if (error) {
                AcpiOsFree(obj);
                return error;
        }
        if (bootverbose) {
                device_printf(dev, "control reg %d %jx\n",
                              sc->pst_creg.pr_gas.SpaceId,
                              (uintmax_t)sc->pst_creg.pr_gas.Address);
        }

        /* Save and try allocating status register */
        error = acpi_pst_alloc_resource(dev, obj, 1, &sc->pst_sreg);
        if (error) {
                AcpiOsFree(obj);
                return error;
        }
        if (bootverbose) {
                device_printf(dev, "status reg %d %jx\n",
                              sc->pst_sreg.pr_gas.SpaceId,
                              (uintmax_t)sc->pst_sreg.pr_gas.Address);
        }

        /* Free _PCT */
        AcpiOsFree(obj);

fetch_pss:
        /*
         * Create P-State table according to _PSS
         */
        buf.Pointer = NULL;
        buf.Length = ACPI_ALLOCATE_BUFFER;
        status = AcpiEvaluateObject(sc->pst_handle, "_PSS", NULL, &buf);
        if (ACPI_FAILURE(status)) {
                /*
                 * No _PSS.  See the comment in acpi_pst_probe() near
                 * _PSD check.
                 *
                 * Assume _PSS are same across all CPUs; well, they
                 * should/have to be so.
                 */
                if (acpi_npstates > 0 && acpi_pstates != NULL) {
                        device_printf(dev, "No _PSS\n");
                        goto fetch_ppc;
                }
                device_printf(dev, "Can't get _PSS package - %s\n",
                              AcpiFormatException(status));
                return ENXIO;
        }

        obj = (ACPI_OBJECT *)buf.Pointer;
        if (!ACPI_PKG_VALID(obj, 1)) {
                device_printf(dev, "Invalid _PSS package\n");
                AcpiOsFree(obj);
                return ENXIO;
        }

        /* Don't create too many P-States */
        npstate = obj->Package.Count;
        if (npstate > ACPI_NPSTATE_MAX) {
                device_printf(dev, "Too many P-States, %d->%d\n",
                              npstate, ACPI_NPSTATE_MAX);
                npstate = ACPI_NPSTATE_MAX;
        }

        /*
         * If we have already created P-State table,
         * we must make sure that number of entries
         * is consistent.
         */
        if (acpi_pstates != NULL && acpi_npstates != npstate) {
                device_printf(dev, "Inconsistent # of P-States "
                              "cross Processor objects\n");
                AcpiOsFree(obj);
                return ENXIO;
        }

        /*
         * Create a temporary P-State table
         */
        pstate = kmalloc(sizeof(*pstate) * npstate, M_TEMP, M_WAITOK);
        for (i = 0, p = pstate; i < npstate; ++i, ++p) {
                ACPI_OBJECT *pkg;
                uint32_t *ptr[ACPI_PSS_PX_NENTRY] = {
                        &p->st_freq, &p->st_power, &p->st_xsit_lat,
                        &p->st_bm_lat, &p->st_cval, &p->st_sval
                };
                int j;

                pkg = &obj->Package.Elements[i];
                if (!ACPI_PKG_VALID(pkg, ACPI_PSS_PX_NENTRY)) {
                        device_printf(dev, "Invalud _PSS P%d\n", i);
                        AcpiOsFree(obj);
                        kfree(pstate, M_TEMP);
                        return ENXIO;
                }
                for (j = 0; j < ACPI_PSS_PX_NENTRY; ++j) {
                        if (acpi_PkgInt32(pkg, j, ptr[j]) != 0) {
                                device_printf(dev, "Can't extract "
                                              "_PSS P%d %dth entry\n", i, j);
                                AcpiOsFree(obj);
                                kfree(pstate, M_TEMP);
                                return ENXIO;
                        }
                }
                if (p->st_freq & 0x80000000) {
                        device_printf(dev, "Invalid _PSS P%d freq: 0x%08x\n",
                            i, p->st_freq);
                        AcpiOsFree(obj);
                        kfree(pstate, M_TEMP);
                        return ENXIO;
                }
        }

        /* Free _PSS */
        AcpiOsFree(obj);

        if (acpi_pstates == NULL) {
                /*
                 * If no P-State table is created yet,
                 * save the temporary one we just created.
                 */
                acpi_pstates = pstate;
                acpi_npstates = npstate;
                pstate = NULL;

                if (bootverbose) {
                        for (i = 0; i < acpi_npstates; ++i) {
                                device_printf(dev,
                                "freq %u, pwr %u, xlat %u, blat %u, "
                                "cv %08x, sv %08x\n",
                                acpi_pstates[i].st_freq,
                                acpi_pstates[i].st_power,
                                acpi_pstates[i].st_xsit_lat,
                                acpi_pstates[i].st_bm_lat,
                                acpi_pstates[i].st_cval,
                                acpi_pstates[i].st_sval);
                        }
                }
        } else {
                /*
                 * Make sure that P-State tables are same
                 * for all processors.
                 */
                if (memcmp(pstate, acpi_pstates,
                           sizeof(*pstate) * npstate) != 0) {
                        device_printf(dev, "Inconsistent _PSS "
                                      "cross Processor objects\n");
#if 0
                        /*
                         * Some BIOSes create different P-State tables;
                         * just trust the one from the BSP and move on.
                         */
                        kfree(pstate, M_TEMP);
                        return ENXIO;
#endif
                }
                kfree(pstate, M_TEMP);
        }

fetch_ppc:
        /* By default, we start from P-State table's first entry */
        sstart = 0;

        /*
         * Allow users to override _PPC (e.g. Apple firmware sets
         * CPLT to the most restrictive value, locking the CPU to
         * the lowest P-State on non-macOS).
         */
        if (acpi_pst_ppc >= 0) {
                if (acpi_pst_ppc < acpi_npstates) {
                        if (bootverbose) {
                                device_printf(dev, "_PPC override %d\n",
                                    acpi_pst_ppc);
                        }
                        sstart = acpi_pst_ppc;
                        goto proc_ppc;
                } else {
                        device_printf(dev, "Invalid _PPC override %d, "
                            "must be less than %d\n", acpi_pst_ppc,
                            acpi_npstates);
                }
        }

        /*
         * Adjust the usable first entry of P-State table,
         * if there is _PPC object.
         */
        error = acpi_pst_eval_ppc(sc, &sstart);
        if (error && error != ENOENT)
                return error;
        else if (!error)
                sc->pst_flags |= ACPI_PST_FLAG_PPC;
proc_ppc:
        if (acpi_pstate_start < 0) {
                acpi_pstate_start = sstart;
        } else if (acpi_pstate_start != sstart) {
                device_printf(dev, "_PPC mismatch, was %d, now %d\n",
                    acpi_pstate_start, sstart);
                if (acpi_pstate_start < sstart) {
                        device_printf(dev, "_PPC %d -> %d\n",
                            acpi_pstate_start, sstart);
                        acpi_pstate_start = sstart;
                }
        }

        /*
         * By default, we assume number of usable P-States is same as
         * number of P-States.
         */
        scount = acpi_npstates;

        /*
         * Allow users to override or set _PDL
         */
        if (acpi_pst_pdl >= 0) {
                if (acpi_pst_pdl < acpi_npstates) {
                        if (bootverbose) {
                                device_printf(dev, "_PDL override %d\n",
                                    acpi_pst_pdl);
                        }
                        scount = acpi_pst_pdl + 1;
                        goto proc_pdl;
                } else {
                        device_printf(dev, "Invalid _PDL override %d, "
                            "must be less than %d\n", acpi_pst_pdl,
                            acpi_npstates);
                }
        }

        /*
         * Adjust the number of usable entries in P-State table,
         * if there is _PDL object.
         */
        error = acpi_pst_eval_pdl(sc, &scount);
        if (error && error != ENOENT)
                return error;
        else if (!error)
                sc->pst_flags |= ACPI_PST_FLAG_PDL;
proc_pdl:
        if (acpi_pstate_count == 0) {
                acpi_pstate_count = scount;
        } else if (acpi_pstate_count != scount) {
                device_printf(dev, "_PDL mismatch, was %d, now %d\n",
                    acpi_pstate_count, scount);
                if (acpi_pstate_count > scount) {
                        device_printf(dev, "_PDL %d -> %d\n",
                            acpi_pstate_count, scount);
                        acpi_pstate_count = scount;
                }
        }

        /*
         * Some CPUs only have package P-states, but some BIOSes put each
         * hyperthread to its own P-state domain; allow user to override.
         */
        if (LIST_EMPTY(&dom->pd_pstlist) &&
            (acpi_pst_force_pkg_domain ||
             (cpu_vendor_id == CPU_VENDOR_INTEL &&
              acpi_pst_intel_pkg_domain))) {
                cpumask_t mask;

                mask = get_cpumask_from_level(sc->pst_cpuid, CHIP_LEVEL);
                if (CPUMASK_TESTNZERO(mask)) {
                        struct acpi_pst_softc *pst = NULL;
                        struct acpi_pst_domain *dom1;

                        LIST_FOREACH(dom1, &acpi_pst_domains, pd_link) {
                                LIST_FOREACH(pst, &dom1->pd_pstlist,
                                    pst_link) {
                                        if (CPUMASK_TESTBIT(mask,
                                            pst->pst_cpuid))
                                                break;
                                }
                                if (pst != NULL)
                                        break;
                        }
                        if (pst != NULL &&
                            memcmp(&pst->pst_creg, &sc->pst_creg,
                                sizeof(sc->pst_creg)) == 0 &&
                            memcmp(&pst->pst_sreg, &sc->pst_sreg,
                                sizeof(sc->pst_sreg)) == 0) {
                                /*
                                 * Use the same domain for CPUs in the
                                 * same package.
                                 */
                                device_printf(dev, "Destroy domain%u, "
                                    "force pkg domain%u\n",
                                    dom->pd_dom, dom1->pd_dom);
                                LIST_REMOVE(dom, pd_link);
                                kfree(dom, M_DEVBUF);
                                dom = dom1;
                                /*
                                 * Make sure that adding us will not
                                 * overflow the domain containing
                                 * siblings in the same package.
                                 */
                                acpi_pst_domain_check_nproc(dev, dom);
                        }
                }
        }

        /* Link us with the domain */
        sc->pst_domain = dom;
        LIST_INSERT_HEAD(&dom->pd_pstlist, sc, pst_link);

        if (device_get_unit(dev) == 0)
                AcpiOsExecute(OSL_NOTIFY_HANDLER, acpi_pst_postattach, NULL);

        if (sc->pst_flags & (ACPI_PST_FLAG_PPC | ACPI_PST_FLAG_PDL))
                sc->pst_parent->cpu_pst_notify = acpi_pst_notify;

        return 0;
}

static struct acpi_pst_domain *
acpi_pst_domain_create_pkg(device_t dev, ACPI_OBJECT *obj)
{
        struct acpi_pst_domain *dom;
        uint32_t val, domain, coord, nproc;

        if (!ACPI_PKG_VALID_EQ(obj, 5)) {
                device_printf(dev, "Invalid _PSD package\n");
                return NULL;
        }

        /* NumberOfEntries */
        if (acpi_PkgInt32(obj, 0, &val) != 0 || val != 5) {
                device_printf(dev, "Invalid _PSD NumberOfEntries\n");
                return NULL;
        }

        /* Revision */
        if (acpi_PkgInt32(obj, 1, &val) != 0 || val != 0) {
                device_printf(dev, "Invalid _PSD Revision\n");
                return NULL;
        }

        if (acpi_PkgInt32(obj, 2, &domain) != 0 ||
            acpi_PkgInt32(obj, 3, &coord) != 0 ||
            acpi_PkgInt32(obj, 4, &nproc) != 0) {
                device_printf(dev, "Can't extract _PSD package\n");
                return NULL;
        }

        if (!ACPI_PSD_COORD_VALID(coord)) {
                device_printf(dev, "Invalid _PSD CoordType (%#x)\n", coord);
                return NULL;
        }

        if (nproc > MAXCPU) {
                /*
                 * If NumProcessors is greater than MAXCPU
                 * and domain's coordination is SWALL, then
                 * we will never be able to start all CPUs
                 * within this domain, and power state
                 * transition will never be completed, so we
                 * just bail out here.
                 */
                if (coord == ACPI_PSD_COORD_SWALL) {
                        device_printf(dev, "Unsupported _PSD NumProcessors "
                                      "(%d)\n", nproc);
                        return NULL;
                }
        } else if (nproc == 0) {
                device_printf(dev, "_PSD NumProcessors are zero\n");
                return NULL;
        }

        dom = acpi_pst_domain_find(domain);
        if (dom != NULL) {
                if (dom->pd_flags & ACPI_PSTDOM_FLAG_INT) {
                        device_printf(dev, "Mixed Integer _PSD and "
                            "Package _PSD\n");
                        return NULL;
                }
                if (dom->pd_coord != coord) {
                        device_printf(dev, "Inconsistent _PSD coord "
                            "information cross Processor objects\n");
                        return NULL;
                }
                if (dom->pd_nproc != nproc) {
                        device_printf(dev, "Inconsistent _PSD nproc "
                            "information cross Processor objects\n");
                        /*
                         * Some stupid BIOSes will set wrong "# of processors",
                         * e.g. 1 for CPU w/ hyperthreading; Be lenient here.
                         */
                }
                return dom;
        }

        dom = acpi_pst_domain_alloc(domain, coord, nproc);
        if (bootverbose) {
                device_printf(dev, "create pkg domain%u, coord %#x\n",
                    dom->pd_dom, dom->pd_coord);
        }

        return dom;
}

static struct acpi_pst_domain *
acpi_pst_domain_create_int(device_t dev, uint32_t domain)
{
        struct acpi_pst_domain *dom;

        dom = acpi_pst_domain_find(domain);
        if (dom != NULL) {
                if ((dom->pd_flags & ACPI_PSTDOM_FLAG_INT) == 0) {
                        device_printf(dev, "Mixed Package _PSD and "
                            "Integer _PSD\n");
                        return NULL;
                }
                KKASSERT(dom->pd_coord == ACPI_PSD_COORD_SWALL);

                dom->pd_nproc++;
                return dom;
        }

        dom = acpi_pst_domain_alloc(domain, ACPI_PSD_COORD_SWALL, 1);
        dom->pd_flags |= ACPI_PSTDOM_FLAG_INT;

        if (bootverbose)
                device_printf(dev, "create int domain%u\n", dom->pd_dom);

        return dom;
}

static struct acpi_pst_domain *
acpi_pst_domain_find(uint32_t domain)
{
        struct acpi_pst_domain *dom;

        LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
                if (dom->pd_flags & ACPI_PSTDOM_FLAG_STUB)
                        continue;
                if (dom->pd_dom == domain)
                        return dom;
        }
        return NULL;
}

static struct acpi_pst_domain *
acpi_pst_domain_alloc(uint32_t domain, uint32_t coord, uint32_t nproc)
{
        struct acpi_pst_domain *dom;

        dom = kmalloc(sizeof(*dom), M_DEVBUF, M_WAITOK | M_ZERO);
        dom->pd_dom = domain;
        dom->pd_coord = coord;
        dom->pd_nproc = nproc;
        LIST_INIT(&dom->pd_pstlist);
        lwkt_serialize_init(&dom->pd_serialize);

        LIST_INSERT_HEAD(&acpi_pst_domains, dom, pd_link);

        return dom;
}

static void
acpi_pst_domain_set_pstate_locked(struct acpi_pst_domain *dom, int i, int *global)
{
        const struct acpi_pstate *pstate;
        struct acpi_pst_softc *sc;
        int done, error;

        ASSERT_SERIALIZED(&dom->pd_serialize);

        KKASSERT(i >= 0 && i < acpi_npstates);
        pstate = &acpi_pstates[i];

        done = 0;
        LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
                if (!done) {
                        error = acpi_pst_set_pstate(sc, pstate);
                        if (error) {
                                device_printf(sc->pst_dev, "can't set "
                                              "freq %d\n", pstate->st_freq);
                                /* XXX error cleanup? */
                        }
                        if (dom->pd_coord == ACPI_PSD_COORD_SWANY)
                                done = 1;
                }
                sc->pst_state = i;
        }
        dom->pd_state = i;

        if (global != NULL)
                *global = i;
}

static void
acpi_pst_domain_set_pstate(struct acpi_pst_domain *dom, int i, int *global)
{
        lwkt_serialize_enter(&dom->pd_serialize);
        acpi_pst_domain_set_pstate_locked(dom, i, global);
        lwkt_serialize_exit(&dom->pd_serialize);
}

static void
acpi_pst_global_set_pstate(int i)
{
        struct acpi_pst_domain *dom;
        int *global = &acpi_pst_global_state;

        LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
                /* Skip dead domain */
                if (dom->pd_flags & ACPI_PSTDOM_FLAG_DEAD)
                        continue;
                acpi_pst_domain_set_pstate(dom, i, global);
                global = NULL;
        }
}

static void
acpi_pst_global_fixup_pstate(void)
{
        struct acpi_pst_domain *dom;
        int *global = &acpi_pst_global_state;
        int sstart, scount;

        sstart = acpi_pstate_start;
        scount = acpi_pstate_count;

        LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
                int i = -1;

                /* Skip dead domain */
                if (dom->pd_flags & ACPI_PSTDOM_FLAG_DEAD)
                        continue;

                lwkt_serialize_enter(&dom->pd_serialize);

                if (global != NULL) {
                        if (*global < sstart)
                                *global = sstart;
                        else if (*global >= scount)
                                *global = scount - 1;
                        global = NULL;
                }
                if (dom->pd_state < sstart)
                        i = sstart;
                else if (dom->pd_state >= scount)
                        i = scount - 1;
                if (i >= 0)
                        acpi_pst_domain_set_pstate_locked(dom, i, NULL);

                lwkt_serialize_exit(&dom->pd_serialize);
        }
}

static void
acpi_pst_postattach(void *arg __unused)
{
        struct acpi_pst_domain *dom;
        struct acpi_cpu_softc *cpu;
        device_t *devices;
        int i, ndevices, error, has_domain;

        devices = NULL;
        ndevices = 0;
        error = devclass_get_devices(acpi_pst_devclass, &devices, &ndevices);
        if (error)
                return;

        if (ndevices == 0)
                return;

        cpu = NULL;
        for (i = 0; i < ndevices; ++i) {
                cpu = device_get_softc(device_get_parent(devices[i]));
                if (cpu->glob_sysctl_tree != NULL)
                        break;
        }
        kfree(devices, M_TEMP);
        KKASSERT(cpu != NULL);

        if (acpi_pst_md == NULL)
                kprintf("ACPI: no P-State CPU driver\n");

        has_domain = 0;
        LIST_FOREACH(dom, &acpi_pst_domains, pd_link) {
                struct acpi_pst_softc *sc;
                char buf[32];

                dom->pd_state = acpi_pstate_start;

                /*
                 * Make sure that all processors belonging to this
                 * domain are located.
                 */
                i = 0;
                LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
                        sc->pst_state = acpi_pstate_start;
                        ++i;
                }
                if (i != dom->pd_nproc) {
                        KKASSERT(i < dom->pd_nproc);

                        kprintf("ACPI: domain%u misses processors, "
                                "should be %d, got %d\n", dom->pd_dom,
                                dom->pd_nproc, i);
                        if (dom->pd_coord == ACPI_PSD_COORD_SWALL) {
                                /*
                                 * If this domain's coordination is
                                 * SWALL and we don't see all of the
                                 * member CPUs of this domain, then
                                 * the P-State transition will never
                                 * be completed, so just leave this
                                 * domain out.
                                 */
                                dom->pd_flags |= ACPI_PSTDOM_FLAG_DEAD;
                                continue;
                        }
                        dom->pd_nproc = i;
                }

                /*
                 * Validate P-State configurations for this domain
                 */
                LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
                        error = acpi_pst_check_csr(sc);
                        if (error)
                                break;

                        error = acpi_pst_check_pstates(sc);
                        if (error)
                                break;
                }
                if (sc != NULL) {
                        kprintf("ACPI: domain%u P-State configuration "
                                "check failed\n", dom->pd_dom);
                        dom->pd_flags |= ACPI_PSTDOM_FLAG_DEAD;
                        continue;
                }

                /*
                 * Do necssary P-State initialization
                 */
                LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
                        error = acpi_pst_init(sc);
                        if (error)
                                break;
                }
                if (sc != NULL) {
                        kprintf("ACPI: domain%u P-State initialization "
                                "check failed\n", dom->pd_dom);
                        dom->pd_flags |= ACPI_PSTDOM_FLAG_DEAD;
                        continue;
                }

                has_domain = 1;

                ksnprintf(buf, sizeof(buf), "px_dom%u", dom->pd_dom);

                sysctl_ctx_init(&dom->pd_sysctl_ctx);
                dom->pd_sysctl_tree =
                SYSCTL_ADD_NODE(&dom->pd_sysctl_ctx,
                        SYSCTL_CHILDREN(cpu->glob_sysctl_tree),
                        OID_AUTO, buf, CTLFLAG_RD, 0,
                        "P-State domain");
                if (dom->pd_sysctl_tree == NULL) {
                        kprintf("ACPI: Can't create sysctl tree for domain%u",
                                dom->pd_dom);
                        continue;
                }

                SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
                                SYSCTL_CHILDREN(dom->pd_sysctl_tree),
                                OID_AUTO, "available",
                                CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP,
                                dom, 0, acpi_pst_sysctl_freqs, "A",
                                "available frequencies");

                SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
                                SYSCTL_CHILDREN(dom->pd_sysctl_tree),
                                OID_AUTO, "avail",
                                CTLTYPE_OPAQUE | CTLFLAG_RD,
                                dom, 0, acpi_pst_sysctl_freqs_bin, "IU",
                                "available frequencies");

                SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
                                SYSCTL_CHILDREN(dom->pd_sysctl_tree),
                                OID_AUTO, "power",
                                CTLTYPE_OPAQUE | CTLFLAG_RD,
                                dom, 0, acpi_pst_sysctl_power, "IU",
                                "power of available frequencies");

                SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
                                SYSCTL_CHILDREN(dom->pd_sysctl_tree),
                                OID_AUTO, "members",
                                CTLTYPE_STRING | CTLFLAG_RD,
                                dom, 0, acpi_pst_sysctl_members, "A",
                                "member cpus");

                if (acpi_pst_md != NULL &&
                    acpi_pst_md->pmd_set_pstate != NULL) {
                        SYSCTL_ADD_PROC(&dom->pd_sysctl_ctx,
                                        SYSCTL_CHILDREN(dom->pd_sysctl_tree),
                                        OID_AUTO, "select",
                                        CTLTYPE_UINT | CTLFLAG_RW,
                                        dom, 0, acpi_pst_sysctl_select,
                                        "IU", "select freq");
                }
        }

        if (has_domain && acpi_pst_md != NULL &&
            acpi_pst_md->pmd_set_pstate != NULL) {
                SYSCTL_ADD_PROC(&cpu->glob_sysctl_ctx,
                                SYSCTL_CHILDREN(cpu->glob_sysctl_tree),
                                OID_AUTO, "px_global",
                                CTLTYPE_UINT | CTLFLAG_RW,
                                NULL, 0, acpi_pst_sysctl_global,
                                "IU", "select freq for all domains");
                SYSCTL_ADD_INT(&cpu->glob_sysctl_ctx,
                               SYSCTL_CHILDREN(cpu->glob_sysctl_tree),
                               OID_AUTO, "px_handle_notify", CTLFLAG_RW,
                               &acpi_pst_handle_notify, 0,
                               "handle type 0x80 notify");

                acpi_pst_global_set_pstate(acpi_pstate_start);
        }
}

static int
acpi_pst_sysctl_freqs(SYSCTL_HANDLER_ARGS)
{
        int i, error, sstart, scount;

        error = 0;
        sstart = acpi_pstate_start;
        scount = acpi_pstate_count;
        for (i = 0; i < acpi_npstates; ++i) {
                if (error == 0 && i)
                        error = SYSCTL_OUT(req, " ", 1);
                if (error == 0) {
                        const char *pat;
                        char buf[32];

                        if (i < sstart || i >= scount)
                                pat = "(%u)";
                        else
                                pat = "%u";

                        ksnprintf(buf, sizeof(buf), pat,
                                  acpi_pstates[i].st_freq);
                        error = SYSCTL_OUT(req, buf, strlen(buf));
                }
        }
        return error;
}

static int
acpi_pst_sysctl_freqs_bin(SYSCTL_HANDLER_ARGS)
{
        uint32_t freqs[ACPI_NPSTATE_MAX];
        int cnt, i, sstart, scount;

        sstart = acpi_pstate_start;
        scount = acpi_pstate_count;

        cnt = scount - sstart;
        for (i = 0; i < cnt; ++i)
                freqs[i] = acpi_pstates[sstart + i].st_freq;

        return sysctl_handle_opaque(oidp, freqs, cnt * sizeof(freqs[0]), req);
}

static int
acpi_pst_sysctl_power(SYSCTL_HANDLER_ARGS)
{
        uint32_t power[ACPI_NPSTATE_MAX];
        int cnt, i, sstart, scount;

        sstart = acpi_pstate_start;
        scount = acpi_pstate_count;

        cnt = scount - sstart;
        for (i = 0; i < cnt; ++i)
                power[i] = acpi_pstates[sstart + i].st_power;

        return sysctl_handle_opaque(oidp, power, cnt * sizeof(power[0]), req);
}

static int
acpi_pst_sysctl_members(SYSCTL_HANDLER_ARGS)
{
        struct acpi_pst_domain *dom = arg1;
        struct acpi_pst_softc *sc;
        int loop, error;

        loop = error = 0;
        LIST_FOREACH(sc, &dom->pd_pstlist, pst_link) {
                char buf[32];

                if (error == 0 && loop)
                        error = SYSCTL_OUT(req, " ", 1);
                if (error == 0) {
                        ksnprintf(buf, sizeof(buf), "cpu%d", sc->pst_cpuid);
                        error = SYSCTL_OUT(req, buf, strlen(buf));
                }

                if (error == 0 && acpi_pst_md && acpi_pst_md->pmd_get_pstate) {
                        const struct acpi_pstate *pstate;
                        const char *str;

                        pstate = acpi_pst_get_pstate(sc);
                        if (pstate == NULL) {
                                str = "(*)";
                        } else {
                                ksnprintf(buf, sizeof(buf), "(%d)",
                                          pstate->st_freq);
                                str = buf;
                        }
                        error = SYSCTL_OUT(req, str, strlen(str));
                }
                ++loop;
        }
        return error;
}

static int
acpi_pst_sysctl_select(SYSCTL_HANDLER_ARGS)
{
        struct acpi_pst_domain *dom = arg1;
        int error, i, freq;

        KKASSERT(dom->pd_state >= 0 && dom->pd_state < acpi_npstates);

        freq = acpi_pstates[dom->pd_state].st_freq;

        error = sysctl_handle_int(oidp, &freq, 0, req);
        if (error || req->newptr == NULL)
                return error;

        i = acpi_pst_freq2index(freq);
        if (i < 0)
                return EINVAL;

        acpi_pst_domain_set_pstate(dom, i, NULL);
        return 0;
}

static int
acpi_pst_sysctl_global(SYSCTL_HANDLER_ARGS)
{
        int error, i, freq;

        KKASSERT(acpi_pst_global_state >= 0 &&
                 acpi_pst_global_state < acpi_npstates);

        freq = acpi_pstates[acpi_pst_global_state].st_freq;

        error = sysctl_handle_int(oidp, &freq, 0, req);
        if (error || req->newptr == NULL)
                return error;

        i = acpi_pst_freq2index(freq);
        if (i < 0)
                return EINVAL;

        acpi_pst_global_set_pstate(i);

        return 0;
}

static void
acpi_pst_check_csr_handler(struct cpuhelper_msg *msg)
{
        struct acpi_pst_chmsg *pmsg = (struct acpi_pst_chmsg *)msg;
        int error;

        error = acpi_pst_md->pmd_check_csr(pmsg->ch_ctrl, pmsg->ch_status);
        cpuhelper_replymsg(msg, error);
}

static int
acpi_pst_check_csr(struct acpi_pst_softc *sc)
{
        struct acpi_pst_chmsg msg;

        if (acpi_pst_md == NULL)
                return 0;

        cpuhelper_initmsg(&msg.ch_msg, &curthread->td_msgport,
            acpi_pst_check_csr_handler, NULL, MSGF_PRIORITY);
        msg.ch_ctrl = &sc->pst_creg;
        msg.ch_status = &sc->pst_sreg;

        return (cpuhelper_domsg(&msg.ch_msg, sc->pst_cpuid));
}

static void
acpi_pst_check_pstates_handler(struct cpuhelper_msg *msg)
{
        int error;

        error = acpi_pst_md->pmd_check_pstates(acpi_pstates, acpi_npstates);
        cpuhelper_replymsg(msg, error);
}

static int
acpi_pst_check_pstates(struct acpi_pst_softc *sc)
{
        struct cpuhelper_msg msg;

        if (acpi_pst_md == NULL)
                return 0;

        cpuhelper_initmsg(&msg, &curthread->td_msgport,
            acpi_pst_check_pstates_handler, NULL, MSGF_PRIORITY);
        return (cpuhelper_domsg(&msg, sc->pst_cpuid));
}

static void
acpi_pst_init_handler(struct cpuhelper_msg *msg)
{
        struct acpi_pst_chmsg *pmsg = (struct acpi_pst_chmsg *)msg;
        int error;

        error = acpi_pst_md->pmd_init(pmsg->ch_ctrl, pmsg->ch_status);
        cpuhelper_replymsg(msg, error);
}

static int
acpi_pst_init(struct acpi_pst_softc *sc)
{
        struct acpi_pst_chmsg msg;

        if (acpi_pst_md == NULL)
                return 0;

        cpuhelper_initmsg(&msg.ch_msg, &curthread->td_msgport,
            acpi_pst_init_handler, NULL, MSGF_PRIORITY);
        msg.ch_ctrl = &sc->pst_creg;
        msg.ch_status = &sc->pst_sreg;

        return (cpuhelper_domsg(&msg.ch_msg, sc->pst_cpuid));
}

static void
acpi_pst_set_pstate_handler(struct cpuhelper_msg *msg)
{
        struct acpi_pst_chmsg *pmsg = (struct acpi_pst_chmsg *)msg;
        int error;

        error = acpi_pst_md->pmd_set_pstate(pmsg->ch_ctrl, pmsg->ch_status,
            msg->ch_cbarg);
        cpuhelper_replymsg(msg, error);
}

static int
acpi_pst_set_pstate(struct acpi_pst_softc *sc, const struct acpi_pstate *pstate)
{
        struct acpi_pst_chmsg msg;

        KKASSERT(acpi_pst_md != NULL);

        cpuhelper_initmsg(&msg.ch_msg, &curthread->td_msgport,
            acpi_pst_set_pstate_handler, __DECONST(void *, pstate),
            MSGF_PRIORITY);
        msg.ch_ctrl = &sc->pst_creg;
        msg.ch_status = &sc->pst_sreg;

        return (cpuhelper_domsg(&msg.ch_msg, sc->pst_cpuid));
}

static void
acpi_pst_get_pstate_handler(struct cpuhelper_msg *msg)
{
        struct acpi_pst_chmsg *pmsg = (struct acpi_pst_chmsg *)msg;
        const struct acpi_pstate *pstate;

        pstate = acpi_pst_md->pmd_get_pstate(pmsg->ch_status, acpi_pstates,
            acpi_npstates);
        msg->ch_cbarg = __DECONST(void *, pstate);
        cpuhelper_replymsg(msg, 0);
}

static const struct acpi_pstate *
acpi_pst_get_pstate(struct acpi_pst_softc *sc)
{
        struct acpi_pst_chmsg msg;

        if (acpi_pst_md == NULL)
                return 0;

        cpuhelper_initmsg(&msg.ch_msg, &curthread->td_msgport,
            acpi_pst_get_pstate_handler, NULL, MSGF_PRIORITY);
        msg.ch_status = &sc->pst_sreg;

        cpuhelper_domsg(&msg.ch_msg, sc->pst_cpuid);
        return (msg.ch_msg.ch_cbarg);
}

static int
acpi_pst_alloc_resource(device_t dev, ACPI_OBJECT *obj, int idx,
                        struct acpi_pst_res *res)
{
        struct acpi_pst_softc *sc = device_get_softc(dev);
        int error, type;

        /* Save GAS */
        error = acpi_PkgRawGas(obj, idx, &res->pr_gas);
        if (error)
                return error;

        /* Allocate resource, if possible */
        res->pr_rid = sc->pst_parent->cpu_next_rid;
        acpi_bus_alloc_gas(dev, &type, &res->pr_rid, &res->pr_gas, &res->pr_res, 0);
        if (res->pr_res != NULL) {
                sc->pst_parent->cpu_next_rid++;
                res->pr_bt = rman_get_bustag(res->pr_res);
                res->pr_bh = rman_get_bushandle(res->pr_res);
        } else {
                res->pr_rid = 0;
        }
        return 0;
}

static void
acpi_pst_domain_check_nproc(device_t dev, struct acpi_pst_domain *dom)
{
        struct acpi_pst_softc *pst;
        int i;

        i = 0;
        LIST_FOREACH(pst, &dom->pd_pstlist, pst_link)
                ++i;
        if (i == dom->pd_nproc) {
                /*
                 * Some stupid BIOSes will set wrong "# of processors",
                 * e.g. 1 for CPU w/ hyperthreading; Be lenient here.
                 */
                if (bootverbose) {
                        device_printf(dev, "domain%u already contains %d "
                            "P-States\n", dom->pd_dom, dom->pd_nproc);
                }
                dom->pd_nproc++;
        }
        KKASSERT(i < dom->pd_nproc);
}

static int
acpi_pst_eval_ppc(struct acpi_pst_softc *sc, int *sstart)
{
        ACPI_BUFFER buf;
        ACPI_STATUS status;
        ACPI_OBJECT *obj;

        buf.Pointer = NULL;
        buf.Length = ACPI_ALLOCATE_BUFFER;
        status = AcpiEvaluateObject(sc->pst_handle, "_PPC", NULL, &buf);
        if (!ACPI_FAILURE(status)) {
                ACPI_OBJECT_LIST arglist;
                ACPI_OBJECT arg[2];

                obj = (ACPI_OBJECT *)buf.Pointer;
                if (obj->Type == ACPI_TYPE_INTEGER) {
                        if (obj->Integer.Value >= acpi_npstates) {
                                /*
                                 * Some broken BIOSes can set this
                                 * to any values; let's simply start
                                 * from the first P-state, which
                                 * should be the highest frequency
                                 * available.
                                 */
                                if (bootverbose) {
                                        device_printf(sc->pst_dev,
                                            "Invalid _PPC value %ju, "
                                            "npstates %d\n",
                                            (uintmax_t)obj->Integer.Value,
                                            acpi_npstates);
                                }
                                *sstart = 0;
                        } else {
                                *sstart = obj->Integer.Value;
                                if (bootverbose) {
                                        device_printf(sc->pst_dev, "_PPC %d\n",
                                            *sstart);
                                }
                        }
                } else {
                        device_printf(sc->pst_dev, "Invalid _PPC object\n");
                        AcpiOsFree(obj);
                        return ENXIO;
                }

                /* Free _PPC */
                AcpiOsFree(obj);

                /* _PPC has been successfully processed */
                arglist.Pointer = arg;
                arglist.Count = 2;
                arg[0].Type = ACPI_TYPE_INTEGER;
                arg[0].Integer.Value = 0x80;
                arg[1].Type = ACPI_TYPE_INTEGER;
                arg[1].Integer.Value = 0;
                AcpiEvaluateObject(sc->pst_handle, "_OST", &arglist, NULL);

                return 0;
        }
        return ENOENT;
}

static int
acpi_pst_eval_pdl(struct acpi_pst_softc *sc, int *scount)
{
        ACPI_BUFFER buf;
        ACPI_STATUS status;
        ACPI_OBJECT *obj;

        buf.Pointer = NULL;
        buf.Length = ACPI_ALLOCATE_BUFFER;
        status = AcpiEvaluateObject(sc->pst_handle, "_PDL", NULL, &buf);
        if (!ACPI_FAILURE(status)) {
                obj = (ACPI_OBJECT *)buf.Pointer;
                if (obj->Type == ACPI_TYPE_INTEGER) {
                        if (obj->Integer.Value >= acpi_npstates) {
                                device_printf(sc->pst_dev,
                                    "Invalid _PDL value\n");
                                AcpiOsFree(obj);
                                return ENXIO;
                        }
                        if (obj->Integer.Value >= acpi_pstate_start) {
                                *scount = obj->Integer.Value + 1;
                                if (bootverbose) {
                                        device_printf(sc->pst_dev, "_PDL %d\n",
                                            *scount);
                                }
                        } else {
                                /* Prefer _PPC as stated in ACPI 5.1 8.4.4.6 */
                                device_printf(sc->pst_dev, "conflict _PDL %ju "
                                    "and _PPC %d, ignore\n",
                                    (uintmax_t)obj->Integer.Value,
                                    acpi_pstate_start);
                        }
                } else {
                        device_printf(sc->pst_dev, "Invalid _PDL object\n");
                        AcpiOsFree(obj);
                        return ENXIO;
                }

                /* Free _PDL */
                AcpiOsFree(obj);

                return 0;
        }
        return ENOENT;
}

/*
 * Notify is serialized by acpi task thread.
 */
static void
acpi_pst_notify(device_t dev)
{
        struct acpi_pst_softc *sc = device_get_softc(dev);
        boolean_t fixup = FALSE;

        if (!acpi_pst_handle_notify)
                return;

        /*
         * NOTE:
         * _PPC and _PDL evaluation order is critical.  _PDL
         * evaluation depends on _PPC evaluation.
         */
        if (sc->pst_flags & ACPI_PST_FLAG_PPC) {
                int sstart = acpi_pstate_start;

                acpi_pst_eval_ppc(sc, &sstart);
                if (acpi_pstate_start != sstart && sc->pst_cpuid == 0) {
                        acpi_pstate_start = sstart;
                        fixup = TRUE;
                }
        }
        if (sc->pst_flags & ACPI_PST_FLAG_PDL) {
                int scount = acpi_pstate_count;

                acpi_pst_eval_pdl(sc, &scount);
                if (acpi_pstate_count != scount && sc->pst_cpuid == 0) {
                        acpi_pstate_count = scount;
                        fixup = TRUE;
                }
        }

        if (fixup && acpi_pst_md != NULL &&
            acpi_pst_md->pmd_set_pstate != NULL)
                acpi_pst_global_fixup_pstate();
}