// SPDX-License-Identifier: GPL-2.0-only
/*
 * Resource Director Technology(RDT)
 * - Intel Application Energy Telemetry
 *
 * Copyright (C) 2025 Intel Corporation
 *
 * Author:
 *    Tony Luck <tony.luck@intel.com>
 */

#define pr_fmt(fmt)   "resctrl: " fmt

#include <linux/bits.h>
#include <linux/compiler_types.h>
#include <linux/container_of.h>
#include <linux/cpumask.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/gfp_types.h>
#include <linux/init.h>
#include <linux/intel_pmt_features.h>
#include <linux/intel_vsec.h>
#include <linux/io.h>
#include <linux/minmax.h>
#include <linux/printk.h>
#include <linux/rculist.h>
#include <linux/rcupdate.h>
#include <linux/resctrl.h>
#include <linux/resctrl_types.h>
#include <linux/slab.h>
#include <linux/stddef.h>
#include <linux/topology.h>
#include <linux/types.h>

#include "internal.h"

/**
 * struct pmt_event - Telemetry event.
 * @id:         Resctrl event id.
 * @idx:        Counter index within each per-RMID block of counters.
 * @bin_bits:   Zero for integer valued events, else number bits in fraction
 *              part of fixed-point.
 */
struct pmt_event {
        enum resctrl_event_id   id;
        unsigned int            idx;
        unsigned int            bin_bits;
};

#define EVT(_id, _idx, _bits) { .id = _id, .idx = _idx, .bin_bits = _bits }

/**
 * struct event_group - Events with the same feature type ("energy" or "perf") and GUID.
 * @pfname:             PMT feature name ("energy" or "perf") of this event group.
 *                      Used by boot rdt= option.
 * @pfg:                Points to the aggregated telemetry space information
 *                      returned by the intel_pmt_get_regions_by_feature()
 *                      call to the INTEL_PMT_TELEMETRY driver that contains
 *                      data for all telemetry regions of type @pfname.
 *                      Valid if the system supports the event group,
 *                      NULL otherwise.
 * @force_off:          True when "rdt" command line or architecture code disables
 *                      this event group due to insufficient RMIDs.
 * @force_on:           True when "rdt" command line overrides disable of this
 *                      event group.
 * @guid:               Unique number per XML description file.
 * @num_rmid:           Number of RMIDs supported by this group. May be
 *                      adjusted downwards if enumeration from
 *                      intel_pmt_get_regions_by_feature() indicates fewer
 *                      RMIDs can be tracked simultaneously.
 * @mmio_size:          Number of bytes of MMIO registers for this group.
 * @num_events:         Number of events in this group.
 * @evts:               Array of event descriptors.
 */
struct event_group {
        /* Data fields for additional structures to manage this group. */
        const char                      *pfname;
        struct pmt_feature_group        *pfg;
        bool                            force_off, force_on;

        /* Remaining fields initialized from XML file. */
        u32                             guid;
        u32                             num_rmid;
        size_t                          mmio_size;
        unsigned int                    num_events;
        struct pmt_event                evts[] __counted_by(num_events);
};

#define XML_MMIO_SIZE(num_rmids, num_events, num_extra_status) \
                      (((num_rmids) * (num_events) + (num_extra_status)) * sizeof(u64))

/*
 * Link: https://github.com/intel/Intel-PMT/blob/main/xml/CWF/OOBMSM/RMID-ENERGY/cwf_aggregator.xml
 */
static struct event_group energy_0x26696143 = {
        .pfname         = "energy",
        .guid           = 0x26696143,
        .num_rmid       = 576,
        .mmio_size      = XML_MMIO_SIZE(576, 2, 3),
        .num_events     = 2,
        .evts           = {
                EVT(PMT_EVENT_ENERGY, 0, 18),
                EVT(PMT_EVENT_ACTIVITY, 1, 18),
        }
};

/*
 * Link: https://github.com/intel/Intel-PMT/blob/main/xml/CWF/OOBMSM/RMID-PERF/cwf_aggregator.xml
 */
static struct event_group perf_0x26557651 = {
        .pfname         = "perf",
        .guid           = 0x26557651,
        .num_rmid       = 576,
        .mmio_size      = XML_MMIO_SIZE(576, 7, 3),
        .num_events     = 7,
        .evts           = {
                EVT(PMT_EVENT_STALLS_LLC_HIT, 0, 0),
                EVT(PMT_EVENT_C1_RES, 1, 0),
                EVT(PMT_EVENT_UNHALTED_CORE_CYCLES, 2, 0),
                EVT(PMT_EVENT_STALLS_LLC_MISS, 3, 0),
                EVT(PMT_EVENT_AUTO_C6_RES, 4, 0),
                EVT(PMT_EVENT_UNHALTED_REF_CYCLES, 5, 0),
                EVT(PMT_EVENT_UOPS_RETIRED, 6, 0),
        }
};

static struct event_group *known_event_groups[] = {
        &energy_0x26696143,
        &perf_0x26557651,
};

#define for_each_event_group(_peg)                                              \
        for (_peg = known_event_groups;                                         \
             _peg < &known_event_groups[ARRAY_SIZE(known_event_groups)];        \
             _peg++)

bool intel_handle_aet_option(bool force_off, char *tok)
{
        struct event_group **peg;
        bool ret = false;
        u32 guid = 0;
        char *name;

        if (!tok)
                return false;

        name = strsep(&tok, ":");
        if (tok && kstrtou32(tok, 16, &guid))
                return false;

        for_each_event_group(peg) {
                if (strcmp(name, (*peg)->pfname))
                        continue;
                if (guid && (*peg)->guid != guid)
                        continue;
                if (force_off)
                        (*peg)->force_off = true;
                else
                        (*peg)->force_on = true;
                ret = true;
        }

        return ret;
}

static bool skip_telem_region(struct telemetry_region *tr, struct event_group *e)
{
        if (tr->guid != e->guid)
                return true;
        if (tr->plat_info.package_id >= topology_max_packages()) {
                pr_warn("Bad package %u in guid 0x%x\n", tr->plat_info.package_id,
                        tr->guid);
                return true;
        }
        if (tr->size != e->mmio_size) {
                pr_warn("MMIO space wrong size (%zu bytes) for guid 0x%x. Expected %zu bytes.\n",
                        tr->size, e->guid, e->mmio_size);
                return true;
        }

        return false;
}

static bool group_has_usable_regions(struct event_group *e, struct pmt_feature_group *p)
{
        bool usable_regions = false;

        for (int i = 0; i < p->count; i++) {
                if (skip_telem_region(&p->regions[i], e)) {
                        /*
                         * Clear the address field of regions that did not pass the checks in
                         * skip_telem_region() so they will not be used by intel_aet_read_event().
                         * This is safe to do because intel_pmt_get_regions_by_feature() allocates
                         * a new pmt_feature_group structure to return to each caller and only makes
                         * use of the pmt_feature_group::kref field when intel_pmt_put_feature_group()
                         * returns the structure.
                         */
                        p->regions[i].addr = NULL;

                        continue;
                }
                usable_regions = true;
        }

        return usable_regions;
}

static bool all_regions_have_sufficient_rmid(struct event_group *e, struct pmt_feature_group *p)
{
        struct telemetry_region *tr;

        for (int i = 0; i < p->count; i++) {
                if (!p->regions[i].addr)
                        continue;
                tr = &p->regions[i];
                if (tr->num_rmids < e->num_rmid) {
                        e->force_off = true;
                        return false;
                }
        }

        return true;
}

static bool enable_events(struct event_group *e, struct pmt_feature_group *p)
{
        struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_PERF_PKG].r_resctrl;
        int skipped_events = 0;

        if (e->force_off)
                return false;

        if (!group_has_usable_regions(e, p))
                return false;

        /*
         * Only enable event group with insufficient RMIDs if the user requested
         * it from the kernel command line.
         */
        if (!all_regions_have_sufficient_rmid(e, p) && !e->force_on) {
                pr_info("%s %s:0x%x monitoring not enabled due to insufficient RMIDs\n",
                        r->name, e->pfname, e->guid);
                return false;
        }

        for (int i = 0; i < p->count; i++) {
                if (!p->regions[i].addr)
                        continue;
                /*
                 * e->num_rmid only adjusted lower if user (via rdt= kernel
                 * parameter) forces an event group with insufficient RMID
                 * to be enabled.
                 */
                e->num_rmid = min(e->num_rmid, p->regions[i].num_rmids);
        }

        for (int j = 0; j < e->num_events; j++) {
                if (!resctrl_enable_mon_event(e->evts[j].id, true,
                                              e->evts[j].bin_bits, &e->evts[j]))
                        skipped_events++;
        }
        if (e->num_events == skipped_events) {
                pr_info("No events enabled in %s %s:0x%x\n", r->name, e->pfname, e->guid);
                return false;
        }

        if (r->mon.num_rmid)
                r->mon.num_rmid = min(r->mon.num_rmid, e->num_rmid);
        else
                r->mon.num_rmid = e->num_rmid;

        if (skipped_events)
                pr_info("%s %s:0x%x monitoring detected (skipped %d events)\n", r->name,
                        e->pfname, e->guid, skipped_events);
        else
                pr_info("%s %s:0x%x monitoring detected\n", r->name, e->pfname, e->guid);

        return true;
}

static enum pmt_feature_id lookup_pfid(const char *pfname)
{
        if (!strcmp(pfname, "energy"))
                return FEATURE_PER_RMID_ENERGY_TELEM;
        else if (!strcmp(pfname, "perf"))
                return FEATURE_PER_RMID_PERF_TELEM;

        pr_warn("Unknown PMT feature name '%s'\n", pfname);

        return FEATURE_INVALID;
}

/*
 * Request a copy of struct pmt_feature_group for each event group. If there is
 * one, the returned structure has an array of telemetry_region structures,
 * each element of the array describes one telemetry aggregator. The
 * telemetry aggregators may have different GUIDs so obtain duplicate struct
 * pmt_feature_group for event groups with same feature type but different
 * GUID. Post-processing ensures an event group can only use the telemetry
 * aggregators that match its GUID. An event group keeps a pointer to its
 * struct pmt_feature_group to indicate that its events are successfully
 * enabled.
 */
bool intel_aet_get_events(void)
{
        struct pmt_feature_group *p;
        enum pmt_feature_id pfid;
        struct event_group **peg;
        bool ret = false;

        for_each_event_group(peg) {
                pfid = lookup_pfid((*peg)->pfname);
                p = intel_pmt_get_regions_by_feature(pfid);
                if (IS_ERR_OR_NULL(p))
                        continue;
                if (enable_events(*peg, p)) {
                        (*peg)->pfg = p;
                        ret = true;
                } else {
                        intel_pmt_put_feature_group(p);
                }
        }

        return ret;
}

void __exit intel_aet_exit(void)
{
        struct event_group **peg;

        for_each_event_group(peg) {
                if ((*peg)->pfg) {
                        intel_pmt_put_feature_group((*peg)->pfg);
                        (*peg)->pfg = NULL;
                }
        }
}

#define DATA_VALID      BIT_ULL(63)
#define DATA_BITS       GENMASK_ULL(62, 0)

/*
 * Read counter for an event on a domain (summing all aggregators on the
 * domain). If an aggregator hasn't received any data for a specific RMID,
 * the MMIO read indicates that data is not valid.  Return success if at
 * least one aggregator has valid data.
 */
int intel_aet_read_event(int domid, u32 rmid, void *arch_priv, u64 *val)
{
        struct pmt_event *pevt = arch_priv;
        struct event_group *e;
        bool valid = false;
        u64 total = 0;
        u64 evtcount;
        void *pevt0;
        u32 idx;

        pevt0 = pevt - pevt->idx;
        e = container_of(pevt0, struct event_group, evts);
        idx = rmid * e->num_events;
        idx += pevt->idx;

        if (idx * sizeof(u64) + sizeof(u64) > e->mmio_size) {
                pr_warn_once("MMIO index %u out of range\n", idx);
                return -EIO;
        }

        for (int i = 0; i < e->pfg->count; i++) {
                if (!e->pfg->regions[i].addr)
                        continue;
                if (e->pfg->regions[i].plat_info.package_id != domid)
                        continue;
                evtcount = readq(e->pfg->regions[i].addr + idx * sizeof(u64));
                if (!(evtcount & DATA_VALID))
                        continue;
                total += evtcount & DATA_BITS;
                valid = true;
        }

        if (valid)
                *val = total;

        return valid ? 0 : -EINVAL;
}

void intel_aet_mon_domain_setup(int cpu, int id, struct rdt_resource *r,
                                struct list_head *add_pos)
{
        struct rdt_perf_pkg_mon_domain *d;
        int err;

        d = kzalloc_node(sizeof(*d), GFP_KERNEL, cpu_to_node(cpu));
        if (!d)
                return;

        d->hdr.id = id;
        d->hdr.type = RESCTRL_MON_DOMAIN;
        d->hdr.rid = RDT_RESOURCE_PERF_PKG;
        cpumask_set_cpu(cpu, &d->hdr.cpu_mask);
        list_add_tail_rcu(&d->hdr.list, add_pos);

        err = resctrl_online_mon_domain(r, &d->hdr);
        if (err) {
                list_del_rcu(&d->hdr.list);
                synchronize_rcu();
                kfree(d);
        }
}