// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright IBM Corp. 2024
 */

#define pr_fmt(fmt) "hd: " fmt

/*
 * Hiperdispatch:
 * Dynamically calculates the optimum number of high capacity COREs
 * by considering the state the system is in. When hiperdispatch decides
 * that a capacity update is necessary, it schedules a topology update.
 * During topology updates the CPU capacities are always re-adjusted.
 *
 * There is two places where CPU capacities are being accessed within
 * hiperdispatch.
 * -> hiperdispatch's reoccuring work function reads CPU capacities to
 *    determine high capacity CPU count.
 * -> during a topology update hiperdispatch's adjustment function
 *    updates CPU capacities.
 * These two can run on different CPUs in parallel which can cause
 * hiperdispatch to make wrong decisions. This can potentially cause
 * some overhead by leading to extra rebuild_sched_domains() calls
 * for correction. Access to capacities within hiperdispatch has to be
 * serialized to prevent the overhead.
 *
 * Hiperdispatch decision making revolves around steal time.
 * HD_STEAL_THRESHOLD value is taken as reference. Whenever steal time
 * crosses the threshold value hiperdispatch falls back to giving high
 * capacities to entitled CPUs. When steal time drops below the
 * threshold boundary, hiperdispatch utilizes all CPUs by giving all
 * of them high capacity.
 *
 * The theory behind HD_STEAL_THRESHOLD is related to the SMP thread
 * performance. Comparing the throughput of;
 * - single CORE, with N threads, running N tasks
 * - N separate COREs running N tasks,
 * using individual COREs for individual tasks yield better
 * performance. This performance difference is roughly ~30% (can change
 * between machine generations)
 *
 * Hiperdispatch tries to hint scheduler to use individual COREs for
 * each task, as long as steal time on those COREs are less than 30%,
 * therefore delaying the throughput loss caused by using SMP threads.
 */

#include <linux/cpufeature.h>
#include <linux/cpumask.h>
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/kernel_stat.h>
#include <linux/kstrtox.h>
#include <linux/ktime.h>
#include <linux/sysctl.h>
#include <linux/types.h>
#include <linux/workqueue.h>
#include <asm/hiperdispatch.h>
#include <asm/setup.h>
#include <asm/smp.h>
#include <asm/topology.h>

#define CREATE_TRACE_POINTS
#include <asm/trace/hiperdispatch.h>

#define HD_DELAY_FACTOR                 (4)
#define HD_DELAY_INTERVAL               (HZ / 4)
#define HD_STEAL_THRESHOLD              10
#define HD_STEAL_AVG_WEIGHT             16

static cpumask_t hd_vl_coremask;        /* Mask containing all vertical low COREs */
static cpumask_t hd_vmvl_cpumask;       /* Mask containing vertical medium and low CPUs */
static int hd_high_capacity_cores;      /* Current CORE count with high capacity */
static int hd_entitled_cores;           /* Total vertical high and medium CORE count */
static int hd_online_cores;             /* Current online CORE count */

static unsigned long hd_previous_steal; /* Previous iteration's CPU steal timer total */
static unsigned long hd_high_time;      /* Total time spent while all cpus have high capacity */
static unsigned long hd_low_time;       /* Total time spent while vl cpus have low capacity */
static atomic64_t hd_adjustments;       /* Total occurrence count of hiperdispatch adjustments */

static unsigned int hd_steal_threshold = HD_STEAL_THRESHOLD;
static unsigned int hd_delay_factor = HD_DELAY_FACTOR;
static int hd_enabled;

static void hd_capacity_work_fn(struct work_struct *work);
static DECLARE_DELAYED_WORK(hd_capacity_work, hd_capacity_work_fn);

static int hd_set_hiperdispatch_mode(int enable)
{
        if (!cpu_has_topology())
                enable = 0;
        if (hd_enabled == enable)
                return 0;
        hd_enabled = enable;
        return 1;
}

void hd_reset_state(void)
{
        cpumask_clear(&hd_vl_coremask);
        cpumask_clear(&hd_vmvl_cpumask);
        hd_entitled_cores = 0;
        hd_online_cores = 0;
}

void hd_add_core(int cpu)
{
        const struct cpumask *siblings;
        int polarization;

        hd_online_cores++;
        polarization = smp_cpu_get_polarization(cpu);
        siblings = topology_sibling_cpumask(cpu);
        switch (polarization) {
        case POLARIZATION_VH:
                hd_entitled_cores++;
                break;
        case POLARIZATION_VM:
                hd_entitled_cores++;
                cpumask_or(&hd_vmvl_cpumask, &hd_vmvl_cpumask, siblings);
                break;
        case POLARIZATION_VL:
                cpumask_set_cpu(cpu, &hd_vl_coremask);
                cpumask_or(&hd_vmvl_cpumask, &hd_vmvl_cpumask, siblings);
                break;
        }
}

/* Serialize update and read operations of debug counters. */
static DEFINE_MUTEX(hd_counter_mutex);

static void hd_update_times(void)
{
        static ktime_t prev;
        ktime_t now;

        /*
         * Check if hiperdispatch is active, if not set the prev to 0.
         * This way it is possible to differentiate the first update iteration after
         * enabling hiperdispatch.
         */
        if (hd_entitled_cores == 0 || hd_enabled == 0) {
                prev = ktime_set(0, 0);
                return;
        }
        now = ktime_get();
        if (ktime_after(prev, 0)) {
                if (hd_high_capacity_cores == hd_online_cores)
                        hd_high_time += ktime_ms_delta(now, prev);
                else
                        hd_low_time += ktime_ms_delta(now, prev);
        }
        prev = now;
}

static void hd_update_capacities(void)
{
        int cpu, upscaling_cores;
        unsigned long capacity;

        upscaling_cores = hd_high_capacity_cores - hd_entitled_cores;
        capacity = upscaling_cores > 0 ? CPU_CAPACITY_HIGH : CPU_CAPACITY_LOW;
        hd_high_capacity_cores = hd_entitled_cores;
        for_each_cpu(cpu, &hd_vl_coremask) {
                smp_set_core_capacity(cpu, capacity);
                if (capacity != CPU_CAPACITY_HIGH)
                        continue;
                hd_high_capacity_cores++;
                upscaling_cores--;
                if (upscaling_cores == 0)
                        capacity = CPU_CAPACITY_LOW;
        }
}

void hd_disable_hiperdispatch(void)
{
        cancel_delayed_work_sync(&hd_capacity_work);
        hd_high_capacity_cores = hd_online_cores;
        hd_previous_steal = 0;
}

int hd_enable_hiperdispatch(void)
{
        mutex_lock(&hd_counter_mutex);
        hd_update_times();
        mutex_unlock(&hd_counter_mutex);
        if (hd_enabled == 0)
                return 0;
        if (hd_entitled_cores == 0)
                return 0;
        if (hd_online_cores <= hd_entitled_cores)
                return 0;
        mod_delayed_work(system_dfl_wq, &hd_capacity_work, HD_DELAY_INTERVAL * hd_delay_factor);
        hd_update_capacities();
        return 1;
}

static unsigned long hd_steal_avg(unsigned long new)
{
        static unsigned long steal;

        steal = (steal * (HD_STEAL_AVG_WEIGHT - 1) + new) / HD_STEAL_AVG_WEIGHT;
        return steal;
}

static unsigned long hd_calculate_steal_percentage(void)
{
        unsigned long time_delta, steal_delta, steal, percentage;
        static ktime_t prev;
        int cpus, cpu;
        ktime_t now;

        cpus = 0;
        steal = 0;
        percentage = 0;
        for_each_cpu(cpu, &hd_vmvl_cpumask) {
                steal += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
                cpus++;
        }
        /*
         * If there is no vertical medium and low CPUs steal time
         * is 0 as vertical high CPUs shouldn't experience steal time.
         */
        if (cpus == 0)
                return percentage;
        now = ktime_get();
        time_delta = ktime_to_ns(ktime_sub(now, prev));
        if (steal > hd_previous_steal && hd_previous_steal != 0) {
                steal_delta = (steal - hd_previous_steal) * 100 / time_delta;
                percentage = steal_delta / cpus;
        }
        hd_previous_steal = steal;
        prev = now;
        return percentage;
}

static void hd_capacity_work_fn(struct work_struct *work)
{
        unsigned long steal_percentage, new_cores;

        mutex_lock(&smp_cpu_state_mutex);
        /*
         * If online cores are less or equal to entitled cores hiperdispatch
         * does not need to make any adjustments, call a topology update to
         * disable hiperdispatch.
         * Normally this check is handled on topology update, but during cpu
         * unhotplug, topology and cpu mask updates are done in reverse
         * order, causing hd_enable_hiperdispatch() to get stale data.
         */
        if (hd_online_cores <= hd_entitled_cores) {
                topology_schedule_update();
                mutex_unlock(&smp_cpu_state_mutex);
                return;
        }
        steal_percentage = hd_steal_avg(hd_calculate_steal_percentage());
        if (steal_percentage < hd_steal_threshold)
                new_cores = hd_online_cores;
        else
                new_cores = hd_entitled_cores;
        if (hd_high_capacity_cores != new_cores) {
                trace_s390_hd_rebuild_domains(hd_high_capacity_cores, new_cores);
                hd_high_capacity_cores = new_cores;
                atomic64_inc(&hd_adjustments);
                topology_schedule_update();
        }
        trace_s390_hd_work_fn(steal_percentage, hd_entitled_cores, hd_high_capacity_cores);
        mutex_unlock(&smp_cpu_state_mutex);
        schedule_delayed_work(&hd_capacity_work, HD_DELAY_INTERVAL);
}

static int hiperdispatch_ctl_handler(const struct ctl_table *ctl, int write,
                                     void *buffer, size_t *lenp, loff_t *ppos)
{
        int hiperdispatch;
        int rc;
        struct ctl_table ctl_entry = {
                .procname       = ctl->procname,
                .data           = &hiperdispatch,
                .maxlen         = sizeof(int),
                .extra1         = SYSCTL_ZERO,
                .extra2         = SYSCTL_ONE,
        };

        hiperdispatch = hd_enabled;
        rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos);
        if (rc < 0 || !write)
                return rc;
        mutex_lock(&smp_cpu_state_mutex);
        if (hd_set_hiperdispatch_mode(hiperdispatch))
                topology_schedule_update();
        mutex_unlock(&smp_cpu_state_mutex);
        return 0;
}

static const struct ctl_table hiperdispatch_ctl_table[] = {
        {
                .procname       = "hiperdispatch",
                .mode           = 0644,
                .proc_handler   = hiperdispatch_ctl_handler,
        },
};

static ssize_t hd_steal_threshold_show(struct device *dev,
                                       struct device_attribute *attr,
                                       char *buf)
{
        return sysfs_emit(buf, "%u\n", hd_steal_threshold);
}

static ssize_t hd_steal_threshold_store(struct device *dev,
                                        struct device_attribute *attr,
                                        const char *buf,
                                        size_t count)
{
        unsigned int val;
        int rc;

        rc = kstrtouint(buf, 0, &val);
        if (rc)
                return rc;
        if (val > 100)
                return -ERANGE;
        hd_steal_threshold = val;
        return count;
}

static DEVICE_ATTR_RW(hd_steal_threshold);

static ssize_t hd_delay_factor_show(struct device *dev,
                                    struct device_attribute *attr,
                                    char *buf)
{
        return sysfs_emit(buf, "%u\n", hd_delay_factor);
}

static ssize_t hd_delay_factor_store(struct device *dev,
                                     struct device_attribute *attr,
                                     const char *buf,
                                     size_t count)
{
        unsigned int val;
        int rc;

        rc = kstrtouint(buf, 0, &val);
        if (rc)
                return rc;
        if (!val)
                return -ERANGE;
        hd_delay_factor = val;
        return count;
}

static DEVICE_ATTR_RW(hd_delay_factor);

static struct attribute *hd_attrs[] = {
        &dev_attr_hd_steal_threshold.attr,
        &dev_attr_hd_delay_factor.attr,
        NULL,
};

static const struct attribute_group hd_attr_group = {
        .name  = "hiperdispatch",
        .attrs = hd_attrs,
};

static int hd_greedy_time_get(void *unused, u64 *val)
{
        mutex_lock(&hd_counter_mutex);
        hd_update_times();
        *val = hd_high_time;
        mutex_unlock(&hd_counter_mutex);
        return 0;
}

DEFINE_SIMPLE_ATTRIBUTE(hd_greedy_time_fops, hd_greedy_time_get, NULL, "%llu\n");

static int hd_conservative_time_get(void *unused, u64 *val)
{
        mutex_lock(&hd_counter_mutex);
        hd_update_times();
        *val = hd_low_time;
        mutex_unlock(&hd_counter_mutex);
        return 0;
}

DEFINE_SIMPLE_ATTRIBUTE(hd_conservative_time_fops, hd_conservative_time_get, NULL, "%llu\n");

static int hd_adjustment_count_get(void *unused, u64 *val)
{
        *val = atomic64_read(&hd_adjustments);
        return 0;
}

DEFINE_SIMPLE_ATTRIBUTE(hd_adjustments_fops, hd_adjustment_count_get, NULL, "%llu\n");

static void __init hd_create_debugfs_counters(void)
{
        struct dentry *dir;

        dir = debugfs_create_dir("hiperdispatch", arch_debugfs_dir);
        debugfs_create_file("conservative_time_ms", 0400, dir, NULL, &hd_conservative_time_fops);
        debugfs_create_file("greedy_time_ms", 0400, dir, NULL, &hd_greedy_time_fops);
        debugfs_create_file("adjustment_count", 0400, dir, NULL, &hd_adjustments_fops);
}

static void __init hd_create_attributes(void)
{
        struct device *dev;

        dev = bus_get_dev_root(&cpu_subsys);
        if (!dev)
                return;
        if (sysfs_create_group(&dev->kobj, &hd_attr_group))
                pr_warn("Unable to create hiperdispatch attribute group\n");
        put_device(dev);
}

static int __init hd_init(void)
{
        if (IS_ENABLED(CONFIG_HIPERDISPATCH_ON)) {
                hd_set_hiperdispatch_mode(1);
                topology_schedule_update();
        }
        if (!register_sysctl("s390", hiperdispatch_ctl_table))
                pr_warn("Failed to register s390.hiperdispatch sysctl attribute\n");
        hd_create_debugfs_counters();
        hd_create_attributes();
        return 0;
}
late_initcall(hd_init);