// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2023 Intel Corporation. All rights reserved. */
#include <linux/acpi.h>
#include <linux/xarray.h>
#include <linux/fw_table.h>
#include <linux/node.h>
#include <linux/overflow.h>
#include "cxlpci.h"
#include "cxlmem.h"
#include "core.h"
#include "cxl.h"

struct dsmas_entry {
        struct range dpa_range;
        u8 handle;
        struct access_coordinate coord[ACCESS_COORDINATE_MAX];
        struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX];
        int entries;
        int qos_class;
};

static u32 cdat_normalize(u16 entry, u64 base, u8 type)
{
        u32 value;

        /*
         * Check for invalid and overflow values
         */
        if (entry == 0xffff || !entry)
                return 0;
        if (base > (UINT_MAX / (entry)))
                return 0;

        /*
         * CDAT fields follow the format of HMAT fields. See table 5 Device
         * Scoped Latency and Bandwidth Information Structure in Coherent Device
         * Attribute Table (CDAT) Specification v1.01.
         */
        value = entry * base;
        switch (type) {
        case ACPI_HMAT_ACCESS_LATENCY:
        case ACPI_HMAT_READ_LATENCY:
        case ACPI_HMAT_WRITE_LATENCY:
                value = DIV_ROUND_UP(value, 1000);
                break;
        default:
                break;
        }
        return value;
}

static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg,
                              const unsigned long end)
{
        struct acpi_cdat_header *hdr = &header->cdat;
        struct acpi_cdat_dsmas *dsmas;
        int size = sizeof(*hdr) + sizeof(*dsmas);
        struct xarray *dsmas_xa = arg;
        struct dsmas_entry *dent;
        u16 len;
        int rc;

        len = le16_to_cpu((__force __le16)hdr->length);
        if (len != size || (unsigned long)hdr + len > end) {
                pr_warn("Malformed DSMAS table length: (%u:%u)\n", size, len);
                return -EINVAL;
        }

        /* Skip common header */
        dsmas = (struct acpi_cdat_dsmas *)(hdr + 1);

        dent = kzalloc_obj(*dent);
        if (!dent)
                return -ENOMEM;

        dent->handle = dsmas->dsmad_handle;
        dent->dpa_range.start = le64_to_cpu((__force __le64)dsmas->dpa_base_address);
        dent->dpa_range.end = le64_to_cpu((__force __le64)dsmas->dpa_base_address) +
                              le64_to_cpu((__force __le64)dsmas->dpa_length) - 1;

        rc = xa_insert(dsmas_xa, dent->handle, dent, GFP_KERNEL);
        if (rc) {
                kfree(dent);
                return rc;
        }

        return 0;
}

static void __cxl_access_coordinate_set(struct access_coordinate *coord,
                                        int access, unsigned int val)
{
        switch (access) {
        case ACPI_HMAT_ACCESS_LATENCY:
                coord->read_latency = val;
                coord->write_latency = val;
                break;
        case ACPI_HMAT_READ_LATENCY:
                coord->read_latency = val;
                break;
        case ACPI_HMAT_WRITE_LATENCY:
                coord->write_latency = val;
                break;
        case ACPI_HMAT_ACCESS_BANDWIDTH:
                coord->read_bandwidth = val;
                coord->write_bandwidth = val;
                break;
        case ACPI_HMAT_READ_BANDWIDTH:
                coord->read_bandwidth = val;
                break;
        case ACPI_HMAT_WRITE_BANDWIDTH:
                coord->write_bandwidth = val;
                break;
        }
}

static void cxl_access_coordinate_set(struct access_coordinate *coord,
                                      int access, unsigned int val)
{
        for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)
                __cxl_access_coordinate_set(&coord[i], access, val);
}

static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg,
                               const unsigned long end)
{
        struct acpi_cdat_header *hdr = &header->cdat;
        struct acpi_cdat_dslbis *dslbis;
        int size = sizeof(*hdr) + sizeof(*dslbis);
        struct xarray *dsmas_xa = arg;
        struct dsmas_entry *dent;
        __le64 le_base;
        __le16 le_val;
        u64 val;
        u16 len;

        len = le16_to_cpu((__force __le16)hdr->length);
        if (len != size || (unsigned long)hdr + len > end) {
                pr_warn("Malformed DSLBIS table length: (%u:%u)\n", size, len);
                return -EINVAL;
        }

        /* Skip common header */
        dslbis = (struct acpi_cdat_dslbis *)(hdr + 1);

        /* Skip unrecognized data type */
        if (dslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH)
                return 0;

        /* Not a memory type, skip */
        if ((dslbis->flags & ACPI_HMAT_MEMORY_HIERARCHY) != ACPI_HMAT_MEMORY)
                return 0;

        dent = xa_load(dsmas_xa, dslbis->handle);
        if (!dent) {
                pr_warn("No matching DSMAS entry for DSLBIS entry.\n");
                return 0;
        }

        le_base = (__force __le64)dslbis->entry_base_unit;
        le_val = (__force __le16)dslbis->entry[0];
        val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base),
                             dslbis->data_type);

        cxl_access_coordinate_set(dent->cdat_coord, dslbis->data_type, val);

        return 0;
}

static int cdat_table_parse_output(int rc)
{
        if (rc < 0)
                return rc;
        if (rc == 0)
                return -ENOENT;

        return 0;
}

static int cxl_cdat_endpoint_process(struct cxl_port *port,
                                     struct xarray *dsmas_xa)
{
        int rc;

        rc = cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler,
                              dsmas_xa, port->cdat.table, port->cdat.length);
        rc = cdat_table_parse_output(rc);
        if (rc)
                return rc;

        rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler,
                              dsmas_xa, port->cdat.table, port->cdat.length);
        return cdat_table_parse_output(rc);
}

static int cxl_port_perf_data_calculate(struct cxl_port *port,
                                        struct xarray *dsmas_xa)
{
        struct access_coordinate ep_c[ACCESS_COORDINATE_MAX];
        struct dsmas_entry *dent;
        int valid_entries = 0;
        unsigned long index;
        int rc;

        rc = cxl_endpoint_get_perf_coordinates(port, ep_c);
        if (rc) {
                dev_dbg(&port->dev, "Failed to retrieve ep perf coordinates.\n");
                return rc;
        }

        struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port);

        if (!cxl_root)
                return -ENODEV;

        if (!cxl_root->ops.qos_class)
                return -EOPNOTSUPP;

        xa_for_each(dsmas_xa, index, dent) {
                int qos_class;

                cxl_coordinates_combine(dent->coord, dent->cdat_coord, ep_c);
                dent->entries = 1;
                rc = cxl_root->ops.qos_class(cxl_root,
                                             &dent->coord[ACCESS_COORDINATE_CPU],
                                             1, &qos_class);
                if (rc != 1)
                        continue;

                valid_entries++;
                dent->qos_class = qos_class;
        }

        if (!valid_entries)
                return -ENOENT;

        return 0;
}

static void update_perf_entry(struct device *dev, struct dsmas_entry *dent,
                              struct cxl_dpa_perf *dpa_perf)
{
        for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
                dpa_perf->coord[i] = dent->coord[i];
                dpa_perf->cdat_coord[i] = dent->cdat_coord[i];
        }
        dpa_perf->dpa_range = dent->dpa_range;
        dpa_perf->qos_class = dent->qos_class;
        dev_dbg(dev,
                "DSMAS: dpa: %pra qos: %d read_bw: %d write_bw %d read_lat: %d write_lat: %d\n",
                &dent->dpa_range, dpa_perf->qos_class,
                dent->coord[ACCESS_COORDINATE_CPU].read_bandwidth,
                dent->coord[ACCESS_COORDINATE_CPU].write_bandwidth,
                dent->coord[ACCESS_COORDINATE_CPU].read_latency,
                dent->coord[ACCESS_COORDINATE_CPU].write_latency);
}

static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds,
                                     struct xarray *dsmas_xa)
{
        struct device *dev = cxlds->dev;
        struct dsmas_entry *dent;
        unsigned long index;

        xa_for_each(dsmas_xa, index, dent) {
                bool found = false;

                for (int i = 0; i < cxlds->nr_partitions; i++) {
                        struct resource *res = &cxlds->part[i].res;
                        struct range range = {
                                .start = res->start,
                                .end = res->end,
                        };

                        if (range_contains(&range, &dent->dpa_range)) {
                                update_perf_entry(dev, dent,
                                                  &cxlds->part[i].perf);
                                found = true;
                                break;
                        }
                }

                if (!found)
                        dev_dbg(dev, "no partition for dsmas dpa: %pra\n",
                                &dent->dpa_range);
        }
}

static int match_cxlrd_qos_class(struct device *dev, void *data)
{
        int dev_qos_class = *(int *)data;
        struct cxl_root_decoder *cxlrd;

        if (!is_root_decoder(dev))
                return 0;

        cxlrd = to_cxl_root_decoder(dev);
        if (cxlrd->qos_class == CXL_QOS_CLASS_INVALID)
                return 0;

        if (cxlrd->qos_class == dev_qos_class)
                return 1;

        return 0;
}

static void reset_dpa_perf(struct cxl_dpa_perf *dpa_perf)
{
        *dpa_perf = (struct cxl_dpa_perf) {
                .qos_class = CXL_QOS_CLASS_INVALID,
        };
}

static bool cxl_qos_match(struct cxl_port *root_port,
                          struct cxl_dpa_perf *dpa_perf)
{
        if (dpa_perf->qos_class == CXL_QOS_CLASS_INVALID)
                return false;

        if (!device_for_each_child(&root_port->dev, &dpa_perf->qos_class,
                                   match_cxlrd_qos_class))
                return false;

        return true;
}

static int match_cxlrd_hb(struct device *dev, void *data)
{
        struct device *host_bridge = data;
        struct cxl_switch_decoder *cxlsd;
        struct cxl_root_decoder *cxlrd;

        if (!is_root_decoder(dev))
                return 0;

        cxlrd = to_cxl_root_decoder(dev);
        cxlsd = &cxlrd->cxlsd;

        guard(rwsem_read)(&cxl_rwsem.region);
        for (int i = 0; i < cxlsd->nr_targets; i++) {
                if (cxlsd->target[i] && host_bridge == cxlsd->target[i]->dport_dev)
                        return 1;
        }

        return 0;
}

static void cxl_qos_class_verify(struct cxl_memdev *cxlmd)
{
        struct cxl_dev_state *cxlds = cxlmd->cxlds;
        struct cxl_port *root_port;

        struct cxl_root *cxl_root __free(put_cxl_root) =
                find_cxl_root(cxlmd->endpoint);

        /*
         * No need to reset_dpa_perf() here as find_cxl_root() is guaranteed to
         * succeed when called in the cxl_endpoint_port_probe() path.
         */
        if (!cxl_root)
                return;

        root_port = &cxl_root->port;

        /*
         * Save userspace from needing to check if a qos class has any matches
         * by hiding qos class info if the memdev is not mapped by a root
         * decoder, or the partition class does not match any root decoder
         * class.
         */
        if (!device_for_each_child(&root_port->dev,
                                   cxlmd->endpoint->host_bridge,
                                   match_cxlrd_hb)) {
                for (int i = 0; i < cxlds->nr_partitions; i++) {
                        struct cxl_dpa_perf *perf = &cxlds->part[i].perf;

                        reset_dpa_perf(perf);
                }
                return;
        }

        for (int i = 0; i < cxlds->nr_partitions; i++) {
                struct cxl_dpa_perf *perf = &cxlds->part[i].perf;

                if (!cxl_qos_match(root_port, perf))
                        reset_dpa_perf(perf);
        }
}

static void discard_dsmas(struct xarray *xa)
{
        unsigned long index;
        void *ent;

        xa_for_each(xa, index, ent) {
                xa_erase(xa, index);
                kfree(ent);
        }
        xa_destroy(xa);
}
DEFINE_FREE(dsmas, struct xarray *, if (_T) discard_dsmas(_T))

void cxl_endpoint_parse_cdat(struct cxl_port *port)
{
        struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
        struct cxl_dev_state *cxlds = cxlmd->cxlds;
        struct xarray __dsmas_xa;
        struct xarray *dsmas_xa __free(dsmas) = &__dsmas_xa;
        int rc;

        xa_init(&__dsmas_xa);
        if (!port->cdat.table)
                return;

        rc = cxl_cdat_endpoint_process(port, dsmas_xa);
        if (rc < 0) {
                dev_dbg(&port->dev, "Failed to parse CDAT: %d\n", rc);
                return;
        }

        rc = cxl_port_perf_data_calculate(port, dsmas_xa);
        if (rc) {
                dev_dbg(&port->dev, "Failed to do perf coord calculations.\n");
                return;
        }

        cxl_memdev_set_qos_class(cxlds, dsmas_xa);
        cxl_qos_class_verify(cxlmd);
        cxl_memdev_update_perf(cxlmd);
}
EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, "CXL");

static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg,
                               const unsigned long end)
{
        struct acpi_cdat_sslbis_table {
                struct acpi_cdat_header header;
                struct acpi_cdat_sslbis sslbis_header;
                struct acpi_cdat_sslbe entries[];
        } *tbl = (struct acpi_cdat_sslbis_table *)header;
        int size = sizeof(header->cdat) + sizeof(tbl->sslbis_header);
        struct acpi_cdat_sslbis *sslbis;
        struct cxl_dport *dport = arg;
        struct device *dev = &dport->port->dev;
        int remain, entries, i;
        u16 len;

        len = le16_to_cpu((__force __le16)header->cdat.length);
        remain = len - size;
        if (!remain || remain % sizeof(tbl->entries[0]) ||
            (unsigned long)header + len > end) {
                dev_warn(dev, "Malformed SSLBIS table length: (%u)\n", len);
                return -EINVAL;
        }

        sslbis = &tbl->sslbis_header;
        /* Unrecognized data type, we can skip */
        if (sslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH)
                return 0;

        entries = remain / sizeof(tbl->entries[0]);
        if (struct_size(tbl, entries, entries) != len)
                return -EINVAL;

        for (i = 0; i < entries; i++) {
                u16 x = le16_to_cpu((__force __le16)tbl->entries[i].portx_id);
                u16 y = le16_to_cpu((__force __le16)tbl->entries[i].porty_id);
                __le64 le_base;
                __le16 le_val;
                u16 dsp_id;
                u64 val;

                switch (x) {
                case ACPI_CDAT_SSLBIS_US_PORT:
                        dsp_id = y;
                        break;
                case ACPI_CDAT_SSLBIS_ANY_PORT:
                        switch (y) {
                        case ACPI_CDAT_SSLBIS_US_PORT:
                                dsp_id = x;
                                break;
                        case ACPI_CDAT_SSLBIS_ANY_PORT:
                                dsp_id = ACPI_CDAT_SSLBIS_ANY_PORT;
                                break;
                        default:
                                dsp_id = y;
                                break;
                        }
                        break;
                default:
                        dsp_id = x;
                        break;
                }

                le_base = (__force __le64)tbl->sslbis_header.entry_base_unit;
                le_val = (__force __le16)tbl->entries[i].latency_or_bandwidth;
                val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base),
                                     sslbis->data_type);

                if (dsp_id == ACPI_CDAT_SSLBIS_ANY_PORT ||
                    dsp_id == dport->port_id) {
                        cxl_access_coordinate_set(dport->coord,
                                                  sslbis->data_type, val);
                        return 0;
                }
        }

        return 0;
}

void cxl_switch_parse_cdat(struct cxl_dport *dport)
{
        struct cxl_port *port = dport->port;
        int rc;

        if (!port->cdat.table)
                return;

        rc = cdat_table_parse(ACPI_CDAT_TYPE_SSLBIS, cdat_sslbis_handler,
                              dport, port->cdat.table, port->cdat.length);
        rc = cdat_table_parse_output(rc);
        if (rc)
                dev_dbg(&port->dev, "Failed to parse SSLBIS: %d\n", rc);
}
EXPORT_SYMBOL_NS_GPL(cxl_switch_parse_cdat, "CXL");

static void __cxl_coordinates_combine(struct access_coordinate *out,
                                      struct access_coordinate *c1,
                                      struct access_coordinate *c2)
{
                if (c1->write_bandwidth && c2->write_bandwidth)
                        out->write_bandwidth = min(c1->write_bandwidth,
                                                   c2->write_bandwidth);
                out->write_latency = c1->write_latency + c2->write_latency;

                if (c1->read_bandwidth && c2->read_bandwidth)
                        out->read_bandwidth = min(c1->read_bandwidth,
                                                  c2->read_bandwidth);
                out->read_latency = c1->read_latency + c2->read_latency;
}

/**
 * cxl_coordinates_combine - Combine the two input coordinates
 *
 * @out: Output coordinate of c1 and c2 combined
 * @c1: input coordinates
 * @c2: input coordinates
 */
void cxl_coordinates_combine(struct access_coordinate *out,
                             struct access_coordinate *c1,
                             struct access_coordinate *c2)
{
        for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)
                __cxl_coordinates_combine(&out[i], &c1[i], &c2[i]);
}

MODULE_IMPORT_NS("CXL");

static void cxl_bandwidth_add(struct access_coordinate *coord,
                              struct access_coordinate *c1,
                              struct access_coordinate *c2)
{
        for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
                coord[i].read_bandwidth = c1[i].read_bandwidth +
                                          c2[i].read_bandwidth;
                coord[i].write_bandwidth = c1[i].write_bandwidth +
                                           c2[i].write_bandwidth;
        }
}

static bool dpa_perf_contains(struct cxl_dpa_perf *perf,
                              struct resource *dpa_res)
{
        struct range dpa = {
                .start = dpa_res->start,
                .end = dpa_res->end,
        };

        return range_contains(&perf->dpa_range, &dpa);
}

static struct cxl_dpa_perf *cxled_get_dpa_perf(struct cxl_endpoint_decoder *cxled)
{
        struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
        struct cxl_dev_state *cxlds = cxlmd->cxlds;
        struct cxl_dpa_perf *perf;

        if (cxled->part < 0)
                return ERR_PTR(-EINVAL);
        perf = &cxlds->part[cxled->part].perf;

        if (!perf)
                return ERR_PTR(-EINVAL);

        if (!dpa_perf_contains(perf, cxled->dpa_res))
                return ERR_PTR(-EINVAL);

        return perf;
}

/*
 * Transient context for containing the current calculation of bandwidth when
 * doing walking the port hierarchy to deal with shared upstream link.
 */
struct cxl_perf_ctx {
        struct access_coordinate coord[ACCESS_COORDINATE_MAX];
        struct cxl_port *port;
};

/**
 * cxl_endpoint_gather_bandwidth - collect all the endpoint bandwidth in an xarray
 * @cxlr: CXL region for the bandwidth calculation
 * @cxled: endpoint decoder to start on
 * @usp_xa: (output) the xarray that collects all the bandwidth coordinates
 *          indexed by the upstream device with data of 'struct cxl_perf_ctx'.
 * @gp_is_root: (output) bool of whether the grandparent is cxl root.
 *
 * Return: 0 for success or -errno
 *
 * Collects aggregated endpoint bandwidth and store the bandwidth in
 * an xarray indexed by the upstream device of the switch or the RP
 * device. Each endpoint consists the minimum of the bandwidth from DSLBIS
 * from the endpoint CDAT, the endpoint upstream link bandwidth, and the
 * bandwidth from the SSLBIS of the switch CDAT for the switch upstream port to
 * the downstream port that's associated with the endpoint. If the
 * device is directly connected to a RP, then no SSLBIS is involved.
 */
static int cxl_endpoint_gather_bandwidth(struct cxl_region *cxlr,
                                         struct cxl_endpoint_decoder *cxled,
                                         struct xarray *usp_xa,
                                         bool *gp_is_root)
{
        struct cxl_port *endpoint = to_cxl_port(cxled->cxld.dev.parent);
        struct cxl_port *parent_port = to_cxl_port(endpoint->dev.parent);
        struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent);
        struct access_coordinate pci_coord[ACCESS_COORDINATE_MAX];
        struct access_coordinate sw_coord[ACCESS_COORDINATE_MAX];
        struct access_coordinate ep_coord[ACCESS_COORDINATE_MAX];
        struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
        struct cxl_dev_state *cxlds = cxlmd->cxlds;
        struct pci_dev *pdev = to_pci_dev(cxlds->dev);
        struct cxl_perf_ctx *perf_ctx;
        struct cxl_dpa_perf *perf;
        unsigned long index;
        void *ptr;
        int rc;

        if (!dev_is_pci(cxlds->dev))
                return -ENODEV;

        if (cxlds->rcd)
                return -ENODEV;

        perf = cxled_get_dpa_perf(cxled);
        if (IS_ERR(perf))
                return PTR_ERR(perf);

        *gp_is_root = is_cxl_root(gp_port);

        /*
         * If the grandparent is cxl root, then index is the root port,
         * otherwise it's the parent switch upstream device.
         */
        if (*gp_is_root)
                index = (unsigned long)endpoint->parent_dport->dport_dev;
        else
                index = (unsigned long)parent_port->uport_dev;

        perf_ctx = xa_load(usp_xa, index);
        if (!perf_ctx) {
                struct cxl_perf_ctx *c __free(kfree) =
                        kzalloc_obj(*perf_ctx);

                if (!c)
                        return -ENOMEM;
                ptr = xa_store(usp_xa, index, c, GFP_KERNEL);
                if (xa_is_err(ptr))
                        return xa_err(ptr);
                perf_ctx = no_free_ptr(c);
                perf_ctx->port = parent_port;
        }

        /* Direct upstream link from EP bandwidth */
        rc = cxl_pci_get_bandwidth(pdev, pci_coord);
        if (rc < 0)
                return rc;

        /*
         * Min of upstream link bandwidth and Endpoint CDAT bandwidth from
         * DSLBIS.
         */
        cxl_coordinates_combine(ep_coord, pci_coord, perf->cdat_coord);

        /*
         * If grandparent port is root, then there's no switch involved and
         * the endpoint is connected to a root port.
         */
        if (!*gp_is_root) {
                /*
                 * Retrieve the switch SSLBIS for switch downstream port
                 * associated with the endpoint bandwidth.
                 */
                rc = cxl_port_get_switch_dport_bandwidth(endpoint, sw_coord);
                if (rc)
                        return rc;

                /*
                 * Min of the earlier coordinates with the switch SSLBIS
                 * bandwidth
                 */
                cxl_coordinates_combine(ep_coord, ep_coord, sw_coord);
        }

        /*
         * Aggregate the computed bandwidth with the current aggregated bandwidth
         * of the endpoints with the same switch upstream device or RP.
         */
        cxl_bandwidth_add(perf_ctx->coord, perf_ctx->coord, ep_coord);

        return 0;
}

static void free_perf_xa(struct xarray *xa)
{
        struct cxl_perf_ctx *ctx;
        unsigned long index;

        if (!xa)
                return;

        xa_for_each(xa, index, ctx)
                kfree(ctx);
        xa_destroy(xa);
        kfree(xa);
}
DEFINE_FREE(free_perf_xa, struct xarray *, if (_T) free_perf_xa(_T))

/**
 * cxl_switch_gather_bandwidth - collect all the bandwidth at switch level in an xarray
 * @cxlr: The region being operated on
 * @input_xa: xarray indexed by upstream device of a switch with data of 'struct
 *            cxl_perf_ctx'
 * @gp_is_root: (output) bool of whether the grandparent is cxl root.
 *
 * Return: a xarray of resulting cxl_perf_ctx per parent switch or root port
 *         or ERR_PTR(-errno)
 *
 * Iterate through the xarray. Take the minimum of the downstream calculated
 * bandwidth, the upstream link bandwidth, and the SSLBIS of the upstream
 * switch if exists. Sum the resulting bandwidth under the switch upstream
 * device or a RP device. The function can be iterated over multiple switches
 * if the switches are present.
 */
static struct xarray *cxl_switch_gather_bandwidth(struct cxl_region *cxlr,
                                                  struct xarray *input_xa,
                                                  bool *gp_is_root)
{
        struct xarray *res_xa __free(free_perf_xa) =
                kzalloc_obj(*res_xa);
        struct access_coordinate coords[ACCESS_COORDINATE_MAX];
        struct cxl_perf_ctx *ctx, *us_ctx;
        unsigned long index, us_index;
        int dev_count = 0;
        int gp_count = 0;
        void *ptr;
        int rc;

        if (!res_xa)
                return ERR_PTR(-ENOMEM);
        xa_init(res_xa);

        xa_for_each(input_xa, index, ctx) {
                struct device *dev = (struct device *)index;
                struct cxl_port *port = ctx->port;
                struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
                struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent);
                struct cxl_dport *dport = port->parent_dport;
                bool is_root = false;

                dev_count++;
                if (is_cxl_root(gp_port)) {
                        is_root = true;
                        gp_count++;
                }

                /*
                 * If the grandparent is cxl root, then index is the root port,
                 * otherwise it's the parent switch upstream device.
                 */
                if (is_root)
                        us_index = (unsigned long)port->parent_dport->dport_dev;
                else
                        us_index = (unsigned long)parent_port->uport_dev;

                us_ctx = xa_load(res_xa, us_index);
                if (!us_ctx) {
                        struct cxl_perf_ctx *n __free(kfree) =
                                kzalloc_obj(*n);

                        if (!n)
                                return ERR_PTR(-ENOMEM);

                        ptr = xa_store(res_xa, us_index, n, GFP_KERNEL);
                        if (xa_is_err(ptr))
                                return ERR_PTR(xa_err(ptr));
                        us_ctx = no_free_ptr(n);
                        us_ctx->port = parent_port;
                }

                /*
                 * If the device isn't an upstream PCIe port, there's something
                 * wrong with the topology.
                 */
                if (!dev_is_pci(dev))
                        return ERR_PTR(-EINVAL);

                /* Retrieve the upstream link bandwidth */
                rc = cxl_pci_get_bandwidth(to_pci_dev(dev), coords);
                if (rc)
                        return ERR_PTR(-ENXIO);

                /*
                 * Take the min of downstream bandwidth and the upstream link
                 * bandwidth.
                 */
                cxl_coordinates_combine(coords, coords, ctx->coord);

                /*
                 * Take the min of the calculated bandwidth and the upstream
                 * switch SSLBIS bandwidth if there's a parent switch
                 */
                if (!is_root)
                        cxl_coordinates_combine(coords, coords, dport->coord);

                /*
                 * Aggregate the calculated bandwidth common to an upstream
                 * switch.
                 */
                cxl_bandwidth_add(us_ctx->coord, us_ctx->coord, coords);
        }

        /* Asymmetric topology detected. */
        if (gp_count) {
                if (gp_count != dev_count) {
                        dev_dbg(&cxlr->dev,
                                "Asymmetric hierarchy detected, bandwidth not updated\n");
                        return ERR_PTR(-EOPNOTSUPP);
                }
                *gp_is_root = true;
        }

        return no_free_ptr(res_xa);
}

/**
 * cxl_rp_gather_bandwidth - handle the root port level bandwidth collection
 * @xa: the xarray that holds the cxl_perf_ctx that has the bandwidth calculated
 *      below each root port device.
 *
 * Return: xarray that holds cxl_perf_ctx per host bridge or ERR_PTR(-errno)
 */
static struct xarray *cxl_rp_gather_bandwidth(struct xarray *xa)
{
        struct xarray *hb_xa __free(free_perf_xa) =
                kzalloc_obj(*hb_xa);
        struct cxl_perf_ctx *ctx;
        unsigned long index;

        if (!hb_xa)
                return ERR_PTR(-ENOMEM);
        xa_init(hb_xa);

        xa_for_each(xa, index, ctx) {
                struct cxl_port *port = ctx->port;
                unsigned long hb_index = (unsigned long)port->uport_dev;
                struct cxl_perf_ctx *hb_ctx;
                void *ptr;

                hb_ctx = xa_load(hb_xa, hb_index);
                if (!hb_ctx) {
                        struct cxl_perf_ctx *n __free(kfree) =
                                kzalloc_obj(*n);

                        if (!n)
                                return ERR_PTR(-ENOMEM);
                        ptr = xa_store(hb_xa, hb_index, n, GFP_KERNEL);
                        if (xa_is_err(ptr))
                                return ERR_PTR(xa_err(ptr));
                        hb_ctx = no_free_ptr(n);
                        hb_ctx->port = port;
                }

                cxl_bandwidth_add(hb_ctx->coord, hb_ctx->coord, ctx->coord);
        }

        return no_free_ptr(hb_xa);
}

/**
 * cxl_hb_gather_bandwidth - handle the host bridge level bandwidth collection
 * @xa: the xarray that holds the cxl_perf_ctx that has the bandwidth calculated
 *      below each host bridge.
 *
 * Return: xarray that holds cxl_perf_ctx per ACPI0017 device or ERR_PTR(-errno)
 */
static struct xarray *cxl_hb_gather_bandwidth(struct xarray *xa)
{
        struct xarray *mw_xa __free(free_perf_xa) =
                kzalloc_obj(*mw_xa);
        struct cxl_perf_ctx *ctx;
        unsigned long index;

        if (!mw_xa)
                return ERR_PTR(-ENOMEM);
        xa_init(mw_xa);

        xa_for_each(xa, index, ctx) {
                struct cxl_port *port = ctx->port;
                struct cxl_port *parent_port;
                struct cxl_perf_ctx *mw_ctx;
                struct cxl_dport *dport;
                unsigned long mw_index;
                void *ptr;

                parent_port = to_cxl_port(port->dev.parent);
                mw_index = (unsigned long)parent_port->uport_dev;

                mw_ctx = xa_load(mw_xa, mw_index);
                if (!mw_ctx) {
                        struct cxl_perf_ctx *n __free(kfree) =
                                kzalloc_obj(*n);

                        if (!n)
                                return ERR_PTR(-ENOMEM);
                        ptr = xa_store(mw_xa, mw_index, n, GFP_KERNEL);
                        if (xa_is_err(ptr))
                                return ERR_PTR(xa_err(ptr));
                        mw_ctx = no_free_ptr(n);
                }

                dport = port->parent_dport;
                cxl_coordinates_combine(ctx->coord, ctx->coord, dport->coord);
                cxl_bandwidth_add(mw_ctx->coord, mw_ctx->coord, ctx->coord);
        }

        return no_free_ptr(mw_xa);
}

/**
 * cxl_region_update_bandwidth - Update the bandwidth access coordinates of a region
 * @cxlr: The region being operated on
 * @input_xa: xarray holds cxl_perf_ctx with calculated bandwidth per ACPI0017 instance
 */
static void cxl_region_update_bandwidth(struct cxl_region *cxlr,
                                        struct xarray *input_xa)
{
        struct access_coordinate coord[ACCESS_COORDINATE_MAX];
        struct cxl_perf_ctx *ctx;
        unsigned long index;

        memset(coord, 0, sizeof(coord));
        xa_for_each(input_xa, index, ctx)
                cxl_bandwidth_add(coord, coord, ctx->coord);

        for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
                cxlr->coord[i].read_bandwidth = coord[i].read_bandwidth;
                cxlr->coord[i].write_bandwidth = coord[i].write_bandwidth;
        }
}

/**
 * cxl_region_shared_upstream_bandwidth_update - Recalculate the bandwidth for
 *                                               the region
 * @cxlr: the cxl region to recalculate
 *
 * The function walks the topology from bottom up and calculates the bandwidth. It
 * starts at the endpoints, processes at the switches if any, processes at the rootport
 * level, at the host bridge level, and finally aggregates at the region.
 */
void cxl_region_shared_upstream_bandwidth_update(struct cxl_region *cxlr)
{
        struct xarray *working_xa;
        int root_count = 0;
        bool is_root;
        int rc;

        lockdep_assert_held(&cxl_rwsem.dpa);

        struct xarray *usp_xa __free(free_perf_xa) =
                kzalloc_obj(*usp_xa);

        if (!usp_xa)
                return;

        xa_init(usp_xa);

        /* Collect bandwidth data from all the endpoints. */
        for (int i = 0; i < cxlr->params.nr_targets; i++) {
                struct cxl_endpoint_decoder *cxled = cxlr->params.targets[i];

                is_root = false;
                rc = cxl_endpoint_gather_bandwidth(cxlr, cxled, usp_xa, &is_root);
                if (rc)
                        return;
                root_count += is_root;
        }

        /* Detect asymmetric hierarchy with some direct attached endpoints. */
        if (root_count && root_count != cxlr->params.nr_targets) {
                dev_dbg(&cxlr->dev,
                        "Asymmetric hierarchy detected, bandwidth not updated\n");
                return;
        }

        /*
         * Walk up one or more switches to deal with the bandwidth of the
         * switches if they exist. Endpoints directly attached to RPs skip
         * over this part.
         */
        if (!root_count) {
                do {
                        working_xa = cxl_switch_gather_bandwidth(cxlr, usp_xa,
                                                                 &is_root);
                        if (IS_ERR(working_xa))
                                return;
                        free_perf_xa(usp_xa);
                        usp_xa = working_xa;
                } while (!is_root);
        }

        /* Handle the bandwidth at the root port of the hierarchy */
        working_xa = cxl_rp_gather_bandwidth(usp_xa);
        if (IS_ERR(working_xa))
                return;
        free_perf_xa(usp_xa);
        usp_xa = working_xa;

        /* Handle the bandwidth at the host bridge of the hierarchy */
        working_xa = cxl_hb_gather_bandwidth(usp_xa);
        if (IS_ERR(working_xa))
                return;
        free_perf_xa(usp_xa);
        usp_xa = working_xa;

        /*
         * Aggregate all the bandwidth collected per CFMWS (ACPI0017) and
         * update the region bandwidth with the final calculated values.
         */
        cxl_region_update_bandwidth(cxlr, usp_xa);
}

void cxl_region_perf_data_calculate(struct cxl_region *cxlr,
                                    struct cxl_endpoint_decoder *cxled)
{
        struct cxl_dpa_perf *perf;

        lockdep_assert_held(&cxl_rwsem.dpa);

        perf = cxled_get_dpa_perf(cxled);
        if (IS_ERR(perf))
                return;

        for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
                /* Get total bandwidth and the worst latency for the cxl region */
                cxlr->coord[i].read_latency = max_t(unsigned int,
                                                    cxlr->coord[i].read_latency,
                                                    perf->coord[i].read_latency);
                cxlr->coord[i].write_latency = max_t(unsigned int,
                                                     cxlr->coord[i].write_latency,
                                                     perf->coord[i].write_latency);
                cxlr->coord[i].read_bandwidth += perf->coord[i].read_bandwidth;
                cxlr->coord[i].write_bandwidth += perf->coord[i].write_bandwidth;
        }
}