// SPDX-License-Identifier: MIT
/*
 * Copyright © 2025 Intel Corporation
 */

#include <linux/bitmap.h>
#include <linux/fault-inject.h>

#include "regs/xe_gsc_regs.h"
#include "regs/xe_hw_error_regs.h"
#include "regs/xe_irq_regs.h"

#include "xe_device.h"
#include "xe_drm_ras.h"
#include "xe_hw_error.h"
#include "xe_mmio.h"
#include "xe_survivability_mode.h"

#define GT_HW_ERROR_MAX_ERR_BITS                16
#define HEC_UNCORR_FW_ERR_BITS                  4
#define XE_RAS_REG_SIZE                         32
#define XE_SOC_NUM_IEH                          2

#define PVC_ERROR_MASK_SET(hw_err, err_bit)     ((hw_err == HARDWARE_ERROR_CORRECTABLE) ? \
                                                 (PVC_COR_ERR_MASK & REG_BIT(err_bit)) : \
                                                 (PVC_FAT_ERR_MASK & REG_BIT(err_bit)))

extern struct fault_attr inject_csc_hw_error;

static const char * const error_severity[] = DRM_XE_RAS_ERROR_SEVERITY_NAMES;

static const char * const hec_uncorrected_fw_errors[] = {
        "Fatal",
        "CSE Disabled",
        "FD Corruption",
        "Data Corruption"
};

enum gt_vector_regs {
        ERR_STAT_GT_VECTOR0 = 0,
        ERR_STAT_GT_VECTOR1,
        ERR_STAT_GT_VECTOR2,
        ERR_STAT_GT_VECTOR3,
        ERR_STAT_GT_VECTOR4,
        ERR_STAT_GT_VECTOR5,
        ERR_STAT_GT_VECTOR6,
        ERR_STAT_GT_VECTOR7,
        ERR_STAT_GT_VECTOR_MAX
};

#define PVC_GT_VECTOR_LEN(hw_err)       ((hw_err == HARDWARE_ERROR_CORRECTABLE) ? \
                                         ERR_STAT_GT_VECTOR4 : ERR_STAT_GT_VECTOR_MAX)

static enum drm_xe_ras_error_severity hw_err_to_severity(const enum hardware_error hw_err)
{
        if (hw_err == HARDWARE_ERROR_CORRECTABLE)
                return DRM_XE_RAS_ERR_SEV_CORRECTABLE;

        /* Uncorrectable errors comprise of both fatal and non-fatal errors */
        return DRM_XE_RAS_ERR_SEV_UNCORRECTABLE;
}

static inline u32 err_src_to_id(u32 err_bit)
{
        switch (err_bit) {
        case XE_GT_ERROR:
                return DRM_XE_RAS_ERR_COMP_CORE_COMPUTE;
        case XE_SOC_ERROR:
                return DRM_XE_RAS_ERR_COMP_SOC_INTERNAL;
        default:
                return 0;
        }
}

static const char * const pvc_master_global_err_reg[] = {
        [0 ... 1]       = "Undefined",
        [2]             = "HBM SS0: Channel0",
        [3]             = "HBM SS0: Channel1",
        [4]             = "HBM SS0: Channel2",
        [5]             = "HBM SS0: Channel3",
        [6]             = "HBM SS0: Channel4",
        [7]             = "HBM SS0: Channel5",
        [8]             = "HBM SS0: Channel6",
        [9]             = "HBM SS0: Channel7",
        [10]            = "HBM SS1: Channel0",
        [11]            = "HBM SS1: Channel1",
        [12]            = "HBM SS1: Channel2",
        [13]            = "HBM SS1: Channel3",
        [14]            = "HBM SS1: Channel4",
        [15]            = "HBM SS1: Channel5",
        [16]            = "HBM SS1: Channel6",
        [17]            = "HBM SS1: Channel7",
        [18 ... 31]     = "Undefined",
};
static_assert(ARRAY_SIZE(pvc_master_global_err_reg) == XE_RAS_REG_SIZE);

static const char * const pvc_slave_global_err_reg[] = {
        [0]             = "Undefined",
        [1]             = "HBM SS2: Channel0",
        [2]             = "HBM SS2: Channel1",
        [3]             = "HBM SS2: Channel2",
        [4]             = "HBM SS2: Channel3",
        [5]             = "HBM SS2: Channel4",
        [6]             = "HBM SS2: Channel5",
        [7]             = "HBM SS2: Channel6",
        [8]             = "HBM SS2: Channel7",
        [9]             = "HBM SS3: Channel0",
        [10]            = "HBM SS3: Channel1",
        [11]            = "HBM SS3: Channel2",
        [12]            = "HBM SS3: Channel3",
        [13]            = "HBM SS3: Channel4",
        [14]            = "HBM SS3: Channel5",
        [15]            = "HBM SS3: Channel6",
        [16]            = "HBM SS3: Channel7",
        [17]            = "Undefined",
        [18]            = "ANR MDFI",
        [19 ... 31]     = "Undefined",
};
static_assert(ARRAY_SIZE(pvc_slave_global_err_reg) == XE_RAS_REG_SIZE);

static const char * const pvc_slave_local_fatal_err_reg[] = {
        [0]             = "Local IEH: Malformed PCIe AER",
        [1]             = "Local IEH: Malformed PCIe ERR",
        [2]             = "Local IEH: UR conditions in IEH",
        [3]             = "Local IEH: From SERR Sources",
        [4 ... 19]      = "Undefined",
        [20]            = "Malformed MCA error packet (HBM/Punit)",
        [21 ... 31]     = "Undefined",
};
static_assert(ARRAY_SIZE(pvc_slave_local_fatal_err_reg) == XE_RAS_REG_SIZE);

static const char * const pvc_master_local_fatal_err_reg[] = {
        [0]             = "Local IEH: Malformed IOSF PCIe AER",
        [1]             = "Local IEH: Malformed IOSF PCIe ERR",
        [2]             = "Local IEH: UR RESPONSE",
        [3]             = "Local IEH: From SERR SPI controller",
        [4]             = "Base Die MDFI T2T",
        [5]             = "Undefined",
        [6]             = "Base Die MDFI T2C",
        [7]             = "Undefined",
        [8]             = "Invalid CSC PSF Command Parity",
        [9]             = "Invalid CSC PSF Unexpected Completion",
        [10]            = "Invalid CSC PSF Unsupported Request",
        [11]            = "Invalid PCIe PSF Command Parity",
        [12]            = "PCIe PSF Unexpected Completion",
        [13]            = "PCIe PSF Unsupported Request",
        [14 ... 19]     = "Undefined",
        [20]            = "Malformed MCA error packet (HBM/Punit)",
        [21 ... 31]     = "Undefined",
};
static_assert(ARRAY_SIZE(pvc_master_local_fatal_err_reg) == XE_RAS_REG_SIZE);

static const char * const pvc_master_local_nonfatal_err_reg[] = {
        [0 ... 3]       = "Undefined",
        [4]             = "Base Die MDFI T2T",
        [5]             = "Undefined",
        [6]             = "Base Die MDFI T2C",
        [7]             = "Undefined",
        [8]             = "Invalid CSC PSF Command Parity",
        [9]             = "Invalid CSC PSF Unexpected Completion",
        [10]            = "Invalid PCIe PSF Command Parity",
        [11 ... 31]     = "Undefined",
};
static_assert(ARRAY_SIZE(pvc_master_local_nonfatal_err_reg) == XE_RAS_REG_SIZE);

#define PVC_MASTER_LOCAL_REG_INFO(hw_err)       ((hw_err == HARDWARE_ERROR_FATAL) ? \
                                                 pvc_master_local_fatal_err_reg : \
                                                 pvc_master_local_nonfatal_err_reg)

static bool fault_inject_csc_hw_error(void)
{
        return IS_ENABLED(CONFIG_DEBUG_FS) && should_fail(&inject_csc_hw_error, 1);
}

static void csc_hw_error_work(struct work_struct *work)
{
        struct xe_tile *tile = container_of(work, typeof(*tile), csc_hw_error_work);
        struct xe_device *xe = tile_to_xe(tile);

        xe_survivability_mode_runtime_enable(xe);
}

static void csc_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err)
{
        const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err);
        const char *severity_str = error_severity[severity];
        struct xe_device *xe = tile_to_xe(tile);
        struct xe_mmio *mmio = &tile->mmio;
        u32 base, err_bit, err_src;
        unsigned long fw_err;

        if (xe->info.platform != XE_BATTLEMAGE)
                return;

        base = BMG_GSC_HECI1_BASE;
        lockdep_assert_held(&xe->irq.lock);
        err_src = xe_mmio_read32(mmio, HEC_UNCORR_ERR_STATUS(base));
        if (!err_src) {
                drm_err_ratelimited(&xe->drm, HW_ERR "Tile%d reported %s HEC_ERR_STATUS register blank\n",
                                    tile->id, severity_str);
                return;
        }

        if (err_src & UNCORR_FW_REPORTED_ERR) {
                fw_err = xe_mmio_read32(mmio, HEC_UNCORR_FW_ERR_DW0(base));
                for_each_set_bit(err_bit, &fw_err, HEC_UNCORR_FW_ERR_BITS) {
                        drm_err_ratelimited(&xe->drm, HW_ERR
                                            "HEC FW %s %s reported, bit[%d] is set\n",
                                             hec_uncorrected_fw_errors[err_bit], severity_str,
                                             err_bit);

                        schedule_work(&tile->csc_hw_error_work);
                }
        }

        xe_mmio_write32(mmio, HEC_UNCORR_ERR_STATUS(base), err_src);
}

static void log_hw_error(struct xe_tile *tile, const char *name,
                         const enum drm_xe_ras_error_severity severity)
{
        const char *severity_str = error_severity[severity];
        struct xe_device *xe = tile_to_xe(tile);

        if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE)
                drm_warn(&xe->drm, HW_ERR "%s %s detected\n", name, severity_str);
        else
                drm_err_ratelimited(&xe->drm, HW_ERR "%s %s detected\n", name, severity_str);
}

static void log_gt_err(struct xe_tile *tile, const char *name, int i, u32 err,
                       const enum drm_xe_ras_error_severity severity)
{
        const char *severity_str = error_severity[severity];
        struct xe_device *xe = tile_to_xe(tile);

        if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE)
                drm_warn(&xe->drm, HW_ERR "%s %s detected, ERROR_STAT_GT_VECTOR%d:0x%08x\n",
                         name, severity_str, i, err);
        else
                drm_err_ratelimited(&xe->drm, HW_ERR "%s %s detected, ERROR_STAT_GT_VECTOR%d:0x%08x\n",
                                    name, severity_str, i, err);
}

static void log_soc_error(struct xe_tile *tile, const char * const *reg_info,
                          const enum drm_xe_ras_error_severity severity, u32 err_bit, u32 index)
{
        const char *severity_str = error_severity[severity];
        struct xe_device *xe = tile_to_xe(tile);
        struct xe_drm_ras *ras = &xe->ras;
        struct xe_drm_ras_counter *info = ras->info[severity];
        const char *name;

        name = reg_info[err_bit];

        if (strcmp(name, "Undefined")) {
                if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE)
                        drm_warn(&xe->drm, HW_ERR "%s SOC %s detected", name, severity_str);
                else
                        drm_err_ratelimited(&xe->drm, HW_ERR "%s SOC %s detected", name, severity_str);
                atomic_inc(&info[index].counter);
        }
}

static void gt_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err,
                                u32 error_id)
{
        const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err);
        struct xe_device *xe = tile_to_xe(tile);
        struct xe_drm_ras *ras = &xe->ras;
        struct xe_drm_ras_counter *info = ras->info[severity];
        struct xe_mmio *mmio = &tile->mmio;
        unsigned long err_stat = 0;
        int i;

        if (xe->info.platform != XE_PVC)
                return;

        if (hw_err == HARDWARE_ERROR_NONFATAL) {
                atomic_inc(&info[error_id].counter);
                log_hw_error(tile, info[error_id].name, severity);
                return;
        }

        for (i = 0; i < PVC_GT_VECTOR_LEN(hw_err); i++) {
                u32 vector, val;

                vector = xe_mmio_read32(mmio, ERR_STAT_GT_VECTOR_REG(hw_err, i));
                if (!vector)
                        continue;

                switch (i) {
                case ERR_STAT_GT_VECTOR0:
                case ERR_STAT_GT_VECTOR1: {
                        u32 errbit;

                        val = hweight32(vector);
                        atomic_add(val, &info[error_id].counter);
                        log_gt_err(tile, "Subslice", i, vector, severity);

                        /*
                         * Error status register is only populated once per error.
                         * Read the register and clear once.
                         */
                        if (err_stat)
                                break;

                        err_stat = xe_mmio_read32(mmio, ERR_STAT_GT_REG(hw_err));
                        for_each_set_bit(errbit, &err_stat, GT_HW_ERROR_MAX_ERR_BITS) {
                                if (PVC_ERROR_MASK_SET(hw_err, errbit))
                                        atomic_inc(&info[error_id].counter);
                        }
                        if (err_stat)
                                xe_mmio_write32(mmio, ERR_STAT_GT_REG(hw_err), err_stat);
                        break;
                }
                case ERR_STAT_GT_VECTOR2:
                case ERR_STAT_GT_VECTOR3:
                        val = hweight32(vector);
                        atomic_add(val, &info[error_id].counter);
                        log_gt_err(tile, "L3 BANK", i, vector, severity);
                        break;
                case ERR_STAT_GT_VECTOR6:
                        val = hweight32(vector);
                        atomic_add(val, &info[error_id].counter);
                        log_gt_err(tile, "TLB", i, vector, severity);
                        break;
                case ERR_STAT_GT_VECTOR7:
                        val = hweight32(vector);
                        atomic_add(val, &info[error_id].counter);
                        log_gt_err(tile, "L3 Fabric", i, vector, severity);
                        break;
                default:
                        log_gt_err(tile, "Undefined", i, vector, severity);
                }

                xe_mmio_write32(mmio, ERR_STAT_GT_VECTOR_REG(hw_err, i), vector);
        }
}

static void soc_slave_ieh_handler(struct xe_tile *tile, const enum hardware_error hw_err, u32 error_id)
{
        const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err);
        unsigned long slave_global_errstat, slave_local_errstat;
        struct xe_mmio *mmio = &tile->mmio;
        u32 regbit, slave;

        slave = SOC_PVC_SLAVE_BASE;
        slave_global_errstat = xe_mmio_read32(mmio, SOC_GLOBAL_ERR_STAT_REG(slave, hw_err));

        if (slave_global_errstat & SOC_IEH1_LOCAL_ERR_STATUS) {
                slave_local_errstat = xe_mmio_read32(mmio, SOC_LOCAL_ERR_STAT_REG(slave, hw_err));

                if (hw_err == HARDWARE_ERROR_FATAL) {
                        for_each_set_bit(regbit, &slave_local_errstat, XE_RAS_REG_SIZE)
                                log_soc_error(tile, pvc_slave_local_fatal_err_reg, severity,
                                              regbit, error_id);
                }

                xe_mmio_write32(mmio, SOC_LOCAL_ERR_STAT_REG(slave, hw_err),
                                slave_local_errstat);
        }

        for_each_set_bit(regbit, &slave_global_errstat, XE_RAS_REG_SIZE)
                log_soc_error(tile, pvc_slave_global_err_reg, severity, regbit, error_id);

        xe_mmio_write32(mmio, SOC_GLOBAL_ERR_STAT_REG(slave, hw_err), slave_global_errstat);
}

static void soc_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err,
                                 u32 error_id)
{
        const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err);
        struct xe_device *xe = tile_to_xe(tile);
        struct xe_mmio *mmio = &tile->mmio;
        unsigned long master_global_errstat, master_local_errstat;
        u32 master, slave, regbit;
        int i;

        if (xe->info.platform != XE_PVC)
                return;

        master = SOC_PVC_MASTER_BASE;
        slave = SOC_PVC_SLAVE_BASE;

        /* Mask error type in GSYSEVTCTL so that no new errors of the type will be reported */
        for (i = 0; i < XE_SOC_NUM_IEH; i++)
                xe_mmio_write32(mmio, SOC_GSYSEVTCTL_REG(master, slave, i), ~REG_BIT(hw_err));

        if (hw_err == HARDWARE_ERROR_CORRECTABLE) {
                xe_mmio_write32(mmio, SOC_GLOBAL_ERR_STAT_REG(master, hw_err), REG_GENMASK(31, 0));
                xe_mmio_write32(mmio, SOC_LOCAL_ERR_STAT_REG(master, hw_err), REG_GENMASK(31, 0));
                xe_mmio_write32(mmio, SOC_GLOBAL_ERR_STAT_REG(slave, hw_err), REG_GENMASK(31, 0));
                xe_mmio_write32(mmio, SOC_LOCAL_ERR_STAT_REG(slave, hw_err), REG_GENMASK(31, 0));
                goto unmask_gsysevtctl;
        }

        /*
         * Read the master global IEH error register, if BIT(1) is set then process
         * the slave IEH first. If BIT(0) in global error register is set then process
         * the corresponding local error registers.
         */
        master_global_errstat = xe_mmio_read32(mmio, SOC_GLOBAL_ERR_STAT_REG(master, hw_err));
        if (master_global_errstat & SOC_SLAVE_IEH)
                soc_slave_ieh_handler(tile, hw_err, error_id);

        if (master_global_errstat & SOC_IEH0_LOCAL_ERR_STATUS) {
                master_local_errstat = xe_mmio_read32(mmio, SOC_LOCAL_ERR_STAT_REG(master, hw_err));

                for_each_set_bit(regbit, &master_local_errstat, XE_RAS_REG_SIZE)
                        log_soc_error(tile, PVC_MASTER_LOCAL_REG_INFO(hw_err), severity, regbit, error_id);

                xe_mmio_write32(mmio, SOC_LOCAL_ERR_STAT_REG(master, hw_err), master_local_errstat);
        }

        for_each_set_bit(regbit, &master_global_errstat, XE_RAS_REG_SIZE)
                log_soc_error(tile, pvc_master_global_err_reg, severity, regbit, error_id);

        xe_mmio_write32(mmio, SOC_GLOBAL_ERR_STAT_REG(master, hw_err), master_global_errstat);

unmask_gsysevtctl:
        for (i = 0; i < XE_SOC_NUM_IEH; i++)
                xe_mmio_write32(mmio, SOC_GSYSEVTCTL_REG(master, slave, i),
                                (HARDWARE_ERROR_MAX << 1) + 1);
}

static void hw_error_source_handler(struct xe_tile *tile, const enum hardware_error hw_err)
{
        const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err);
        const char *severity_str = error_severity[severity];
        struct xe_device *xe = tile_to_xe(tile);
        struct xe_drm_ras *ras = &xe->ras;
        struct xe_drm_ras_counter *info = ras->info[severity];
        unsigned long flags, err_src;
        u32 err_bit;

        if (!IS_DGFX(xe))
                return;

        spin_lock_irqsave(&xe->irq.lock, flags);
        err_src = xe_mmio_read32(&tile->mmio, DEV_ERR_STAT_REG(hw_err));
        if (!err_src) {
                drm_err_ratelimited(&xe->drm, HW_ERR "Tile%d reported %s DEV_ERR_STAT register blank!\n",
                                    tile->id, severity_str);
                goto unlock;
        }

        /*
         * On encountering CSC firmware errors, the graphics device becomes unrecoverable
         * so return immediately on error. The only way to recover from these errors is
         * firmware flash. The device will enter Runtime Survivability mode when such
         * errors are detected.
         */
        if (err_src & REG_BIT(XE_CSC_ERROR)) {
                csc_hw_error_handler(tile, hw_err);
                goto clear_reg;
        }

        if (!info)
                goto clear_reg;

        for_each_set_bit(err_bit, &err_src, XE_RAS_REG_SIZE) {
                const char *name;
                u32 error_id;

                error_id = err_src_to_id(err_bit);
                if (!error_id)
                        continue;

                name = info[error_id].name;
                if (!name)
                        continue;

                if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) {
                        drm_warn(&xe->drm, HW_ERR
                                 "TILE%d reported %s %s, bit[%d] is set\n",
                                 tile->id, name, severity_str, err_bit);
                } else {
                        drm_err_ratelimited(&xe->drm, HW_ERR
                                            "TILE%d reported %s %s, bit[%d] is set\n",
                                            tile->id, name, severity_str, err_bit);
                }

                if (err_bit == XE_GT_ERROR)
                        gt_hw_error_handler(tile, hw_err, error_id);
                if (err_bit == XE_SOC_ERROR)
                        soc_hw_error_handler(tile, hw_err, error_id);
        }

clear_reg:
        xe_mmio_write32(&tile->mmio, DEV_ERR_STAT_REG(hw_err), err_src);
unlock:
        spin_unlock_irqrestore(&xe->irq.lock, flags);
}

/**
 * xe_hw_error_irq_handler - irq handling for hw errors
 * @tile: tile instance
 * @master_ctl: value read from master interrupt register
 *
 * Xe platforms add three error bits to the master interrupt register to support error handling.
 * These three bits are used to convey the class of error FATAL, NONFATAL, or CORRECTABLE.
 * To process the interrupt, determine the source of error by reading the Device Error Source
 * Register that corresponds to the class of error being serviced.
 */
void xe_hw_error_irq_handler(struct xe_tile *tile, const u32 master_ctl)
{
        enum hardware_error hw_err;

        if (fault_inject_csc_hw_error())
                schedule_work(&tile->csc_hw_error_work);

        for (hw_err = 0; hw_err < HARDWARE_ERROR_MAX; hw_err++) {
                if (master_ctl & ERROR_IRQ(hw_err))
                        hw_error_source_handler(tile, hw_err);
        }
}

static int hw_error_info_init(struct xe_device *xe)
{
        if (xe->info.platform != XE_PVC)
                return 0;

        return xe_drm_ras_init(xe);
}

/*
 * Process hardware errors during boot
 */
static void process_hw_errors(struct xe_device *xe)
{
        struct xe_tile *tile;
        u32 master_ctl;
        u8 id;

        for_each_tile(tile, xe, id) {
                master_ctl = xe_mmio_read32(&tile->mmio, GFX_MSTR_IRQ);
                xe_hw_error_irq_handler(tile, master_ctl);
                xe_mmio_write32(&tile->mmio, GFX_MSTR_IRQ, master_ctl);
        }
}

/**
 * xe_hw_error_init - Initialize hw errors
 * @xe: xe device instance
 *
 * Initialize and check for errors that occurred during boot
 * prior to driver load
 */
void xe_hw_error_init(struct xe_device *xe)
{
        struct xe_tile *tile = xe_device_get_root_tile(xe);
        int ret;

        if (!IS_DGFX(xe) || IS_SRIOV_VF(xe))
                return;

        INIT_WORK(&tile->csc_hw_error_work, csc_hw_error_work);

        ret = hw_error_info_init(xe);
        if (ret)
                drm_err(&xe->drm, "Failed to initialize XE DRM RAS (%pe)\n", ERR_PTR(ret));

        process_hw_errors(xe);
}